未验证 提交 9f7d90d2 编写于 作者: H hong 提交者: GitHub

Load inference enhance (#21919)

* enhance load interface; test=develop

* add uni test and add comment; test=develop

* fix converage; test=develop

* use path.joint replace "/"; test=develop

* windows debug; test=develop

* fix window unitest error; test=develop

* fix commet error; test=develop

* add model shuffix check; test=develop

* fix example error; test=develop
上级 4c2df8e4
......@@ -55,6 +55,8 @@ __all__ = [
'load',
'load_program_state',
'set_program_state',
'get_program_parameter',
'get_program_persistable_vars',
] + reader.__all__ + paddle.reader.__all__
_logger = get_logger(
......@@ -114,6 +116,50 @@ def is_belong_to_optimizer(var):
return False
def get_program_parameter(program):
"""
Get all the parameters from Program.
Args:
var(Program): The Program to get parameters
Returns:
list: The list contains all parameters in the program
Examples:
.. code-block:: python
import paddle.fluid as fluid
data = fluid.data(name="img", shape=[64, 784])
w = fluid.layers.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
b = fluid.layers.create_parameter(shape=[200], dtype='float32', name='fc_b')
list_para = fluid.io.get_program_parameter( fluid.default_main_program() )
"""
return list(filter(is_parameter, program.list_vars()))
def get_program_persistable_vars(program):
"""
Get all the persistable vars from Program.
Args:
var(Program): The Program to get persistable vars
Returns:
list: The list contains all persistable vars in the program
Examples:
.. code-block:: python
import paddle.fluid as fluid
data = fluid.data(name="img", shape=[64, 784])
w = fluid.layers.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
b = fluid.layers.create_parameter(shape=[200], dtype='float32', name='fc_b')
list_para = fluid.io.get_program_persistable_vars( fluid.default_main_program() )
"""
return list(filter(is_persistable, program.list_vars()))
def _clone_var_in_block_(block, var):
assert isinstance(var, Variable)
if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR:
......@@ -1497,16 +1543,23 @@ def save(program, model_path):
f.write(program.desc.serialize_to_string())
def load(program, model_path, executor=None):
def load(program, model_path, executor=None, var_list=None):
"""
This function filter out parameters and optimizer information from program, and then get corresponding value from file.
This function get parameters and optimizer information from program, and then get corresponding value from file.
An exception will throw if shape or dtype of the parameters is not match.
This function can also load model file saved with [ save_params, save_persistables, save_vars ].
var_list can not be None when load single model file
( filename is not None When save_params, save_persistables or save_vars is called ).
Args:
program(Program): The program will be loaded
model_path(str): The file prefix store the program
executor(Executor, optional): The executor used for initialize the parameter
When startup program is not run.
var_list(list, optional): The variable list to load single model file saved with
[ save_params, save_persistables, save_vars ].
Default: None
Returns:
None
......@@ -1525,9 +1578,85 @@ def load(program, model_path, executor=None):
assert executor is None or isinstance(executor, Executor)
parameter_file_name = model_path + ".pdparams"
assert os.path.exists(parameter_file_name), \
"Parameter file [{}] not exits".format(parameter_file_name)
model_prefix = model_path
if model_prefix.endswith(".pdparams"):
model_prefix = model_prefix[:-9]
elif model_prefix.endswith(".pdopt"):
model_prefix = model_prefix[:-6]
elif model_prefix.endswith(".pdmodel"):
model_prefix = model_prefix[:-8]
parameter_file_name = model_prefix + ".pdparams"
if not os.path.exists(parameter_file_name):
# model file save by fluid.save not found, try to load model file saved with
# [save_vars, save_params, save_persistables]
_logger.warning(
"{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".
format(parameter_file_name))
if executor is None:
raise ValueError(
"executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
)
if os.path.isdir(model_path):
binary_file_set = set()
for root, dirs, files in os.walk(model_path, topdown=False):
for f in files:
binary_file_set.add(
os.path.join(root, f).replace("\\", "/"))
program_var_list = list(program.list_vars())
loaded_var_list = []
for var in program_var_list:
var_path = os.path.join(model_path, var.name).replace("\\", "/")
if var_path in binary_file_set:
loaded_var_list.append(var)
binary_file_set.remove(var_path)
if len(binary_file_set) > 0:
unused_var_list = " ".join(list(binary_file_set))
_logger.warning("variable file [ %s ] not used" %
(" ".join(list(binary_file_set))))
try:
load_vars(
executor=executor, dirname=model_path, vars=loaded_var_list)
except RuntimeError as e:
_logger.error(e)
raise e
except:
raise RuntimeError(
"Failed to load model file , please make sure model file is saved with the "
"following APIs: save_params, save_persistables, save_vars")
return
elif os.path.isfile(model_path):
if var_list == None:
raise ValueError(
"var_list is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
)
program_var_list = program.list_vars()
program_var_name_set = set([var.name for var in program_var_list])
# check all the variable inlcuded in program
for var in var_list:
if var.name not in program_var_name_set:
raise LookupError(
"loaded var [{}] not included in program variable list")
dir_name, file_name = os.path.split(model_path)
try:
load_vars(
executor=executor,
dirname=dir_name,
vars=var_list,
filename=file_name)
except RuntimeError as e:
_logger.error(e)
raise e
except:
raise RuntimeError( "Failed to load model file , please make sure model file is saved with the " \
"the following APIs: [ save_params, save_persistables, save_vars ]. " \
"When these API called, filename CANNOT be None")
return
def set_var(var, ndarray):
t = global_scope().find_var(var.name).get_tensor()
......@@ -1561,7 +1690,7 @@ def load(program, model_path, executor=None):
filter(is_belong_to_optimizer, program.list_vars()))
if len(optimizer_var_list) > 0:
opt_file_name = model_path + ".pdopt"
opt_file_name = model_prefix + ".pdopt"
assert os.path.exists(opt_file_name), \
"Optimizer file [{}] not exits".format(opt_file_name)
......@@ -1603,8 +1732,6 @@ def load_program_state(model_path):
fluid.save( prog, "./temp")
program_state = fluid.load_program_state( "./temp")
fluid.set_program_state( prog, program_state)
"""
parameter_file_name = model_path + ".pdparams"
assert os.path.exists(parameter_file_name), \
......@@ -1653,6 +1780,8 @@ def set_program_state(program, state_dict):
fluid.save( prog, "./temp")
program_state = fluid.load_program_state( "./temp")
fluid.set_program_state( prog, program_state)
"""
parameter_list = list(filter(is_persistable, program.list_vars()))
......
......@@ -27,6 +27,7 @@ from paddle.fluid.executor import global_scope
import numpy as np
import six
import pickle
import os
class SimpleLSTMRNN(fluid.Layer):
......@@ -37,7 +38,7 @@ class SimpleLSTMRNN(fluid.Layer):
num_layers=2,
init_scale=0.1,
dropout=None):
super(SimpleLSTMRNN, self).__init__(name_scope)
super(SimpleLSTMRNN, self).__init__()
self._hidden_size = hidden_size
self._num_layers = num_layers
self._init_scale = init_scale
......@@ -47,7 +48,6 @@ class SimpleLSTMRNN(fluid.Layer):
self.cell_array = []
self.hidden_array = []
def _build_once(self, input_embedding, init_hidden=None, init_cell=None):
self.weight_1_arr = []
self.weight_2_arr = []
self.bias_arr = []
......@@ -143,7 +143,7 @@ class PtbModel(fluid.Layer):
num_steps=20,
init_scale=0.1,
dropout=None):
super(PtbModel, self).__init__(name_scope)
super(PtbModel, self).__init__()
self.hidden_size = hidden_size
self.vocab_size = vocab_size
self.init_scale = init_scale
......@@ -302,7 +302,7 @@ class TestSaveLoadBase(unittest.TestCase):
# make sure all the paramerter or optimzier var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1", exe)
fluid.load(main_program, "./test_1.pdparams", exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -411,15 +411,15 @@ class TestSaveLoadPartial(unittest.TestCase):
# make sure all the paramerter or optimzier var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(test_program, "./test_1", None)
fluid.load(test_program, "./test_1.pdopt", None)
for var in test_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
print(var.name)
new_t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
fluid.load(test_program, "./test_1.pdmodel", None)
class TestSaveLoadSetStateDict(unittest.TestCase):
......@@ -628,7 +628,6 @@ class TestProgramStatePartial(unittest.TestCase):
for var in test_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
print(var.name)
new_t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor())
base_t = base_map[var.name]
......@@ -718,5 +717,271 @@ class TestVariableInit(unittest.TestCase):
self.assertTrue(np.array_equal(new_t, base_t))
class TestLoadFromOldInterface(unittest.TestCase):
def setUp(self):
if os.path.exists("test_path.pdparams"):
os.remove("test_path.pdparams")
def test_load_from_old_interface(self):
seed = 90
hidden_size = 10
vocab_size = 1000
num_layers = 1
num_steps = 3
init_scale = 0.1
batch_size = 4
batch_num = 200
with new_program_scope():
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
ptb_model = PtbModel(
"ptb_model",
hidden_size=hidden_size,
vocab_size=vocab_size,
num_layers=num_layers,
num_steps=num_steps,
init_scale=init_scale)
place = fluid.CPUPlace() if not core.is_compiled_with_cuda(
) else fluid.CUDAPlace(0)
exe = fluid.Executor(place)
sgd = Adam(learning_rate=1e-3)
x = fluid.layers.data(
name="x", shape=[-1, num_steps], dtype='int64')
y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32')
init_hidden = fluid.layers.data(
name="init_hidden", shape=[1], dtype='float32')
init_cell = fluid.layers.data(
name="init_cell", shape=[1], dtype='float32')
static_loss, static_last_hidden, static_last_cell = ptb_model(
x, y, init_hidden, init_cell)
test_clone_program = fluid.default_main_program().clone()
sgd.minimize(static_loss)
static_param_updated = dict()
static_param_init = dict()
out = exe.run(framework.default_startup_program())
static_loss_value = None
static_last_cell_value = None
static_last_hidden_value = None
for i in range(batch_num):
x_data = np.arange(12).reshape(4, 3).astype('int64')
y_data = np.arange(1, 13).reshape(4, 3).astype('int64')
x_data = x_data.reshape((-1, num_steps, 1))
y_data = y_data.reshape((-1, 1))
init_hidden_data = np.zeros(
(num_layers, batch_size, hidden_size), dtype='float32')
init_cell_data = np.zeros(
(num_layers, batch_size, hidden_size), dtype='float32')
fetch_list = [static_loss, static_last_hidden, static_last_cell]
out = exe.run(fluid.default_main_program(),
feed={
"x": x_data,
"y": y_data,
"init_hidden": init_hidden_data,
"init_cell": init_cell_data
},
fetch_list=fetch_list)
static_loss_value = out[0]
static_last_hidden_value = out[1]
static_last_cell_value = out[2]
# get value before save
main_program = framework.default_main_program()
base_map = {}
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor())
# make sure all the paramerter or optimzier var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
#fluid.save(main_program, "./test_1")
fluid.io.save_persistables(exe, "test_path", main_program)
# set var to zero
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
ten = fluid.global_scope().find_var(var.name).get_tensor()
ten.set(np.zeros_like(np.array(ten)), place)
new_t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor())
# make sure all the paramerter or optimzier var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "test_path", exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
new_t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
ten = fluid.global_scope().find_var(var.name).get_tensor()
old_shape = np.array(ten).shape
new_shape = [e + 10 for e in old_shape]
var.desc.set_shape(new_shape)
with self.assertRaises(RuntimeError):
fluid.load(main_program, "test_path", exe)
# check unused paramter
fluid.load(test_clone_program, "test_path", exe)
class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
def test_load_from_old_interface(self):
seed = 90
hidden_size = 10
vocab_size = 1000
num_layers = 1
num_steps = 3
init_scale = 0.1
batch_size = 4
batch_num = 200
with new_program_scope():
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
ptb_model = PtbModel(
"ptb_model",
hidden_size=hidden_size,
vocab_size=vocab_size,
num_layers=num_layers,
num_steps=num_steps,
init_scale=init_scale)
place = fluid.CPUPlace() if not core.is_compiled_with_cuda(
) else fluid.CUDAPlace(0)
exe = fluid.Executor(place)
sgd = Adam(learning_rate=1e-3)
x = fluid.layers.data(
name="x", shape=[-1, num_steps], dtype='int64')
y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32')
init_hidden = fluid.layers.data(
name="init_hidden", shape=[1], dtype='float32')
init_cell = fluid.layers.data(
name="init_cell", shape=[1], dtype='float32')
static_loss, static_last_hidden, static_last_cell = ptb_model(
x, y, init_hidden, init_cell)
sgd.minimize(static_loss)
static_param_updated = dict()
static_param_init = dict()
out = exe.run(framework.default_startup_program())
static_loss_value = None
static_last_cell_value = None
static_last_hidden_value = None
for i in range(batch_num):
x_data = np.arange(12).reshape(4, 3).astype('int64')
y_data = np.arange(1, 13).reshape(4, 3).astype('int64')
x_data = x_data.reshape((-1, num_steps, 1))
y_data = y_data.reshape((-1, 1))
init_hidden_data = np.zeros(
(num_layers, batch_size, hidden_size), dtype='float32')
init_cell_data = np.zeros(
(num_layers, batch_size, hidden_size), dtype='float32')
fetch_list = [static_loss, static_last_hidden, static_last_cell]
out = exe.run(fluid.default_main_program(),
feed={
"x": x_data,
"y": y_data,
"init_hidden": init_hidden_data,
"init_cell": init_cell_data
},
fetch_list=fetch_list)
static_loss_value = out[0]
static_last_hidden_value = out[1]
static_last_cell_value = out[2]
# get value before save
main_program = framework.default_main_program()
base_map = {}
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor())
# make sure all the paramerter or optimzier var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
#fluid.save(main_program, "./test_1")
fluid.io.save_persistables(
exe, "test_path", main_program, filename="model_single")
# set var to zero
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
ten = fluid.global_scope().find_var(var.name).get_tensor()
ten.set(np.zeros_like(np.array(ten)), place)
new_t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor())
# make sure all the paramerter or optimzier var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
file_model_path = os.path.join("test_path", "model_single")
fluid.load(main_program, file_model_path, exe,
fluid.io.get_program_persistable_vars(main_program))
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
new_t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
# test exception
# change shape
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
ten = fluid.global_scope().find_var(var.name).get_tensor()
old_shape = np.array(ten).shape
new_shape = [e + 10 for e in old_shape]
var.desc.set_shape(new_shape)
with self.assertRaises(RuntimeError):
fluid.load(main_program, file_model_path, exe,
fluid.io.get_program_persistable_vars(main_program))
fluid.io.save_params(
exe, "test_path", main_program, filename="model_single")
with self.assertRaises(RuntimeError):
fluid.load(main_program, file_model_path, exe,
fluid.io.get_program_persistable_vars(main_program))
# check when executor is None
with self.assertRaises(ValueError):
fluid.load(main_program, file_model_path, None,
fluid.io.get_program_persistable_vars(main_program))
# check when var list is None
with self.assertRaises(ValueError):
fluid.load(main_program, file_model_path, exe, None)
# check save params, load var_list = get_program_persistable_vars
with self.assertRaises(RuntimeError):
temp_var = framework.Variable(
main_program.global_block(),
shape=[1],
name="test_temp_var")
all_var_list = list(main_program.list_vars())
fluid.load(main_program, file_model_path, exe,
all_var_list + [temp_var])
if __name__ == '__main__':
unittest.main()
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册