From 9f7d90d20361d53d72e6c0862287e06602a52aca Mon Sep 17 00:00:00 2001 From: hong <43953930+phlrain@users.noreply.github.com> Date: Wed, 8 Jan 2020 00:15:07 +0800 Subject: [PATCH] Load inference enhance (#21919) * enhance load interface; test=develop * add uni test and add comment; test=develop * fix converage; test=develop * use path.joint replace "/"; test=develop * windows debug; test=develop * fix window unitest error; test=develop * fix commet error; test=develop * add model shuffix check; test=develop * fix example error; test=develop --- python/paddle/fluid/io.py | 145 ++++++++- .../tests/unittests/test_static_save_load.py | 279 +++++++++++++++++- 2 files changed, 409 insertions(+), 15 deletions(-) diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index 6ca49eb335..0436137a80 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -55,6 +55,8 @@ __all__ = [ 'load', 'load_program_state', 'set_program_state', + 'get_program_parameter', + 'get_program_persistable_vars', ] + reader.__all__ + paddle.reader.__all__ _logger = get_logger( @@ -114,6 +116,50 @@ def is_belong_to_optimizer(var): return False +def get_program_parameter(program): + """ + Get all the parameters from Program. + + Args: + var(Program): The Program to get parameters + + Returns: + list: The list contains all parameters in the program + + Examples: + .. code-block:: python + + import paddle.fluid as fluid + data = fluid.data(name="img", shape=[64, 784]) + w = fluid.layers.create_parameter(shape=[784, 200], dtype='float32', name='fc_w') + b = fluid.layers.create_parameter(shape=[200], dtype='float32', name='fc_b') + list_para = fluid.io.get_program_parameter( fluid.default_main_program() ) + """ + return list(filter(is_parameter, program.list_vars())) + + +def get_program_persistable_vars(program): + """ + Get all the persistable vars from Program. + + Args: + var(Program): The Program to get persistable vars + + Returns: + list: The list contains all persistable vars in the program + + Examples: + .. code-block:: python + + import paddle.fluid as fluid + data = fluid.data(name="img", shape=[64, 784]) + w = fluid.layers.create_parameter(shape=[784, 200], dtype='float32', name='fc_w') + b = fluid.layers.create_parameter(shape=[200], dtype='float32', name='fc_b') + list_para = fluid.io.get_program_persistable_vars( fluid.default_main_program() ) + """ + return list(filter(is_persistable, program.list_vars())) + + def _clone_var_in_block_(block, var): assert isinstance(var, Variable) if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR: @@ -1497,16 +1543,23 @@ def save(program, model_path): f.write(program.desc.serialize_to_string()) -def load(program, model_path, executor=None): +def load(program, model_path, executor=None, var_list=None): """ - This function filter out parameters and optimizer information from program, and then get corresponding value from file. + This function get parameters and optimizer information from program, and then get corresponding value from file. An exception will throw if shape or dtype of the parameters is not match. + This function can also load model file saved with [ save_params, save_persistables, save_vars ]. + var_list can not be None when load single model file + ( filename is not None When save_params, save_persistables or save_vars is called ). + Args: program(Program): The program will be loaded model_path(str): The file prefix store the program executor(Executor, optional): The executor used for initialize the parameter When startup program is not run. + var_list(list, optional): The variable list to load single model file saved with + [ save_params, save_persistables, save_vars ]. + Default: None Returns: None @@ -1525,9 +1578,85 @@ def load(program, model_path, executor=None): assert executor is None or isinstance(executor, Executor) - parameter_file_name = model_path + ".pdparams" - assert os.path.exists(parameter_file_name), \ - "Parameter file [{}] not exits".format(parameter_file_name) + model_prefix = model_path + if model_prefix.endswith(".pdparams"): + model_prefix = model_prefix[:-9] + elif model_prefix.endswith(".pdopt"): + model_prefix = model_prefix[:-6] + elif model_prefix.endswith(".pdmodel"): + model_prefix = model_prefix[:-8] + + parameter_file_name = model_prefix + ".pdparams" + + if not os.path.exists(parameter_file_name): + # model file save by fluid.save not found, try to load model file saved with + # [save_vars, save_params, save_persistables] + _logger.warning( + "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]". + format(parameter_file_name)) + if executor is None: + raise ValueError( + "executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]" + ) + if os.path.isdir(model_path): + binary_file_set = set() + for root, dirs, files in os.walk(model_path, topdown=False): + for f in files: + binary_file_set.add( + os.path.join(root, f).replace("\\", "/")) + program_var_list = list(program.list_vars()) + loaded_var_list = [] + for var in program_var_list: + var_path = os.path.join(model_path, var.name).replace("\\", "/") + if var_path in binary_file_set: + loaded_var_list.append(var) + binary_file_set.remove(var_path) + if len(binary_file_set) > 0: + unused_var_list = " ".join(list(binary_file_set)) + _logger.warning("variable file [ %s ] not used" % + (" ".join(list(binary_file_set)))) + try: + load_vars( + executor=executor, dirname=model_path, vars=loaded_var_list) + except RuntimeError as e: + _logger.error(e) + raise e + except: + raise RuntimeError( + "Failed to load model file , please make sure model file is saved with the " + "following APIs: save_params, save_persistables, save_vars") + + return + elif os.path.isfile(model_path): + if var_list == None: + raise ValueError( + "var_list is required when loading model file saved with [ save_params, save_persistables, save_vars ]" + ) + program_var_list = program.list_vars() + program_var_name_set = set([var.name for var in program_var_list]) + + # check all the variable inlcuded in program + for var in var_list: + if var.name not in program_var_name_set: + raise LookupError( + "loaded var [{}] not included in program variable list") + + dir_name, file_name = os.path.split(model_path) + try: + load_vars( + executor=executor, + dirname=dir_name, + vars=var_list, + filename=file_name) + except RuntimeError as e: + _logger.error(e) + raise e + except: + raise RuntimeError( "Failed to load model file , please make sure model file is saved with the " \ + "the following APIs: [ save_params, save_persistables, save_vars ]. " \ + "When these API called, filename CANNOT be None") + + return def set_var(var, ndarray): t = global_scope().find_var(var.name).get_tensor() @@ -1561,7 +1690,7 @@ def load(program, model_path, executor=None): filter(is_belong_to_optimizer, program.list_vars())) if len(optimizer_var_list) > 0: - opt_file_name = model_path + ".pdopt" + opt_file_name = model_prefix + ".pdopt" assert os.path.exists(opt_file_name), \ "Optimizer file [{}] not exits".format(opt_file_name) @@ -1603,8 +1732,6 @@ def load_program_state(model_path): fluid.save( prog, "./temp") program_state = fluid.load_program_state( "./temp") - fluid.set_program_state( prog, program_state) - """ parameter_file_name = model_path + ".pdparams" assert os.path.exists(parameter_file_name), \ @@ -1653,6 +1780,8 @@ def set_program_state(program, state_dict): fluid.save( prog, "./temp") program_state = fluid.load_program_state( "./temp") + fluid.set_program_state( prog, program_state) + """ parameter_list = list(filter(is_persistable, program.list_vars())) diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load.py b/python/paddle/fluid/tests/unittests/test_static_save_load.py index a0fd85f032..0dd767edc4 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load.py @@ -27,6 +27,7 @@ from paddle.fluid.executor import global_scope import numpy as np import six import pickle +import os class SimpleLSTMRNN(fluid.Layer): @@ -37,7 +38,7 @@ class SimpleLSTMRNN(fluid.Layer): num_layers=2, init_scale=0.1, dropout=None): - super(SimpleLSTMRNN, self).__init__(name_scope) + super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers self._init_scale = init_scale @@ -47,7 +48,6 @@ class SimpleLSTMRNN(fluid.Layer): self.cell_array = [] self.hidden_array = [] - def _build_once(self, input_embedding, init_hidden=None, init_cell=None): self.weight_1_arr = [] self.weight_2_arr = [] self.bias_arr = [] @@ -143,7 +143,7 @@ class PtbModel(fluid.Layer): num_steps=20, init_scale=0.1, dropout=None): - super(PtbModel, self).__init__(name_scope) + super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size self.init_scale = init_scale @@ -302,7 +302,7 @@ class TestSaveLoadBase(unittest.TestCase): # make sure all the paramerter or optimzier var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "./test_1", exe) + fluid.load(main_program, "./test_1.pdparams", exe) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -411,15 +411,15 @@ class TestSaveLoadPartial(unittest.TestCase): # make sure all the paramerter or optimzier var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(test_program, "./test_1", None) + fluid.load(test_program, "./test_1.pdopt", None) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - print(var.name) new_t = np.array(fluid.global_scope().find_var(var.name) .get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + fluid.load(test_program, "./test_1.pdmodel", None) class TestSaveLoadSetStateDict(unittest.TestCase): @@ -628,7 +628,6 @@ class TestProgramStatePartial(unittest.TestCase): for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - print(var.name) new_t = np.array(fluid.global_scope().find_var(var.name) .get_tensor()) base_t = base_map[var.name] @@ -718,5 +717,271 @@ class TestVariableInit(unittest.TestCase): self.assertTrue(np.array_equal(new_t, base_t)) +class TestLoadFromOldInterface(unittest.TestCase): + def setUp(self): + if os.path.exists("test_path.pdparams"): + os.remove("test_path.pdparams") + + def test_load_from_old_interface(self): + seed = 90 + hidden_size = 10 + vocab_size = 1000 + num_layers = 1 + num_steps = 3 + init_scale = 0.1 + batch_size = 4 + batch_num = 200 + + with new_program_scope(): + fluid.default_startup_program().random_seed = seed + fluid.default_main_program().random_seed = seed + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale) + + place = fluid.CPUPlace() if not core.is_compiled_with_cuda( + ) else fluid.CUDAPlace(0) + exe = fluid.Executor(place) + sgd = Adam(learning_rate=1e-3) + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64') + y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32') + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32') + + static_loss, static_last_hidden, static_last_cell = ptb_model( + x, y, init_hidden, init_cell) + + test_clone_program = fluid.default_main_program().clone() + sgd.minimize(static_loss) + static_param_updated = dict() + static_param_init = dict() + + out = exe.run(framework.default_startup_program()) + + static_loss_value = None + static_last_cell_value = None + static_last_hidden_value = None + for i in range(batch_num): + x_data = np.arange(12).reshape(4, 3).astype('int64') + y_data = np.arange(1, 13).reshape(4, 3).astype('int64') + x_data = x_data.reshape((-1, num_steps, 1)) + y_data = y_data.reshape((-1, 1)) + init_hidden_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32') + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32') + fetch_list = [static_loss, static_last_hidden, static_last_cell] + out = exe.run(fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data + }, + fetch_list=fetch_list) + static_loss_value = out[0] + static_last_hidden_value = out[1] + static_last_cell_value = out[2] + + # get value before save + main_program = framework.default_main_program() + base_map = {} + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimzier var have been update + self.assertTrue(np.sum(np.abs(t)) != 0) + base_map[var.name] = t + + #fluid.save(main_program, "./test_1") + fluid.io.save_persistables(exe, "test_path", main_program) + + # set var to zero + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + ten = fluid.global_scope().find_var(var.name).get_tensor() + ten.set(np.zeros_like(np.array(ten)), place) + + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimzier var have been set to zero + self.assertTrue(np.sum(np.abs(new_t)) == 0) + + fluid.load(main_program, "test_path", exe) + + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + base_t = base_map[var.name] + self.assertTrue(np.array_equal(new_t, base_t)) + + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + ten = fluid.global_scope().find_var(var.name).get_tensor() + old_shape = np.array(ten).shape + new_shape = [e + 10 for e in old_shape] + + var.desc.set_shape(new_shape) + with self.assertRaises(RuntimeError): + fluid.load(main_program, "test_path", exe) + + # check unused paramter + + fluid.load(test_clone_program, "test_path", exe) + + +class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): + def test_load_from_old_interface(self): + seed = 90 + hidden_size = 10 + vocab_size = 1000 + num_layers = 1 + num_steps = 3 + init_scale = 0.1 + batch_size = 4 + batch_num = 200 + + with new_program_scope(): + fluid.default_startup_program().random_seed = seed + fluid.default_main_program().random_seed = seed + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale) + + place = fluid.CPUPlace() if not core.is_compiled_with_cuda( + ) else fluid.CUDAPlace(0) + exe = fluid.Executor(place) + sgd = Adam(learning_rate=1e-3) + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64') + y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32') + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32') + + static_loss, static_last_hidden, static_last_cell = ptb_model( + x, y, init_hidden, init_cell) + sgd.minimize(static_loss) + static_param_updated = dict() + static_param_init = dict() + + out = exe.run(framework.default_startup_program()) + + static_loss_value = None + static_last_cell_value = None + static_last_hidden_value = None + for i in range(batch_num): + x_data = np.arange(12).reshape(4, 3).astype('int64') + y_data = np.arange(1, 13).reshape(4, 3).astype('int64') + x_data = x_data.reshape((-1, num_steps, 1)) + y_data = y_data.reshape((-1, 1)) + init_hidden_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32') + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32') + fetch_list = [static_loss, static_last_hidden, static_last_cell] + out = exe.run(fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data + }, + fetch_list=fetch_list) + static_loss_value = out[0] + static_last_hidden_value = out[1] + static_last_cell_value = out[2] + + # get value before save + main_program = framework.default_main_program() + base_map = {} + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimzier var have been update + self.assertTrue(np.sum(np.abs(t)) != 0) + base_map[var.name] = t + + #fluid.save(main_program, "./test_1") + fluid.io.save_persistables( + exe, "test_path", main_program, filename="model_single") + + # set var to zero + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + ten = fluid.global_scope().find_var(var.name).get_tensor() + ten.set(np.zeros_like(np.array(ten)), place) + + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimzier var have been set to zero + self.assertTrue(np.sum(np.abs(new_t)) == 0) + + file_model_path = os.path.join("test_path", "model_single") + fluid.load(main_program, file_model_path, exe, + fluid.io.get_program_persistable_vars(main_program)) + + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + base_t = base_map[var.name] + self.assertTrue(np.array_equal(new_t, base_t)) + + # test exception + # change shape + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + ten = fluid.global_scope().find_var(var.name).get_tensor() + old_shape = np.array(ten).shape + new_shape = [e + 10 for e in old_shape] + + var.desc.set_shape(new_shape) + + with self.assertRaises(RuntimeError): + fluid.load(main_program, file_model_path, exe, + fluid.io.get_program_persistable_vars(main_program)) + + fluid.io.save_params( + exe, "test_path", main_program, filename="model_single") + with self.assertRaises(RuntimeError): + fluid.load(main_program, file_model_path, exe, + fluid.io.get_program_persistable_vars(main_program)) + + # check when executor is None + with self.assertRaises(ValueError): + fluid.load(main_program, file_model_path, None, + fluid.io.get_program_persistable_vars(main_program)) + + # check when var list is None + with self.assertRaises(ValueError): + fluid.load(main_program, file_model_path, exe, None) + + # check save params, load var_list = get_program_persistable_vars + with self.assertRaises(RuntimeError): + temp_var = framework.Variable( + main_program.global_block(), + shape=[1], + name="test_temp_var") + all_var_list = list(main_program.list_vars()) + fluid.load(main_program, file_model_path, exe, + all_var_list + [temp_var]) + + if __name__ == '__main__': unittest.main() -- GitLab