Load inference enhance (#21919)

* enhance load interface; test=develop

* add uni test and add comment; test=develop

* fix converage; test=develop

* use path.joint replace "/"; test=develop

* windows debug; test=develop

* fix window unitest error; test=develop

* fix commet error; test=develop

* add model shuffix check; test=develop

* fix example error; test=develop

python/paddle/fluid/io.py

@@ -55,6 +55,8 @@ __all__ = [
     'load',
     'load_program_state',
     'set_program_state',
+    'get_program_parameter',
+    'get_program_persistable_vars',
 ] + reader.__all__ + paddle.reader.__all__
 
 _logger = get_logger(
@@ -114,6 +116,50 @@ def is_belong_to_optimizer(var):
     return False
 
 
+def get_program_parameter(program):
+    """
+    Get all the parameters from Program.
+
+    Args:
+        var(Program): The Program to get parameters
+
+    Returns:
+        list: The list contains all parameters in the program
+
+    Examples:
+        .. code-block:: python
+
+            import paddle.fluid as fluid
+            data = fluid.data(name="img", shape=[64, 784])
+            w = fluid.layers.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
+            b = fluid.layers.create_parameter(shape=[200], dtype='float32', name='fc_b')
+            list_para  = fluid.io.get_program_parameter(  fluid.default_main_program() )
+    """
+    return list(filter(is_parameter, program.list_vars()))
+
+
+def get_program_persistable_vars(program):
+    """
+    Get all the persistable vars from Program.
+
+    Args:
+        var(Program): The Program to get persistable vars
+
+    Returns:
+        list: The list contains all persistable vars in the program
+
+    Examples:
+        .. code-block:: python
+
+            import paddle.fluid as fluid
+            data = fluid.data(name="img", shape=[64, 784])
+            w = fluid.layers.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
+            b = fluid.layers.create_parameter(shape=[200], dtype='float32', name='fc_b')
+            list_para  = fluid.io.get_program_persistable_vars(  fluid.default_main_program() )
+    """
+    return list(filter(is_persistable, program.list_vars()))
+
+
 def _clone_var_in_block_(block, var):
     assert isinstance(var, Variable)
     if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR:
@@ -1497,16 +1543,23 @@ def save(program, model_path):
         f.write(program.desc.serialize_to_string())
 
 
-def load(program, model_path, executor=None):
+def load(program, model_path, executor=None, var_list=None):
     """
-    This function filter out parameters and optimizer information from program, and then get corresponding value from file.
+    This function get parameters and optimizer information from program, and then get corresponding value from file.
     An exception will throw if shape or dtype of the parameters is not match.
 
+    This function can also load model file saved with [ save_params, save_persistables, save_vars ]. 
+    var_list can not be None  when load single model file 
+    ( filename is not None When save_params, save_persistables or save_vars is called ).
+
     Args: 
         program(Program): The program will be loaded
         model_path(str): The file prefix store the program
         executor(Executor, optional): The executor used for initialize the parameter 
                                       When startup program is not run.
+        var_list(list, optional): The variable list to load single model file saved with 
+                                  [ save_params, save_persistables, save_vars ]. 
+                                  Default: None
 
     Returns:
         None
@@ -1525,9 +1578,85 @@ def load(program, model_path, executor=None):
 
     assert executor is None or isinstance(executor, Executor)
 
-    parameter_file_name = model_path + ".pdparams"
-    assert os.path.exists(parameter_file_name), \
-        "Parameter file [{}] not exits".format(parameter_file_name)
+    model_prefix = model_path
+    if model_prefix.endswith(".pdparams"):
+        model_prefix = model_prefix[:-9]
+    elif model_prefix.endswith(".pdopt"):
+        model_prefix = model_prefix[:-6]
+    elif model_prefix.endswith(".pdmodel"):
+        model_prefix = model_prefix[:-8]
+
+    parameter_file_name = model_prefix + ".pdparams"
+
+    if not os.path.exists(parameter_file_name):
+        # model file save by fluid.save not found, try to load model file saved with
+        # [save_vars, save_params, save_persistables]
+        _logger.warning(
+            "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".
+            format(parameter_file_name))
+        if executor is None:
+            raise ValueError(
+                "executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
+            )
+        if os.path.isdir(model_path):
+            binary_file_set = set()
+            for root, dirs, files in os.walk(model_path, topdown=False):
+                for f in files:
+                    binary_file_set.add(
+                        os.path.join(root, f).replace("\\", "/"))
+            program_var_list = list(program.list_vars())
+            loaded_var_list = []
+            for var in program_var_list:
+                var_path = os.path.join(model_path, var.name).replace("\\", "/")
+                if var_path in binary_file_set:
+                    loaded_var_list.append(var)
+                    binary_file_set.remove(var_path)
+            if len(binary_file_set) > 0:
+                unused_var_list = " ".join(list(binary_file_set))
+                _logger.warning("variable file [ %s ] not used" %
+                                (" ".join(list(binary_file_set))))
+            try:
+                load_vars(
+                    executor=executor, dirname=model_path, vars=loaded_var_list)
+            except RuntimeError as e:
+                _logger.error(e)
+                raise e
+            except:
+                raise RuntimeError(
+                    "Failed to load model file , please make sure model file is saved with the "
+                    "following APIs: save_params, save_persistables, save_vars")
+
+            return
+        elif os.path.isfile(model_path):
+            if var_list == None:
+                raise ValueError(
+                    "var_list is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
+                )
+            program_var_list = program.list_vars()
+            program_var_name_set = set([var.name for var in program_var_list])
+
+            # check all the variable inlcuded in program
+            for var in var_list:
+                if var.name not in program_var_name_set:
+                    raise LookupError(
+                        "loaded var [{}] not included in program variable list")
+
+            dir_name, file_name = os.path.split(model_path)
+            try:
+                load_vars(
+                    executor=executor,
+                    dirname=dir_name,
+                    vars=var_list,
+                    filename=file_name)
+            except RuntimeError as e:
+                _logger.error(e)
+                raise e
+            except:
+                raise RuntimeError( "Failed to load model file , please make sure model file is saved with the " \
+                                    "the following APIs: [ save_params, save_persistables, save_vars ]. " \
+                                    "When these API called, filename CANNOT be None")
+
+            return
 
     def set_var(var, ndarray):
         t = global_scope().find_var(var.name).get_tensor()
@@ -1561,7 +1690,7 @@ def load(program, model_path, executor=None):
         filter(is_belong_to_optimizer, program.list_vars()))
 
     if len(optimizer_var_list) > 0:
-        opt_file_name = model_path + ".pdopt"
+        opt_file_name = model_prefix + ".pdopt"
         assert os.path.exists(opt_file_name), \
             "Optimizer file [{}] not exits".format(opt_file_name)
 
@@ -1603,8 +1732,6 @@ def load_program_state(model_path):
             fluid.save( prog, "./temp")
             program_state = fluid.load_program_state( "./temp")
             
-            fluid.set_program_state( prog, program_state)
-
     """
     parameter_file_name = model_path + ".pdparams"
     assert os.path.exists(parameter_file_name), \
@@ -1653,6 +1780,8 @@ def set_program_state(program, state_dict):
             fluid.save( prog, "./temp")
             program_state = fluid.load_program_state( "./temp")
 
+            fluid.set_program_state( prog, program_state)
+
     """
     parameter_list = list(filter(is_persistable, program.list_vars()))
 

python/paddle/fluid/tests/unittests/test_static_save_load.py

@@ -27,6 +27,7 @@ from paddle.fluid.executor import global_scope
 import numpy as np
 import six
 import pickle
+import os
 
 
 class SimpleLSTMRNN(fluid.Layer):
@@ -37,7 +38,7 @@ class SimpleLSTMRNN(fluid.Layer):
                  num_layers=2,
                  init_scale=0.1,
                  dropout=None):
-        super(SimpleLSTMRNN, self).__init__(name_scope)
+        super(SimpleLSTMRNN, self).__init__()
         self._hidden_size = hidden_size
         self._num_layers = num_layers
         self._init_scale = init_scale
@@ -47,7 +48,6 @@ class SimpleLSTMRNN(fluid.Layer):
         self.cell_array = []
         self.hidden_array = []
 
-    def _build_once(self, input_embedding, init_hidden=None, init_cell=None):
         self.weight_1_arr = []
         self.weight_2_arr = []
         self.bias_arr = []
@@ -143,7 +143,7 @@ class PtbModel(fluid.Layer):
                  num_steps=20,
                  init_scale=0.1,
                  dropout=None):
-        super(PtbModel, self).__init__(name_scope)
+        super(PtbModel, self).__init__()
         self.hidden_size = hidden_size
         self.vocab_size = vocab_size
         self.init_scale = init_scale
@@ -302,7 +302,7 @@ class TestSaveLoadBase(unittest.TestCase):
                     # make sure all the paramerter or optimzier var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.load(main_program, "./test_1", exe)
+            fluid.load(main_program, "./test_1.pdparams", exe)
 
             for var in main_program.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
@@ -411,15 +411,15 @@ class TestSaveLoadPartial(unittest.TestCase):
                     # make sure all the paramerter or optimzier var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.load(test_program, "./test_1", None)
+            fluid.load(test_program, "./test_1.pdopt", None)
 
             for var in test_program.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
-                    print(var.name)
                     new_t = np.array(fluid.global_scope().find_var(var.name)
                                      .get_tensor())
                     base_t = base_map[var.name]
                     self.assertTrue(np.array_equal(new_t, base_t))
+            fluid.load(test_program, "./test_1.pdmodel", None)
 
 
 class TestSaveLoadSetStateDict(unittest.TestCase):
@@ -628,7 +628,6 @@ class TestProgramStatePartial(unittest.TestCase):
 
             for var in test_program.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
-                    print(var.name)
                     new_t = np.array(fluid.global_scope().find_var(var.name)
                                      .get_tensor())
                     base_t = base_map[var.name]
@@ -718,5 +717,271 @@ class TestVariableInit(unittest.TestCase):
                 self.assertTrue(np.array_equal(new_t, base_t))
 
 
+class TestLoadFromOldInterface(unittest.TestCase):
+    def setUp(self):
+        if os.path.exists("test_path.pdparams"):
+            os.remove("test_path.pdparams")
+
+    def test_load_from_old_interface(self):
+        seed = 90
+        hidden_size = 10
+        vocab_size = 1000
+        num_layers = 1
+        num_steps = 3
+        init_scale = 0.1
+        batch_size = 4
+        batch_num = 200
+
+        with new_program_scope():
+            fluid.default_startup_program().random_seed = seed
+            fluid.default_main_program().random_seed = seed
+            ptb_model = PtbModel(
+                "ptb_model",
+                hidden_size=hidden_size,
+                vocab_size=vocab_size,
+                num_layers=num_layers,
+                num_steps=num_steps,
+                init_scale=init_scale)
+
+            place = fluid.CPUPlace() if not core.is_compiled_with_cuda(
+            ) else fluid.CUDAPlace(0)
+            exe = fluid.Executor(place)
+            sgd = Adam(learning_rate=1e-3)
+            x = fluid.layers.data(
+                name="x", shape=[-1, num_steps], dtype='int64')
+            y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32')
+            init_hidden = fluid.layers.data(
+                name="init_hidden", shape=[1], dtype='float32')
+            init_cell = fluid.layers.data(
+                name="init_cell", shape=[1], dtype='float32')
+
+            static_loss, static_last_hidden, static_last_cell = ptb_model(
+                x, y, init_hidden, init_cell)
+
+            test_clone_program = fluid.default_main_program().clone()
+            sgd.minimize(static_loss)
+            static_param_updated = dict()
+            static_param_init = dict()
+
+            out = exe.run(framework.default_startup_program())
+
+            static_loss_value = None
+            static_last_cell_value = None
+            static_last_hidden_value = None
+            for i in range(batch_num):
+                x_data = np.arange(12).reshape(4, 3).astype('int64')
+                y_data = np.arange(1, 13).reshape(4, 3).astype('int64')
+                x_data = x_data.reshape((-1, num_steps, 1))
+                y_data = y_data.reshape((-1, 1))
+                init_hidden_data = np.zeros(
+                    (num_layers, batch_size, hidden_size), dtype='float32')
+                init_cell_data = np.zeros(
+                    (num_layers, batch_size, hidden_size), dtype='float32')
+                fetch_list = [static_loss, static_last_hidden, static_last_cell]
+                out = exe.run(fluid.default_main_program(),
+                              feed={
+                                  "x": x_data,
+                                  "y": y_data,
+                                  "init_hidden": init_hidden_data,
+                                  "init_cell": init_cell_data
+                              },
+                              fetch_list=fetch_list)
+                static_loss_value = out[0]
+                static_last_hidden_value = out[1]
+                static_last_cell_value = out[2]
+
+            # get value before save
+            main_program = framework.default_main_program()
+            base_map = {}
+            for var in main_program.list_vars():
+                if isinstance(var, framework.Parameter) or var.persistable:
+                    t = np.array(fluid.global_scope().find_var(var.name)
+                                 .get_tensor())
+                    # make sure all the paramerter or optimzier var have been update
+                    self.assertTrue(np.sum(np.abs(t)) != 0)
+                    base_map[var.name] = t
+
+            #fluid.save(main_program, "./test_1")
+            fluid.io.save_persistables(exe, "test_path", main_program)
+
+            # set var to zero
+            for var in main_program.list_vars():
+                if isinstance(var, framework.Parameter) or var.persistable:
+                    ten = fluid.global_scope().find_var(var.name).get_tensor()
+                    ten.set(np.zeros_like(np.array(ten)), place)
+
+                    new_t = np.array(fluid.global_scope().find_var(var.name)
+                                     .get_tensor())
+                    # make sure all the paramerter or optimzier var have been set to zero
+                    self.assertTrue(np.sum(np.abs(new_t)) == 0)
+
+            fluid.load(main_program, "test_path", exe)
+
+            for var in main_program.list_vars():
+                if isinstance(var, framework.Parameter) or var.persistable:
+                    new_t = np.array(fluid.global_scope().find_var(var.name)
+                                     .get_tensor())
+                    base_t = base_map[var.name]
+                    self.assertTrue(np.array_equal(new_t, base_t))
+
+            for var in main_program.list_vars():
+                if isinstance(var, framework.Parameter) or var.persistable:
+                    ten = fluid.global_scope().find_var(var.name).get_tensor()
+                    old_shape = np.array(ten).shape
+                    new_shape = [e + 10 for e in old_shape]
+
+                    var.desc.set_shape(new_shape)
+            with self.assertRaises(RuntimeError):
+                fluid.load(main_program, "test_path", exe)
+
+            # check unused paramter
+
+            fluid.load(test_clone_program, "test_path", exe)
+
+
+class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
+    def test_load_from_old_interface(self):
+        seed = 90
+        hidden_size = 10
+        vocab_size = 1000
+        num_layers = 1
+        num_steps = 3
+        init_scale = 0.1
+        batch_size = 4
+        batch_num = 200
+
+        with new_program_scope():
+            fluid.default_startup_program().random_seed = seed
+            fluid.default_main_program().random_seed = seed
+            ptb_model = PtbModel(
+                "ptb_model",
+                hidden_size=hidden_size,
+                vocab_size=vocab_size,
+                num_layers=num_layers,
+                num_steps=num_steps,
+                init_scale=init_scale)
+
+            place = fluid.CPUPlace() if not core.is_compiled_with_cuda(
+            ) else fluid.CUDAPlace(0)
+            exe = fluid.Executor(place)
+            sgd = Adam(learning_rate=1e-3)
+            x = fluid.layers.data(
+                name="x", shape=[-1, num_steps], dtype='int64')
+            y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32')
+            init_hidden = fluid.layers.data(
+                name="init_hidden", shape=[1], dtype='float32')
+            init_cell = fluid.layers.data(
+                name="init_cell", shape=[1], dtype='float32')
+
+            static_loss, static_last_hidden, static_last_cell = ptb_model(
+                x, y, init_hidden, init_cell)
+            sgd.minimize(static_loss)
+            static_param_updated = dict()
+            static_param_init = dict()
+
+            out = exe.run(framework.default_startup_program())
+
+            static_loss_value = None
+            static_last_cell_value = None
+            static_last_hidden_value = None
+            for i in range(batch_num):
+                x_data = np.arange(12).reshape(4, 3).astype('int64')
+                y_data = np.arange(1, 13).reshape(4, 3).astype('int64')
+                x_data = x_data.reshape((-1, num_steps, 1))
+                y_data = y_data.reshape((-1, 1))
+                init_hidden_data = np.zeros(
+                    (num_layers, batch_size, hidden_size), dtype='float32')
+                init_cell_data = np.zeros(
+                    (num_layers, batch_size, hidden_size), dtype='float32')
+                fetch_list = [static_loss, static_last_hidden, static_last_cell]
+                out = exe.run(fluid.default_main_program(),
+                              feed={
+                                  "x": x_data,
+                                  "y": y_data,
+                                  "init_hidden": init_hidden_data,
+                                  "init_cell": init_cell_data
+                              },
+                              fetch_list=fetch_list)
+                static_loss_value = out[0]
+                static_last_hidden_value = out[1]
+                static_last_cell_value = out[2]
+
+            # get value before save
+            main_program = framework.default_main_program()
+            base_map = {}
+            for var in main_program.list_vars():
+                if isinstance(var, framework.Parameter) or var.persistable:
+                    t = np.array(fluid.global_scope().find_var(var.name)
+                                 .get_tensor())
+                    # make sure all the paramerter or optimzier var have been update
+                    self.assertTrue(np.sum(np.abs(t)) != 0)
+                    base_map[var.name] = t
+
+            #fluid.save(main_program, "./test_1")
+            fluid.io.save_persistables(
+                exe, "test_path", main_program, filename="model_single")
+
+            # set var to zero
+            for var in main_program.list_vars():
+                if isinstance(var, framework.Parameter) or var.persistable:
+                    ten = fluid.global_scope().find_var(var.name).get_tensor()
+                    ten.set(np.zeros_like(np.array(ten)), place)
+
+                    new_t = np.array(fluid.global_scope().find_var(var.name)
+                                     .get_tensor())
+                    # make sure all the paramerter or optimzier var have been set to zero
+                    self.assertTrue(np.sum(np.abs(new_t)) == 0)
+
+            file_model_path = os.path.join("test_path", "model_single")
+            fluid.load(main_program, file_model_path, exe,
+                       fluid.io.get_program_persistable_vars(main_program))
+
+            for var in main_program.list_vars():
+                if isinstance(var, framework.Parameter) or var.persistable:
+                    new_t = np.array(fluid.global_scope().find_var(var.name)
+                                     .get_tensor())
+                    base_t = base_map[var.name]
+                    self.assertTrue(np.array_equal(new_t, base_t))
+
+            # test exception
+            # change shape
+            for var in main_program.list_vars():
+                if isinstance(var, framework.Parameter) or var.persistable:
+                    ten = fluid.global_scope().find_var(var.name).get_tensor()
+                    old_shape = np.array(ten).shape
+                    new_shape = [e + 10 for e in old_shape]
+
+                    var.desc.set_shape(new_shape)
+
+            with self.assertRaises(RuntimeError):
+                fluid.load(main_program, file_model_path, exe,
+                           fluid.io.get_program_persistable_vars(main_program))
+
+            fluid.io.save_params(
+                exe, "test_path", main_program, filename="model_single")
+            with self.assertRaises(RuntimeError):
+                fluid.load(main_program, file_model_path, exe,
+                           fluid.io.get_program_persistable_vars(main_program))
+
+            # check when executor is None
+            with self.assertRaises(ValueError):
+                fluid.load(main_program, file_model_path, None,
+                           fluid.io.get_program_persistable_vars(main_program))
+
+            # check when var list is None
+            with self.assertRaises(ValueError):
+                fluid.load(main_program, file_model_path, exe, None)
+
+            # check save params, load var_list = get_program_persistable_vars
+            with self.assertRaises(RuntimeError):
+                temp_var = framework.Variable(
+                    main_program.global_block(),
+                    shape=[1],
+                    name="test_temp_var")
+                all_var_list = list(main_program.list_vars())
+                fluid.load(main_program, file_model_path, exe,
+                           all_var_list + [temp_var])
+
+
 if __name__ == '__main__':
     unittest.main()