[Fluid Clean]Move and Remove some io API in Fluid (#50842)

* move io

* fix ci bugs

* fix ci bugs

* fix py3 bugs

* fix example code

* fix example code

* fix text

* fix text

* deal with ci bugs

* perfect code according comment

* delete import batch

python/paddle/distributed/auto_parallel/utils.py

@@ -22,8 +22,8 @@ from functools import reduce
 import numpy as np
 
 import paddle
-from paddle.fluid.io import is_belong_to_optimizer, is_parameter
 from paddle.framework import core
+from paddle.framework.io_utils import is_belong_to_optimizer, is_parameter
 from paddle.static import Variable
 
 from .dist_attribute import OperatorDistAttr, TensorDistAttr

python/paddle/distributed/fleet/runtime/parameter_server_runtime.py

@@ -109,7 +109,6 @@ class ParameterServerRuntime(RuntimeBase):
             assert isinstance(each_var, Variable)
 
             origin_varname, _, _ = _get_varname_parts(each_var.name)
-
             new_var = paddle.static.io._clone_var_in_block(load_block, each_var)
             var_path = os.path.join(dirname, origin_varname)
             if not os.path.exists(var_path):

python/paddle/distributed/io.py

@@ -16,10 +16,175 @@ import os
 
 import paddle
 from paddle.fluid.framework import Program, static_only
-from paddle.fluid.io import load_persistables
 from paddle.framework import core, dygraph_not_support
 
 
+def _load_distributed_persistables(executor, dirname, main_program=None):
+    """
+    customized load_persistables for distributed training.
+    it should be used on parameter server,
+
+    Args:
+        executor(Executor): The executor to run for saving parameters.
+        dirname(str): The load directory path.
+        main_program(Program): The program whose parameters will be
+                            loaded. the main_program must be the pserver_program
+                            get after transpiler.
+
+    Returns:
+        None
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.fluid as fluid
+
+            paddle.enable_static()
+            exe = fluid.Executor(fluid.CPUPlace())
+            param_path = "./my_paddle_model"
+            t = paddle.distributed.transpiler.DistributeTranspiler()
+            t.transpile(...)
+            pserver_prog = t.get_pserver_program(...)
+            _load_distributed_persistables(executor=exe, dirname=param_path, main_program=pserver_prog)
+    """
+
+    def __is_distributed_part_var(varname):
+        trainer_idx = varname.find(".trainer_")
+        block_idx = varname.find(".block")
+        return trainer_idx or block_idx
+
+    def __load_persistable_vars(executor, dirname, need_load_vars):
+        load_prog = Program()
+        load_block = load_prog.global_block()
+        need_delete_vars = []
+
+        for param in need_load_vars:
+            origin_var = param.origin
+            slice_var = param.slice
+            is_slice = param.is_slice
+            offset = param.offset
+
+            if is_slice:
+                slice = load_block.create_var(
+                    name=slice_var.name,
+                    type=slice_var.type,
+                    shape=slice_var.shape,
+                    dtype=slice_var.dtype,
+                    persistable=True,
+                )
+
+                load_block.append_op(
+                    type='load',
+                    inputs={},
+                    outputs={'Out': [slice]},
+                    attrs={
+                        'file_path': os.path.join(dirname, origin_var.name),
+                        'seek': offset,
+                        'shape': slice.shape,
+                    },
+                )
+            else:
+                origin = load_block.create_var(
+                    name="{}".format(origin_var.name),
+                    type=origin_var.type,
+                    shape=origin_var.shape,
+                    dtype=origin_var.dtype,
+                    persistable=True,
+                )
+                load_block.append_op(
+                    type='load',
+                    inputs={},
+                    outputs={'Out': [origin]},
+                    attrs={'file_path': os.path.join(dirname, origin_var.name)},
+                )
+
+        load_block.append_op(
+            type='delete_var',
+            inputs={'X': need_delete_vars},
+        )
+
+        executor.run(load_prog)
+
+    if not isinstance(main_program, Program):
+        raise TypeError("'main_program' should be an instance of Program.")
+
+    if not main_program._is_distributed:
+        raise ValueError(
+            "'_load_distributed_persistables' just be designed for distributed training."
+        )
+
+    if not main_program._ps_endpoint:
+        raise ValueError(
+            "'_load_distributed_persistables' need current_endpoint set in DistributeTranspiler.transpile"
+        )
+
+    need_load_vars = (
+        main_program._parameters_on_pservers.get_distributed_vars_by_ep(
+            main_program._ps_endpoint
+        )
+    )
+    __load_persistable_vars(executor, dirname, need_load_vars)
+
+
+@dygraph_not_support
+def load_persistables(executor, dirname, main_program=None, filename=None):
+    """
+    :api_attr: Static Graph
+
+    This API filters out all variables with ``persistable==True`` from the
+    given ``main_program`` and then tries to load these variables from the
+    directory ``dirname`` or the file ``filename``.
+
+    Use the ``dirname`` to specify the directory where persistable variables
+    (refer to :ref:`api_guide_model_save_reader_en`) were saved. If variables
+    were saved in separate files, set ``filename`` as None; if all variables
+    were saved in a single file, use ``filename`` to specify the file name.
+
+    Args:
+        executor(Executor): The executor used for loading persistable variables.
+                            See :ref:`api_guide_executor_en` for more details about it.
+        dirname(str): The directory path.
+        main_program(Program, optional): The program whose persistable variables will
+                                    be loaded. If it is None, the ``default_main_program``
+                                    will be used automatically. See :ref:`api_guide_Program_en`
+                                    for more about ``Program``.
+                                    Default: None.
+        filename(str, optional): The file which saved all persistable variables. If variables
+                                 were saved in separated files, set it to None.
+                                 Default: None.
+
+    Returns:
+        None
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.fluid as fluid
+
+            paddle.enable_static()
+            exe = fluid.Executor(fluid.CPUPlace())
+            param_path = "./my_paddle_model"
+            prog = fluid.default_main_program()
+            paddle.distributed.io.load_persistables(executor=exe, dirname=param_path,
+                                       main_program=None)
+    """
+
+    if main_program and main_program._is_distributed:
+        _load_distributed_persistables(
+            executor, dirname=dirname, main_program=main_program
+        )
+    else:
+        paddle.static.io.load_vars(
+            executor,
+            dirname=dirname,
+            main_program=main_program,
+            predicate=is_persistable,
+            filename=filename,
+        )
+
+
 def _save_distributed_persistables(executor, dirname, main_program):
     """
     save_persistables for distributed training.

python/paddle/fluid/__init__.py

@@ -89,7 +89,6 @@ from .compiler import *
 from paddle.fluid.layers.math_op_patch import monkey_patch_variable
 from .dygraph.layers import *
 from .dygraph.base import enable_dygraph, disable_dygraph
-from .io import save, load, load_program_state, set_program_state
 from .dygraph.varbase_patch_methods import monkey_patch_varbase
 from .core import _cuda_synchronize
 from .trainer_desc import (
@@ -146,8 +145,6 @@ __all__ = (
         'profiler',
         'unique_name',
         'Scope',
-        'save',
-        'load',
         '_cuda_synchronize',
     ]
 )

python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py

@@ -34,7 +34,7 @@ class PaddleModel(SerializableBase):
         self._file_name = "_paddle_fleet_param__"
 
     def serialize(self, path):
-        from ...io import save_persistables
+        from paddle.distributed.io import save_persistables
 
         save_persistables(
             executor=self._exe,
@@ -44,7 +44,7 @@ class PaddleModel(SerializableBase):
         )
 
     def deserialize(self, path):
-        from ...io import load_persistables
+        from paddle.distributed.io import load_persistables
 
         load_persistables(
             executor=self._exe,

python/paddle/fluid/io.py

@@ -62,1205 +62,15 @@ from . import core
 from paddle.utils import deprecated
 from paddle.fluid.framework import static_only
 
-batch = paddle.batch
-
-__all__ = [
-    'save_vars',
-    'save_params',
-    'save_persistables',
-    'load_vars',
-    'load_params',
-    'load_persistables',
-    'save_inference_model',
-    'load_inference_model',
-    'batch',
-    'save',
-    'load',
-    'load_program_state',
-    'set_program_state',
-    'get_program_parameter',
-    'get_program_persistable_vars',
-] + reader.__all__
-
-_logger = get_logger(
-    __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
-)
-
-
-class _open_buffer:
-    def __init__(self, buffer):
-        self.buffer = buffer
-
-    def __enter__(self):
-        return self.buffer
-
-
-class _buffer_reader(_open_buffer):
-    def __init__(self, buffer):
-        super().__init__(buffer)
-        self.initial_tell = self.buffer.tell()
-
-    def __exit__(self, *args):
-        # `args[0]` is type of exception. When the `read` is abnormal, the file pointer returns to the initial position.
-        if args[0] is not None:
-            self.buffer.seek(self.initial_tell)
-
-
-class _buffer_writer(_open_buffer):
-    def __exit__(self, *args):
-        self.buffer.flush()
-
-
-def _is_file_path(path):
-    return isinstance(path, str)
-
-
-def _open_file_buffer(path_or_buffer, mode):
-
-    if _is_file_path(path_or_buffer):
-        return open(path_or_buffer, mode)
-    else:
-        if 'w' in mode:
-            return _buffer_writer(path_or_buffer)
-        elif 'r' in mode:
-            return _buffer_reader(path_or_buffer)
-        else:
-            raise ValueError(
-                "Expected 'r' or 'w' in mode but got {}".format(mode)
-            )
-
-
-def _is_memory_buffer(buffer):
-    return isinstance(buffer, BytesIO)
-
-
-def is_parameter(var):
-    """
-    Check whether the given variable is an instance of Parameter.
-
-    Args:
-        var(Variable): The variable to be checked.
-
-    Returns:
-        bool: True if the given `var` is an instance of Parameter,
-        False if not.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            param = fluid.default_main_program().global_block().var('fc.w')
-            res = fluid.io.is_parameter(param)
-    """
-    return isinstance(var, Parameter)
-
-
-def is_persistable(var):
-    """
-    Check whether the given variable is persistable.
-
-    Args:
-        var(Variable): The variable to be checked.
-
-    Returns:
-        bool: True if the given `var` is persistable
-        False if not.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            param = fluid.default_main_program().global_block().var('fc.b')
-            res = fluid.io.is_persistable(param)
-    """
-    if (
-        var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH
-        or var.desc.type() == core.VarDesc.VarType.FETCH_LIST
-        or var.desc.type() == core.VarDesc.VarType.READER
-    ):
-        return False
-    return var.persistable
-
-
-def is_belong_to_optimizer(var):
-    if not (isinstance(var, Parameter) or var.desc.need_check_feed()):
-        return is_persistable(var)
-
-    return False
-
-
-@dygraph_not_support
-def get_program_parameter(program):
-    """
-    Get all the parameters from Program.
-
-    Args:
-        var(Program): The Program to get parameters
-
-    Returns:
-        list: The list contains all parameters in the program
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            data = fluid.data(name="img", shape=[64, 784])
-            w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
-            b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
-            list_para  = fluid.io.get_program_parameter(  fluid.default_main_program() )
-    """
-    return list(filter(is_parameter, program.list_vars()))
-
-
-@dygraph_not_support
-def get_program_persistable_vars(program):
-    """
-    Get all the persistable vars from Program.
-
-    Args:
-        var(Program): The Program to get persistable vars
-
-    Returns:
-        list: The list contains all persistable vars in the program
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            data = fluid.data(name="img", shape=[64, 784])
-            w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
-            b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
-            list_para  = fluid.io.get_program_persistable_vars(  fluid.default_main_program() )
-    """
-    return list(filter(is_persistable, program.list_vars()))
-
-
-def _clone_var_in_block_(block, var):
-    assert isinstance(var, Variable)
-    if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR:
-        return block.create_var(
-            name=var.name,
-            shape=var.shape,
-            dtype=var.dtype,
-            type=var.type,
-            lod_level=var.lod_level,
-            persistable=True,
-        )
-    else:
-        return block.create_var(
-            name=var.name,
-            shape=var.shape,
-            dtype=var.dtype,
-            type=var.type,
-            persistable=True,
-        )
-
-
-@signature_safe_contextmanager
-def _load_program_scope(main=None, startup=None, scope=None):
-    prog = main if main else paddle.fluid.Program()
-    startup_prog = startup if startup else paddle.fluid.Program()
-    scope = scope if scope else paddle.fluid.core.Scope()
-    with paddle.fluid.scope_guard(scope):
-        with paddle.fluid.program_guard(prog, startup_prog):
-            with paddle.fluid.unique_name.guard():
-                with paddle.fluid.framework._dygraph_guard(None):
-                    yield
-
-
-def _get_valid_program(main_program=None):
-    if main_program is None:
-        main_program = default_main_program()
-    elif isinstance(main_program, CompiledProgram):
-        main_program = main_program._program
-        if main_program is None:
-            raise TypeError(
-                "The type of input main_program is invalid, expected tyep is Program, but received None"
-            )
-        warnings.warn(
-            "The input is a CompiledProgram, this is not recommended."
-        )
-    if not isinstance(main_program, Program):
-        raise TypeError(
-            "The type of input main_program is invalid, expected type is fluid.Program, but received %s"
-            % type(main_program)
-        )
-    return main_program
-
-
-@dygraph_not_support
-def save_vars(
-    executor,
-    dirname,
-    main_program=None,
-    vars=None,
-    predicate=None,
-    filename=None,
-):
-    """
-    Save specific variables in the `Program` to files.
-
-    There are two ways to specify the variables to be saved: set variables in
-    a list and assign it to the `vars`, or use the `predicate` function to select
-    variables that make `predicate(variable) == True`. The first way has a higher priority.
-
-    The `dirname` is used to specify the folder where to save variables.
-    If you prefer to save variables in separate files in the `dirname` folder,
-    do not set `filename`. If you prefer to save all variables in a single file,
-    use `filename` to specify it.
-
-    Args:
-        executor(Executor): The executor to run for saving variables.
-        dirname(str, optional): The folder where to save variables.
-                            When you need to save the parameter to the memory, set it to None.
-        main_program(Program, optional): The program whose variables will be saved.
-                                    If it is None, the default main program will
-                                    be used automatically.
-                                    Default: None
-        vars(list[Variable], optional): The list contains all variables to be saved.
-                                        Default: None
-        predicate(function, optional): The function selects the variables that make
-                                       `predicate(variable) == True`.
-                                       Default: None
-        filename(str, optional): If you prefer to save all variables in a single file,
-                                 use `filename` to specify it. Otherwise, let `filename` be None.
-                                 Default: None
-
-    Returns:
-        str: When saving parameters to a file, returns None.
-             When saving parameters to memory, returns a binary string containing parameters.
-
-    Raises:
-        TypeError: If `main_program` is not an instance of Program nor None.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            main_prog = fluid.Program()
-            startup_prog = fluid.Program()
-            with fluid.program_guard(main_prog, startup_prog):
-                data = paddle.static.data(name="img", shape=[64, 784])
-                w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
-                b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
-                hidden_w = paddle.matmul(x=data, y=w)
-                hidden_b = paddle.add(hidden_w, b)
-            place = fluid.CPUPlace()
-            exe = fluid.Executor(place)
-            exe.run(startup_prog)
-
-            # The first usage: use `vars` to set the saved variables.
-            var_list = [w, b]
-            path = "./my_paddle_vars"
-            fluid.io.save_vars(executor=exe, dirname=path, vars=var_list,
-                            filename="vars_file")
-            # w and b will be save in a file named "var_file".
-
-            # The second usage: use `predicate` to select the saved variable.
-            def name_has_fc(var):
-                res = "fc" in var.name
-                return res
-            param_path = "./my_paddle_model"
-            fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog, vars=None, predicate = name_has_fc)
-            # all variables whose names contain "fc " are saved.
-    """
-    save_to_memory = False
-    if dirname is None and filename is None:
-        save_to_memory = True
-
-    main_program = _get_valid_program(main_program)
-
-    if vars is None:
-        return save_vars(
-            executor,
-            main_program=main_program,
-            dirname=dirname,
-            vars=list(filter(predicate, main_program.list_vars())),
-            filename=filename,
-        )
-    else:
-        params_var_name = "saved_params"
-        # give warning when there is no var in model
-        if len(list(vars)) == 0:
-            warnings.warn(
-                "no variable in your model, please ensure there are any variables in your model to save"
-            )
-            return None
-
-        save_program = Program()
-        save_block = save_program.global_block()
-
-        save_var_map = {}
-        for each_var in vars:
-            # NOTE: don't save the variable which type is RAW
-            if each_var.type == core.VarDesc.VarType.RAW:
-                continue
-            new_var = _clone_var_in_block_(save_block, each_var)
-            if filename is None and save_to_memory is False:
-                save_file_path = os.path.join(
-                    os.path.normpath(dirname), new_var.name
-                )
-                save_block.append_op(
-                    type='save',
-                    inputs={'X': [new_var]},
-                    outputs={},
-                    attrs={'file_path': os.path.normpath(save_file_path)},
-                )
-            else:
-                save_var_map[new_var.name] = new_var
-
-        if filename is not None or save_to_memory:
-            save_var_list = []
-            for name in sorted(save_var_map.keys()):
-                save_var_list.append(save_var_map[name])
-
-            save_path = str()
-            if save_to_memory is False:
-                save_path = os.path.join(os.path.normpath(dirname), filename)
-
-            saved_params = save_block.create_var(
-                type=core.VarDesc.VarType.RAW, name=params_var_name
-            )
-            saved_params.desc.set_persistable(True)
-            save_block.append_op(
-                type='save_combine',
-                inputs={'X': save_var_list},
-                outputs={'Y': saved_params},
-                attrs={
-                    'file_path': save_path,
-                    'save_to_memory': save_to_memory,
-                },
-            )
-
-        # NOTE(zhiqiu): save op will add variable kLookupTablePath in save_program.desc,
-        # which leads to diff on save_program and its desc. Call _sync_with_cpp
-        # to keep consistency.
-        save_program._sync_with_cpp()
-        executor.run(save_program)
-        if save_to_memory:
-            return global_scope().find_var(params_var_name).get_bytes()
-
-
-@dygraph_not_support
-def save_params(executor, dirname, main_program=None, filename=None):
-    """
-    Save all parameters from the :code:`main_program` to
-    the folder :code:`dirname` or file :code:`filename`. You can refer to
-    :ref:`api_guide_model_save_reader_en` for more details.
-
-    Use the :code:`dirname` to specify the saving folder. If you would like to
-    save parameters in separate files, set :code:`filename` None; if you would
-    like to save all parameters in a single file, use :code:`filename` to specify
-    the file name.
-
-    Note:
-        Some variables are not Parameter while they are necessary for
-        training, such as learning rate, global step, etc. So you can NOT save
-        and continue your training just by :ref:`api_fluid_io_save_params`
-        and :ref:`api_fluid_io_load_params`. Please use :ref:`api_fluid_io_save_persistables`
-        and :ref:`api_fluid_io_load_persistables` instead.
-
-        If you want to save your model for the inference, please use the
-        :ref:`api_fluid_io_save_inference_model`. You can refer to
-        :ref:`api_guide_model_save_reader_en` for more details.
-
-    Args:
-        executor(Executor): The executor to run for saving parameters, You can
-                            refer to :ref:`api_guide_executor_en`.
-        dirname(str, optional): The saving directory path.
-                            When you need to save the parameter to the memory, set it to None.
-        main_program(Program, optional): The program whose parameters will be
-                                         saved. You can refer to
-                                         :ref:`api_guide_Program_en` for more
-                                         details. If it is None, the default main
-                                         program will be used.
-                                         Default: None
-        filename(str, optional): The file to save all parameters. If you prefer
-                                 to save parameters in different files, set it
-                                 to None.
-                                 Default: None
-
-    Returns:
-        str: When saving parameters to a file, returns None.
-             When saving parameters to memory, returns a binary string containing parameters.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-
-            paddle.enable_static()
-            params_path = "./my_paddle_model"
-            image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32')
-            label = fluid.data(name='label', shape=[None, 1], dtype='int64')
-            feeder = fluid.DataFeeder(feed_list=[image, label], place=fluid.CPUPlace())
-            predict = paddle.static.nn.fc(x=image, size=10, activation='softmax')
-
-            loss = paddle.nn.functional.cross_entropy(
-                input=predict, label=label,
-                reduction='none', use_softmax=False
-            )
-            avg_loss = paddle.mean(loss)
-
-            exe = fluid.Executor(fluid.CPUPlace())
-            exe.run(fluid.default_startup_program())
-            fluid.io.save_params(executor=exe, dirname=params_path)
-            # The parameters weights and bias of the fc layer in the network are going to
-            # be saved in different files in the path "./my_paddle_model"
-    """
-    return save_vars(
-        executor,
-        dirname=dirname,
-        main_program=main_program,
-        vars=None,
-        predicate=is_parameter,
-        filename=filename,
-    )
-
-
-def _save_distributed_persistables(executor, dirname, main_program):
-    """
-    save_persistables for distributed training.
-    the method will do things listed below:
-    1.save part of persistable variables on trainer.
-    2.receive "remote prefetch variables" from parameter servers and merge them.
-    3.save "distributed lookup table" on parameter servers.
-    4.receive "optimizer variables" from parameter servers and merge them.
-
-    Args:
-        executor(Executor): The executor to run for saving parameters.
-        dirname(str): The saving directory path.
-        main_program(Program): The program whose parameters will be
-                            saved. the main_program must be the trainer_program
-                            get after transpiler.
-
-    Returns:
-        None
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            exe = fluid.Executor(fluid.CPUPlace())
-            param_path = "./my_paddle_model"
-            t = paddle.distributed.transpiler.DistributeTranspiler()
-            t.transpile(...)
-            train_program = t.get_trainer_program()
-            _save_distributed_persistables(executor=exe, dirname=param_path, main_program=train_program)
-    """
-
-    def __save_remote_params(executor, dirname, remote_params_map):
-        """
-        receive params on pserver through rpc.
-        if the params are be sliced, will concat them to one, then save it.
-        """
-        if not remote_params_map:
-            return
-
-        prog = Program()
-        block = prog.global_block()
-
-        # recv optimize vars from pserver
-        for name, remote_params in remote_params_map.items():
-            origin = remote_params[0].origin
-            is_slice = remote_params[0].is_slice
-
-            slices = [None] * len(remote_params)
-            slice_varnames = [None] * len(remote_params)
-            remote_varnames = [None] * len(remote_params)
-            endpoints = [None] * len(remote_params)
-
-            for idx, optimizer in enumerate(remote_params):
-                block_id = optimizer.block_id
-                slice = optimizer.slice
-                endpoint = optimizer.endpoint
-
-                index = block_id if is_slice else idx
-                slices[index] = slice
-                slice_varnames[index] = "{}.slice.{}".format(slice.name, idx)
-                remote_varnames[index] = slice.name
-                endpoints[index] = endpoint
-
-            slice_shapes = []
-            for slice in slices:
-                tmp = [str(dim) for dim in slice.shape]
-                slice_shapes.append(",".join(tmp))
-
-            block.append_op(
-                type='recv_save',
-                attrs={
-                    "trainer_id": 0,
-                    "shape": origin.shape,
-                    "slice_shapes": slice_shapes,
-                    "slice_varnames": slice_varnames,
-                    "remote_varnames": remote_varnames,
-                    "endpoints": endpoints,
-                    "file_path": os.path.join(dirname, origin.name),
-                },
-            )
-
-        executor.run(prog)
-
-    def __save_distributed_lookup_tables(
-        executor, dirname, distributed_lookup_table, endpoints
-    ):
-        """
-        because the distributed lookup table may too huge to merge and save at one place,
-        it will be saved at parameter server independent respectively.
-
-        the save directory is dirname/"__lookup_table__".
-
-        """
-        prog = Program()
-        block = prog.global_block()
-
-        # if there is lookup table, the trainer 0 will notify all pserver to save.
-        lookup_table_filename = os.path.join(dirname, "__lookup_table__")
-        attrs = {}
-        attrs['epmap'] = endpoints
-        attrs['dir'] = lookup_table_filename
-        attrs['lookup_table'] = distributed_lookup_table
-        block.append_op(
-            type='checkpoint_notify', inputs={}, outputs={}, attrs=attrs
-        )
-        executor.run(prog)
-
-    def __exclude_vars(exclude_var_names=[]):
-        def is_valid(var):
-            if var.name in exclude_var_names:
-                return False
-            if (
-                var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH
-                or var.desc.type() == core.VarDesc.VarType.FETCH_LIST
-                or var.desc.type() == core.VarDesc.VarType.READER
-            ):
-                return False
-            return var.persistable
-
-        return is_valid
-
-    if not isinstance(main_program, Program):
-        raise TypeError("'main_program' should be an instance of Program.")
-
-    if not main_program._is_distributed:
-        raise ValueError(
-            "'_save_distributed_persistables' just be designed for distributed training."
-        )
-
-    remote_params_map = (
-        main_program._parameters_on_pservers.get_distributed_vars_by_vtypes(
-            ["Optimizer", "RemotePrefetch"], groupby=True
-        )
-    )
-
-    exclude_var_names = []
-    if remote_params_map:
-        exclude_var_names.extend(remote_params_map.keys())
-
-    if main_program._distributed_lookup_table:
-        if isinstance(main_program._distributed_lookup_table, list):
-            exclude_var_names.extend(main_program._distributed_lookup_table)
-        else:
-            exclude_var_names.append(main_program._distributed_lookup_table)
-
-    local_vars = list(
-        filter(__exclude_vars(exclude_var_names), main_program.list_vars())
-    )
-    save_vars(
-        executor, main_program=main_program, dirname=dirname, vars=local_vars
-    )
-
-    if main_program._is_chief:
-        if remote_params_map:
-            __save_remote_params(executor, dirname, remote_params_map)
-        if main_program._distributed_lookup_table:
-            __save_distributed_lookup_tables(
-                executor,
-                dirname,
-                main_program._distributed_lookup_table,
-                main_program._endpoints,
-            )
-
-
-@dygraph_not_support
-def save_persistables(executor, dirname, main_program=None, filename=None):
-    """
-    Save all persistable variables from :code:`main_program` to
-    the folder :code:`dirname` or file :code:`filename`. You can refer to
-    :ref:`api_guide_model_save_reader_en` for more details. And then
-    saves these persistables variables to the folder :code:`dirname` or file
-    :code:`filename`.
-
-    The :code:`dirname` is used to specify the folder where persistable variables
-    are going to be saved. If you would like to save variables in separate
-    files, set :code:`filename` None; if you would like to save all variables in a
-    single file, use :code:`filename` to specify the file name.
-
-    Args:
-        executor(Executor): The executor to run for saving persistable variables.
-                            You can refer to :ref:`api_guide_executor_en` for
-                            more details.
-
-        dirname(str, optional): The saving directory path.
-                            When you need to save the parameter to the memory, set it to None.
-        main_program(Program, optional): The program whose persistbale variables will
-                                         be saved. You can refer to
-                                         :ref:`api_guide_Program_en` for more details.
-                                         If it is None, the default main program will
-                                         be used.
-                                         Default: None.
-        filename(str, optional): The file to save all variables. If you prefer to
-                                 save variables in different files, set it to None.
-                                 Default: None.
-
-    Returns:
-        str: When saving parameters to a file, returns None.
-             When saving parameters to memory, returns a binary string containing parameters.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            dir_path = "./my_paddle_model"
-            file_name = "persistables"
-            image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32')
-            label = fluid.data(name='label', shape=[None, 1], dtype='int64')
-            feeder = fluid.DataFeeder(feed_list=[image, label], place=fluid.CPUPlace())
-
-            predict = paddle.static.nn.fc(x=image, size=10, activation='softmax')
-            loss = paddle.nn.functional.cross_entropy(
-                input=predict, label=label,
-                reduction='none', use_softmax=False
-            )
-            avg_loss = paddle.mean(loss)
-            exe = fluid.Executor(fluid.CPUPlace())
-            exe.run(fluid.default_startup_program())
-            fluid.io.save_persistables(executor=exe, dirname=dir_path, filename=file_name)
-            # The persistables variables weights and bias in the fc layer of the network
-            # are going to be saved in the same file named "persistables" in the path
-            # "./my_paddle_model"
-    """
-    if main_program and main_program._is_distributed:
-        return _save_distributed_persistables(
-            executor, dirname=dirname, main_program=main_program
-        )
-    else:
-        return save_vars(
-            executor,
-            dirname=dirname,
-            main_program=main_program,
-            vars=None,
-            predicate=is_persistable,
-            filename=filename,
-        )
-
-
-def load_vars(
-    executor,
-    dirname,
-    main_program=None,
-    vars=None,
-    predicate=None,
-    filename=None,
-):
-    """
-    :api_attr: Static Graph
-
-    This API loads variables from files by executor.
-
-    There are two ways to specify the variables to be loaded: the first way, set
-    variables in a list and assign it to the `vars`; the second way, use the
-    `predicate` function to select variables that make `predicate(variable) == True`.
-    The first way has a higher priority.
-
-    The `dirname` is used to specify the folder where to load variables.
-    If variables were saved in separate files in the folder `dirname`,
-    set `filename` None. If all variables were saved in a single file,
-    use `filename` to specify it.
-
-    Args:
-        executor(Executor): The executor to run for loading variables.
-        dirname(str): The folder where to load the variables.
-        main_program(Program, optional): The program whose variables will be loaded.
-                                    If it is None, the default main program will
-                                    be used automatically.
-                                    Default: None
-        vars(list[Variable], optional): The list that contains all variables to be loaded.
-                                   Default: None
-        predicate(function, optional): The function selects variables that make
-                                        `predicate(variable) == True`.
-                                        Default: None
-        filename(str, optional): The file which saved all required variables. If variables
-                                were saved in separate files, set it to be None.
-                                Default: None
-
-    Returns:
-        None
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            main_prog = fluid.Program()
-            startup_prog = fluid.Program()
-            with fluid.program_guard(main_prog, startup_prog):
-                data = paddle.static.data(name="img", shape=[64, 784])
-                w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
-                b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
-                hidden_w = paddle.matmul(x=data, y=w)
-                hidden_b = paddle.add(hidden_w, b)
-            place = fluid.CPUPlace()
-            exe = fluid.Executor(place)
-            exe.run(startup_prog)
-
-            # The first usage: using `vars` to specify the variables.
-            path = "./my_paddle_vars"
-            var_list = [w, b]
-            fluid.io.save_vars(executor=exe, dirname=path, vars=var_list,
-                               filename="vars_file")
-            fluid.io.load_vars(executor=exe, dirname=path, vars=var_list,
-                               filename="vars_file")
-            # w and b will be loaded, and they are supposed to
-            # be saved in the same file named 'var_file' in the path "./my_paddle_vars".
-
-            # The second usage: using the `predicate` function to select variables
-            param_path = "./my_paddle_model"
-            def name_has_fc(var):
-                res = "fc" in var.name
-                return res
-            fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog,
-                              vars=None, predicate=name_has_fc)
-            fluid.io.load_vars(executor=exe, dirname=param_path, main_program=main_prog,
-                               vars=None, predicate=name_has_fc)
-            # Load All variables in the `main_program` whose name includes "fc".
-            # And all the variables are supposed to be saved in separate files.
-
-    """
-    vars_from_memory = False
-    if dirname is not None:
-        dirname = os.path.normpath(dirname)
-    else:
-        vars_from_memory = True
-
-    if vars is None:
-        if main_program is None:
-            main_program = default_main_program()
-        if not isinstance(main_program, Program):
-            raise TypeError(
-                "The type of input main_program is invalid, expected type is fluid.Program, but received %s"
-                % type(main_program)
-            )
-
-        load_vars(
-            executor,
-            dirname=dirname,
-            main_program=main_program,
-            vars=list(filter(predicate, main_program.list_vars())),
-            filename=filename,
-        )
-    else:
-        load_prog = Program()
-        load_block = load_prog.global_block()
-
-        if main_program is None:
-            main_program = default_main_program()
-
-        if not isinstance(main_program, Program):
-            raise TypeError(
-                "The type of input main_program is invalid, expected type is fluid.Program, but received %s"
-                % type(main_program)
-            )
-
-        # save origin param shape
-        orig_para_shape = {}
-        load_var_map = {}
-
-        check_vars = []
-        sparse_vars = []
-
-        for each_var in vars:
-            assert isinstance(each_var, Variable)
-
-            if each_var.type == core.VarDesc.VarType.RAW:
-                continue
-
-            if isinstance(each_var, Parameter):
-                orig_para_shape[each_var.name] = tuple(
-                    each_var.desc.get_shape()
-                )
-
-            if each_var.type == core.VarDesc.VarType.SELECTED_ROWS:
-                sparse_vars.append(each_var)
-                continue
-
-            new_var = _clone_var_in_block_(load_block, each_var)
-            check_vars.append(each_var)
-
-            if filename is None:
-                if dirname is None:
-                    raise ValueError(
-                        "The directory path and params cannot be None at the same time."
-                    )
-                load_block.append_op(
-                    type='load',
-                    inputs={},
-                    outputs={'Out': [new_var]},
-                    attrs={'file_path': os.path.join(dirname, new_var.name)},
-                )
-            else:
-                load_var_map[new_var.name] = new_var
-
-        for each_var in sparse_vars:
-            assert isinstance(each_var, Variable)
-
-            if filename is not None:
-                raise ValueError(
-                    "SelectedRows can not be load with load_combine"
-                )
-
-            new_var = _clone_var_in_block_(load_block, each_var)
-
-            var_path = os.path.join(dirname, new_var.name)
-            if not os.path.exists(var_path):
-                raise ValueError(
-                    "SelectedRows var {} can not find at {}".format(
-                        new_var.name, var_path
-                    )
-                )
-
-            if os.path.isfile(var_path):
-                load_block.append_op(
-                    type='load',
-                    inputs={},
-                    outputs={'Out': [new_var]},
-                    attrs={'file_path': os.path.join(dirname, new_var.name)},
-                )
-            else:
-                blocks = []
-                block_paths = os.listdir(var_path)
-
-                for block in block_paths:
-                    if block.startswith(new_var.name):
-                        blocks.append(block)
-
-                slices = []
-                for block in blocks:
-                    slice = load_block.create_var(
-                        name=block,
-                        type=new_var.type,
-                        shape=new_var.shape,
-                        dtype=new_var.dtype,
-                        persistable=False,
-                    )
-                    slices.append(slice)
-
-                    file_path = os.path.join(var_path, block, "Param")
-                    load_block.append_op(
-                        type='load',
-                        inputs={},
-                        outputs={'Out': [slice]},
-                        attrs={'file_path': file_path},
-                    )
-
-                load_block.append_op(
-                    type='lookup_sparse_table_merge',
-                    inputs={'X': slices},
-                    outputs={'Out': new_var},
-                    attrs={},
-                )
-
-        if filename is not None:
-            load_var_list = []
-            for name in sorted(load_var_map.keys()):
-                load_var_list.append(load_var_map[name])
-
-            if vars_from_memory is False:
-                filename = os.path.join(dirname, filename)
-
-            load_block.append_op(
-                type='load_combine',
-                inputs={},
-                outputs={"Out": load_var_list},
-                attrs={
-                    'file_path': filename,
-                    'model_from_memory': vars_from_memory,
-                },
-            )
-        executor.run(load_prog)
-
-        # check var shape
-        for each_var in check_vars:
-            if not isinstance(each_var, Parameter):
-                continue
-            var_temp = paddle.fluid.global_scope().find_var(each_var.name)
-            assert var_temp is not None, "can't not find var: " + each_var.name
-            new_shape = (np.array(var_temp.get_tensor())).shape
-            assert each_var.name in orig_para_shape, (
-                each_var.name + "MUST in var list"
-            )
-            orig_shape = orig_para_shape.get(each_var.name)
-            if new_shape != orig_shape:
-                raise RuntimeError(
-                    "Variable's shape does not match, the Program requires a parameter with the shape of ({}), "
-                    "while the loaded parameter (namely [ {} ]) has a shape of  ({}).".format(
-                        orig_shape, each_var.name, new_shape
-                    )
-                )
-
-
-@dygraph_not_support
-def load_params(executor, dirname, main_program=None, filename=None):
-    """
-    :api_attr: Static Graph
-
-    This API filters out all parameters from the give ``main_program``
-    and then tries to load these parameters from the directory ``dirname`` or
-    the file ``filename``.
-
-    Use the ``dirname`` to specify the directory where parameters were saved. If
-    parameters were saved in separate files under the directory `dirname`, set
-    ``filename`` as None; if all parameters were saved in a single file, use
-    ``filename`` to specify the file name.
-
-    **Note**:
-        Some variables are not Parameter while they are necessary for
-        training, such as learning rate, global step, etc. So you cannot save and
-        continue your training just by using :ref:`api_fluid_io_save_params` and
-        :ref:`api_fluid_io_load_params`. Please use :ref:`api_fluid_io_save_persistables`
-        and :ref:`api_fluid_io_load_persistables` instead.
-
-        If you want to load the pre-trained model structure and parameters
-        for the inference, please use the :ref:`api_fluid_io_load_inference_model` API. You can
-        refer to :ref:`api_guide_model_save_reader_en` for more details.
-
-    Args:
-        executor(Executor): The executor used for loading parameters.
-                            See :ref:`api_guide_executor_en` for more details about it.
-        dirname(str): The directory path.
-        main_program(Program, optional): The program whose parameters will be
-                                    loaded. If it is None, the ``default_main_program``
-                                    will be used automatically. See :ref:`api_guide_Program_en`
-                                    for more about ``Program``.
-                                    Default: None.
-        filename(str, optional): The file which saved all parameters. If parameters
-                            were saved in separated files, set it to None.
-                            Default: None.
-
-    Returns:
-        None
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            exe = fluid.Executor(fluid.CPUPlace())
-            param_path = "./my_paddle_model"
-            prog = fluid.default_main_program()
-            fluid.io.load_params(executor=exe, dirname=param_path,
-                                main_program=None)
-    """
-    load_vars(
-        executor,
-        dirname=dirname,
-        main_program=main_program,
-        predicate=is_parameter,
-        filename=filename,
-    )
-
-
-@dygraph_not_support
-def load_persistables(executor, dirname, main_program=None, filename=None):
-    """
-    :api_attr: Static Graph
-
-    This API filters out all variables with ``persistable==True`` from the
-    given ``main_program`` and then tries to load these variables from the
-    directory ``dirname`` or the file ``filename``.
-
-    Use the ``dirname`` to specify the directory where persistable variables
-    (refer to :ref:`api_guide_model_save_reader_en`) were saved. If variables
-    were saved in separate files, set ``filename`` as None; if all variables
-    were saved in a single file, use ``filename`` to specify the file name.
-
-    Args:
-        executor(Executor): The executor used for loading persistable variables.
-                            See :ref:`api_guide_executor_en` for more details about it.
-        dirname(str): The directory path.
-        main_program(Program, optional): The program whose persistable variables will
-                                    be loaded. If it is None, the ``default_main_program``
-                                    will be used automatically. See :ref:`api_guide_Program_en`
-                                    for more about ``Program``.
-                                    Default: None.
-        filename(str, optional): The file which saved all persistable variables. If variables
-                                 were saved in separated files, set it to None.
-                                 Default: None.
-
-    Returns:
-        None
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            exe = fluid.Executor(fluid.CPUPlace())
-            param_path = "./my_paddle_model"
-            prog = fluid.default_main_program()
-            fluid.io.load_persistables(executor=exe, dirname=param_path,
-                                       main_program=None)
-    """
-
-    if main_program and main_program._is_distributed:
-        _load_distributed_persistables(
-            executor, dirname=dirname, main_program=main_program
-        )
-    else:
-        load_vars(
-            executor,
-            dirname=dirname,
-            main_program=main_program,
-            predicate=is_persistable,
-            filename=filename,
-        )
-
-
-def _load_distributed_persistables(executor, dirname, main_program=None):
-    """
-    customized load_persistables for distributed training.
-    it should be used on parameter server,
-
-    Args:
-        executor(Executor): The executor to run for saving parameters.
-        dirname(str): The load directory path.
-        main_program(Program): The program whose parameters will be
-                            loaded. the main_program must be the pserver_program
-                            get after transpiler.
-
-    Returns:
-        None
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            exe = fluid.Executor(fluid.CPUPlace())
-            param_path = "./my_paddle_model"
-            t = paddle.distributed.transpiler.DistributeTranspiler()
-            t.transpile(...)
-            pserver_prog = t.get_pserver_program(...)
-            _load_distributed_persistables(executor=exe, dirname=param_path, main_program=pserver_prog)
-    """
-
-    def __is_distributed_part_var(varname):
-        trainer_idx = varname.find(".trainer_")
-        block_idx = varname.find(".block")
-        return trainer_idx or block_idx
-
-    def __load_persistable_vars(executor, dirname, need_load_vars):
-        load_prog = Program()
-        load_block = load_prog.global_block()
-        need_delete_vars = []
-
-        for param in need_load_vars:
-            origin_var = param.origin
-            slice_var = param.slice
-            is_slice = param.is_slice
-            offset = param.offset
-
-            if is_slice:
-                slice = load_block.create_var(
-                    name=slice_var.name,
-                    type=slice_var.type,
-                    shape=slice_var.shape,
-                    dtype=slice_var.dtype,
-                    persistable=True,
-                )
-
-                load_block.append_op(
-                    type='load',
-                    inputs={},
-                    outputs={'Out': [slice]},
-                    attrs={
-                        'file_path': os.path.join(dirname, origin_var.name),
-                        'seek': offset,
-                        'shape': slice.shape,
-                    },
-                )
-            else:
-                origin = load_block.create_var(
-                    name="{}".format(origin_var.name),
-                    type=origin_var.type,
-                    shape=origin_var.shape,
-                    dtype=origin_var.dtype,
-                    persistable=True,
-                )
-                load_block.append_op(
-                    type='load',
-                    inputs={},
-                    outputs={'Out': [origin]},
-                    attrs={'file_path': os.path.join(dirname, origin_var.name)},
-                )
-
-        load_block.append_op(
-            type='delete_var',
-            inputs={'X': need_delete_vars},
-        )
-
-        executor.run(load_prog)
-
-    if not isinstance(main_program, Program):
-        raise TypeError("'main_program' should be an instance of Program.")
-
-    if not main_program._is_distributed:
-        raise ValueError(
-            "'_load_distributed_persistables' just be designed for distributed training."
-        )
+__all__ = [
+    'save_inference_model',
+    'load_inference_model',
+] + reader.__all__
 
-    if not main_program._ps_endpoint:
-        raise ValueError(
-            "'_load_distributed_persistables' need current_endpoint set in DistributeTranspiler.transpile"
-        )
 
-    need_load_vars = (
-        main_program._parameters_on_pservers.get_distributed_vars_by_ep(
-            main_program._ps_endpoint
-        )
-    )
-    __load_persistable_vars(executor, dirname, need_load_vars)
+_logger = get_logger(
+    __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
+)
 
 
 def prepend_feed_ops(
@@ -1430,7 +240,7 @@ def save_inference_model(
         ):
             raise ValueError("'target_vars' should be a list of Variable.")
 
-    main_program = _get_valid_program(main_program)
+    main_program = paddle.static.io._get_valid_program(main_program)
 
     # remind user to set auc_states to zeros if the program contains auc op
     all_ops = main_program.global_block().ops
@@ -1534,7 +344,9 @@ def save_inference_model(
     if params_filename is not None:
         params_filename = os.path.basename(params_filename)
 
-    save_persistables(executor, save_dirname, main_program, params_filename)
+    paddle.distributed.io.save_persistables(
+        executor, save_dirname, main_program, params_filename
+    )
     return target_var_name_list
 
 
@@ -1671,7 +483,9 @@ def load_inference_model(
             "Unsupported program version: %d\n" % program._version()
         )
     # Binary data also need versioning.
-    load_persistables(executor, load_dirname, program, params_filename)
+    paddle.distributed.io.load_persistables(
+        executor, load_dirname, program, params_filename
+    )
 
     if pserver_endpoints:
         program = _endpoints_replacement(program, pserver_endpoints)
@@ -1692,859 +506,3 @@ def _endpoints_replacement(program, endpoints):
             op.set_attr(ENDPOINT_MAP, endpoints)
     program._sync_with_cpp()
     return program
-
-
-def get_parameter_value(para, executor):
-    """
-    Get the LoDTensor value of the given parameter.
-
-    Args:
-        para(Parameter): The parameter to get value from.
-        executor(Executor): The executor to run for retrieving the value.
-
-    Returns:
-        numpy.array: The given parameter's values.
-
-    Raises:
-        AssertionError: If the `para` is not an instance of Parameter.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            exe = fluid.Executor(fluid.CPUPlace())
-            param = fluid.default_main_program().global_block().var('fc.w')
-            p = fluid.io.get_parameter_value(param, exe)
-
-    """
-    assert is_parameter(para), "The input variable is not parameter."
-
-    get_program = Program()
-    block = get_program.global_block()
-    new_var = _clone_var_in_block_(block, para)
-    return executor.run(get_program, feed={}, fetch_list=[new_var])[0]
-
-
-def get_parameter_value_by_name(name, executor, program=None):
-    """
-    Get the LoDTensor value of a certain parameter by its name.
-
-    Args:
-        name(str): The parameter's name.
-        executor(Executor): The executor to run for retrieving the value.
-        program(Program | None): The program where to find the parameter.
-                               If it's set to be None, the function will
-                               try to find the parameter in the default
-                               main program.
-
-    Returns:
-        numpy.array: The parameter's values.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            exe = fluid.Executor(fluid.CPUPlace())
-            p = fluid.io.get_parameter_value('fc.w', exe)
-    """
-    if program is None:
-        program = default_main_program()
-    var = program.global_block().var(name)
-    return get_parameter_value(var, executor)
-
-
-def _save_persistable_nodes(executor, dirname, graph):
-    """
-    Save persistable nodes to the given directory by the executor.
-
-    Args:
-        executor(Executor): The executor to run for saving node values.
-        dirname(str): The directory path.
-        graph(IrGraph): All the required persistable nodes in the graph will be saved.
-    """
-    persistable_node_names = set()
-    persistable_nodes = []
-    all_persistable_nodes = graph.all_persistable_nodes()
-    for node in all_persistable_nodes:
-        name = node.name()
-        if name not in persistable_node_names:
-            persistable_node_names.add(name)
-            persistable_nodes.append(node)
-    program = Program()
-    var_list = []
-    for node in persistable_nodes:
-        var_desc = node.var()
-        if (
-            var_desc.type() == core.VarDesc.VarType.RAW
-            or var_desc.type() == core.VarDesc.VarType.READER
-        ):
-            continue
-        var = program.global_block().create_var(
-            name=var_desc.name(),
-            shape=var_desc.shape(),
-            dtype=var_desc.dtype(),
-            type=var_desc.type(),
-            lod_level=var_desc.lod_level(),
-            persistable=var_desc.persistable(),
-        )
-        var_list.append(var)
-    save_vars(executor=executor, dirname=dirname, vars=var_list)
-
-
-def _load_persistable_nodes(executor, dirname, graph):
-    """
-    Load persistable node values from the given directory by the executor.
-
-    Args:
-        executor(Executor): The executor to run for loading node values.
-        dirname(str): The directory path.
-        graph(IrGraph): All the required persistable nodes in the graph will be loaded.
-    """
-    persistable_node_names = set()
-    persistable_nodes = []
-    all_persistable_nodes = graph.all_persistable_nodes()
-    for node in all_persistable_nodes:
-        name = node.name()
-        if name not in persistable_node_names:
-            persistable_node_names.add(name)
-            persistable_nodes.append(node)
-    program = Program()
-    var_list = []
-
-    def _exist(var):
-        return os.path.exists(os.path.join(dirname, var.name))
-
-    for node in persistable_nodes:
-        var_desc = node.var()
-        if (
-            var_desc.type() == core.VarDesc.VarType.RAW
-            or var_desc.type() == core.VarDesc.VarType.READER
-        ):
-            continue
-        var = program.global_block().create_var(
-            name=var_desc.name(),
-            shape=var_desc.shape(),
-            dtype=var_desc.dtype(),
-            type=var_desc.type(),
-            lod_level=var_desc.lod_level(),
-            persistable=var_desc.persistable(),
-        )
-        if _exist(var):
-            var_list.append(var)
-        else:
-            _logger.warn("Cannot find the var %s!!!" % (node.name()))
-    load_vars(executor=executor, dirname=dirname, vars=var_list)
-
-
-def _unpack_saved_dict(saved_obj, protocol):
-    temp_saved_obj = {}
-    unpack_infor = {}
-    # When pickle protocol=2 or protocol=3 the serialized object cannot be larger than 4G.
-    if 1 < protocol < 4:
-        if isinstance(saved_obj, dict):
-            for key, value in saved_obj.items():
-                if isinstance(value, np.ndarray):
-                    MAX_NUMBER_OF_ELEMENT = int(
-                        (2**30 - 1) / value.dtype.itemsize
-                    )
-                    num_element = np.prod(value.shape)
-                    if num_element > MAX_NUMBER_OF_ELEMENT:
-                        unpack_infor[key] = {}
-                        unpack_infor[key]["OriginShape"] = value.shape
-                        unpack_infor[key]["slices"] = []
-                        value = value.flatten()
-                        for i in range(
-                            int(
-                                math.ceil(
-                                    num_element * 1.0 / MAX_NUMBER_OF_ELEMENT
-                                )
-                            )
-                        ):
-                            part_name = key + "@@." + str(i)
-                            unpack_infor[key]["slices"].append(part_name)
-                            temp_saved_obj[part_name] = value[
-                                i
-                                * MAX_NUMBER_OF_ELEMENT : MAX_NUMBER_OF_ELEMENT
-                                * (i + 1)
-                            ]
-
-    if unpack_infor:
-        for key, value in unpack_infor.items():
-            if key in saved_obj:
-                saved_obj.pop(key)
-                for part in value['slices']:
-                    saved_obj[part] = temp_saved_obj[part]
-        saved_obj['UnpackBigParamInfor@@'] = unpack_infor
-    return saved_obj
-
-
-def _pack_loaded_dict(load_obj):
-    if isinstance(load_obj, dict):
-        unpack_info = 'UnpackBigParamInfor@@'
-        if unpack_info in load_obj:
-            removes = []
-            for key, value in load_obj[unpack_info].items():
-                slices = [load_obj[part] for part in value["slices"]]
-                load_obj[key] = np.concatenate(slices).reshape(
-                    value["OriginShape"]
-                )
-                removes += value["slices"]
-            for key in removes:
-                load_obj.pop(key)
-            load_obj.pop(unpack_info)
-
-    return load_obj
-
-
-@static_only
-def _legacy_save(param_dict, model_path, protocol=2):
-    def get_tensor(var):
-        if isinstance(var, (core.VarBase, core.eager.Tensor)):
-            return var.numpy()
-        elif isinstance(var, core.LoDTensor):
-            return np.array(var)
-        return var
-
-    param_dict = {name: get_tensor(param_dict[name]) for name in param_dict}
-
-    # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
-    if (
-        _is_file_path(model_path)
-        and sys.platform == 'darwin'
-        and sys.version_info.major == 3
-    ):
-        pickle_bytes = pickle.dumps(param_dict, protocol=protocol)
-        with open(model_path, 'wb') as f:
-            max_bytes = 2**30
-            for i in range(0, len(pickle_bytes), max_bytes):
-                f.write(pickle_bytes[i : i + max_bytes])
-    else:
-        with _open_file_buffer(model_path, 'wb') as f:
-            pickle.dump(param_dict, f, protocol=protocol)
-
-
-@static_only
-def save(program, model_path, protocol=4, **configs):
-    """
-
-    This function save parameters, optimizer information and network description to model_path.
-
-    The parameters contains all the trainable Tensor, will save to a file with suffix ".pdparams".
-    The optimizer information contains all the Tensor used by optimizer. For Adam optimizer, contains beta1, beta2, momentum etc. All the information will save to a file with suffix ".pdopt". (If the optimizer have no Tensor need to save (like SGD), the fill will not generated).
-    The network description is the description of the program. It's only used for deployment. The description  will save to a file with a suffix ".pdmodel".
-
-    Args:
-        program(Program) : The program to saved.
-        model_path(str): the file prefix to save the program. The format is "dirname/file_prefix". If file_prefix is empty str. A exception will be raised
-        protocol(int, optional): The protocol version of pickle module must be greater than 1 and less than 5.
-                                 Default: 4
-        configs(dict, optional) : optional keyword arguments.
-
-    Returns:
-        None
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.static as static
-
-            paddle.enable_static()
-
-            x = static.data(name="x", shape=[10, 10], dtype='float32')
-            y = static.nn.fc(x, 10)
-            z = static.nn.fc(y, 10)
-
-            place = paddle.CPUPlace()
-            exe = static.Executor(place)
-            exe.run(static.default_startup_program())
-            prog = static.default_main_program()
-
-            static.save(prog, "./temp")
-    """
-
-    base_name = os.path.basename(model_path)
-    assert (
-        base_name != ""
-    ), "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received model_path is empty string."
-    if 'pickle_protocol' in configs:
-        protocol = configs['pickle_protocol']
-        warnings.warn(
-            "'pickle_protocol' is a deprecated argument. Please use 'protocol' instead."
-        )
-
-    if not isinstance(protocol, int):
-        raise ValueError(
-            "The 'protocol' MUST be `int`, but received {}".format(
-                type(protocol)
-            )
-        )
-
-    if protocol < 2 or protocol > 4:
-        raise ValueError(
-            "Expected 1<'protocol'<5, but received protocol={}".format(protocol)
-        )
-
-    dir_name = os.path.dirname(model_path)
-    if dir_name and not os.path.exists(dir_name):
-        os.makedirs(dir_name)
-
-    def get_tensor(var):
-        t = global_scope().find_var(var.name).get_tensor()
-        return np.array(t)
-
-    parameter_list = list(filter(is_parameter, program.list_vars()))
-    param_dict = {p.name: get_tensor(p) for p in parameter_list}
-
-    param_dict = _unpack_saved_dict(param_dict, protocol)
-
-    # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
-    if sys.platform == 'darwin' and sys.version_info.major == 3:
-        pickle_bytes = pickle.dumps(param_dict, protocol=protocol)
-        with open(model_path + ".pdparams", 'wb') as f:
-            max_bytes = 2**30
-            for i in range(0, len(pickle_bytes), max_bytes):
-                f.write(pickle_bytes[i : i + max_bytes])
-    else:
-        with open(model_path + ".pdparams", 'wb') as f:
-            pickle.dump(param_dict, f, protocol=protocol)
-
-    optimizer_var_list = list(
-        filter(is_belong_to_optimizer, program.list_vars())
-    )
-
-    opt_dict = {p.name: get_tensor(p) for p in optimizer_var_list}
-    with open(model_path + ".pdopt", 'wb') as f:
-        pickle.dump(opt_dict, f, protocol=protocol)
-
-    main_program = program.clone()
-    program.desc.flush()
-    main_program.desc._set_version()
-    paddle.fluid.core.save_op_version_info(program.desc)
-
-    with open(model_path + ".pdmodel", "wb") as f:
-        f.write(program.desc.serialize_to_string())
-
-
-def _pickle_loads_mac(path, f):
-    pickle_bytes = bytearray(0)
-    file_size = os.path.getsize(path)
-    max_bytes = 2**30
-    for _ in range(0, file_size, max_bytes):
-        pickle_bytes += f.read(max_bytes)
-    load_result = pickle.loads(pickle_bytes, encoding='latin1')
-    return load_result
-
-
-@static_only
-def load(program, model_path, executor=None, var_list=None):
-    """
-    :api_attr: Static Graph
-
-    This function get parameters and optimizer information from program, and then get corresponding value from file.
-    An exception will throw if shape or dtype of the parameters is not match.
-
-    This function can also load model file saved with [ save_params, save_persistables, save_vars ].
-    var_list can not be None  when load single model file
-    ( filename is not None When save_params, save_persistables or save_vars is called ).
-
-    Args:
-        program(Program): The program will be loaded
-        model_path(str): The file prefix store the program
-        executor(Executor, optional): The executor used for initialize the parameter
-                                      When startup program is not run.
-        var_list(list|tuple, optional): The Tensor list/tuple to load single model file saved with
-                                  [ save_params, save_persistables, save_vars ].
-                                  Default: None
-
-    Returns:
-        None
-
-     Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.static as static
-
-            paddle.enable_static()
-
-            x = static.data(name="x", shape=[10, 10], dtype='float32')
-            y = static.nn.fc(x, 10)
-            z = static.nn.fc(y, 10)
-
-            place = paddle.CPUPlace()
-            exe = static.Executor(place)
-            exe.run(static.default_startup_program())
-            prog = static.default_main_program()
-
-            static.save(prog, "./temp")
-            static.load(prog, "./temp")
-    """
-
-    assert executor is None or isinstance(executor, Executor)
-
-    model_prefix = model_path
-    if model_prefix.endswith(".pdparams"):
-        model_prefix = model_prefix[:-9]
-    elif model_prefix.endswith(".pdopt"):
-        model_prefix = model_prefix[:-6]
-    elif model_prefix.endswith(".pdmodel"):
-        model_prefix = model_prefix[:-8]
-
-    parameter_file_name = model_prefix + ".pdparams"
-
-    if not os.path.exists(parameter_file_name):
-        # model file save by fluid.save not found, try to load model file saved with
-        # [save_vars, save_params, save_persistables]
-        _logger.debug(
-            "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format(
-                parameter_file_name
-            )
-        )
-        if executor is None:
-            raise ValueError(
-                "executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
-            )
-
-        if var_list is not None:
-            var_list_names = [var.name for var in var_list]
-        else:
-            var_list_names = None
-
-        if os.path.isdir(model_path):
-            binary_file_set = set()
-            for root, dirs, files in os.walk(model_path, topdown=False):
-                for f in files:
-                    binary_file_set.add(
-                        os.path.join(root, f).replace("\\", "/")
-                    )
-            program_var_list = list(program.list_vars())
-            loaded_var_list = []
-            for var in program_var_list:
-                var_path = os.path.join(model_path, var.name).replace("\\", "/")
-                load_condition = (
-                    var_list_names is None or var.name in var_list_names
-                )
-                if var_path in binary_file_set and load_condition:
-                    loaded_var_list.append(var)
-                    binary_file_set.remove(var_path)
-            if len(binary_file_set) > 0:
-                unused_var_list = " ".join(list(binary_file_set))
-                _logger.warning(
-                    "variable file [ %s ] not used"
-                    % (" ".join(list(binary_file_set)))
-                )
-            try:
-                load_vars(
-                    executor=executor, dirname=model_path, vars=loaded_var_list
-                )
-            except RuntimeError as e:
-                _logger.error(e)
-                raise e
-            except:
-                raise RuntimeError(
-                    "Failed to load model file, please make sure model file is saved with the "
-                    "following APIs: save_params, save_persistables, save_vars"
-                )
-
-            return
-        elif os.path.isfile(model_path):
-            if var_list is None:
-                raise ValueError(
-                    "var_list is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
-                )
-            program_var_list = program.list_vars()
-            program_var_name_set = set([var.name for var in program_var_list])
-
-            # check all the variable inlcuded in program
-            for var in var_list:
-                if var.name not in program_var_name_set:
-                    raise LookupError(
-                        "loaded var [{}] is not in program variable list"
-                    )
-
-            dir_name, file_name = os.path.split(model_path)
-            try:
-                load_vars(
-                    executor=executor,
-                    dirname=dir_name,
-                    vars=var_list,
-                    filename=file_name,
-                )
-            except RuntimeError as e:
-                _logger.error(e)
-                raise e
-            except:
-                raise RuntimeError(
-                    "Failed to load model file , please make sure model file is saved with the "
-                    "the following APIs: [ save_params, save_persistables, save_vars ]. "
-                    "When these API called, filename CANNOT be None"
-                )
-
-            return
-
-    def set_var(var, ndarray):
-        t = global_scope().find_var(var.name).get_tensor()
-        p = t._place()
-        if p.is_cpu_place():
-            place = paddle.fluid.CPUPlace()
-        elif p.is_cuda_pinned_place():
-            place = paddle.fluid.CUDAPinnedPlace()
-        elif p.is_xpu_place():
-            p = paddle.fluid.core.Place()
-            p.set_place(t._place())
-            place = paddle.fluid.XPUPlace(p.xpu_device_id())
-        elif p.is_npu_place():
-            p = paddle.fluid.core.Place()
-            p.set_place(t._place())
-            place = paddle.fluid.NPUPlace(p.npu_device_id())
-        elif p.is_mlu_place():
-            p = paddle.fluid.core.Place()
-            p.set_place(t._place())
-            place = paddle.fluid.MLUPlace(p.mlu_device_id())
-        else:
-            p = paddle.fluid.core.Place()
-            p.set_place(t._place())
-            place = paddle.fluid.CUDAPlace(p.gpu_device_id())
-
-        t.set(ndarray, place)
-
-    parameter_list = list(filter(is_parameter, program.list_vars()))
-
-    if executor:
-        paddle.fluid.core._create_loaded_parameter(
-            parameter_list, global_scope(), executor._default_executor
-        )
-    with open(parameter_file_name, 'rb') as f:
-
-        # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
-        if sys.platform == 'darwin' and sys.version_info.major == 3:
-            load_dict = _pickle_loads_mac(parameter_file_name, f)
-        else:
-            load_dict = pickle.load(f, encoding='latin1')
-        load_dict = _pack_loaded_dict(load_dict)
-    for v in parameter_list:
-        assert (
-            v.name in load_dict
-        ), "Can not find [{}] in model file [{}]".format(
-            v.name, parameter_file_name
-        )
-        set_var(v, load_dict[v.name])
-
-    optimizer_var_list = list(
-        filter(is_belong_to_optimizer, program.list_vars())
-    )
-
-    if len(optimizer_var_list) > 0:
-        opt_file_name = model_prefix + ".pdopt"
-        assert os.path.exists(
-            opt_file_name
-        ), "Optimizer file [{}] not exits".format(opt_file_name)
-
-        if executor:
-            paddle.fluid.core._create_loaded_parameter(
-                optimizer_var_list, global_scope(), executor._default_executor
-            )
-
-        with open(opt_file_name, 'rb') as f:
-            load_dict = pickle.load(f, encoding='latin1')
-        for v in optimizer_var_list:
-            assert (
-                v.name in load_dict
-            ), "Can not find [{}] in model file [{}]".format(
-                v.name, opt_file_name
-            )
-            set_var(v, load_dict[v.name])
-
-
-def load_program_state(model_path, var_list=None):
-    """
-
-    Load program state from local file
-
-    Args:
-        model_path(str): The file prefix store the program
-        var_list(list|tuple, optional): The Tensor list/tuple to load saved with
-                                  [ save_params, save_persistables, save_vars ].
-                                  Default: None.
-                                  The var_list is only used to get name,
-                                  will not be modified.
-    Returns:
-        state_dict(dict): the dict store Parameter and optimizer information
-
-    Examples:
-
-        .. code-block:: python
-
-            import paddle
-            import paddle.static as static
-
-            paddle.enable_static()
-
-            x = static.data(name="x", shape=[10, 10], dtype='float32')
-            y = static.nn.fc(x, 10)
-            z = static.nn.fc(y, 10)
-
-            place = paddle.CPUPlace()
-            exe = static.Executor(place)
-            exe.run(static.default_startup_program())
-            prog = static.default_main_program()
-
-            static.save(prog, "./temp")
-            program_state = static.load_program_state("./temp")
-    """
-    model_prefix = model_path
-    if model_prefix.endswith(".pdparams"):
-        model_prefix = model_prefix[:-9]
-    elif model_prefix.endswith(".pdopt"):
-        model_prefix = model_prefix[:-6]
-    elif model_prefix.endswith(".pdmodel"):
-        model_prefix = model_prefix[:-8]
-
-    parameter_file_name = model_prefix + ".pdparams"
-    if not os.path.exists(parameter_file_name):
-        # model file saved with fluid.save is not found, try to load model file saved with
-        # [save_vars, save_params, save_persistables]
-        _logger.debug(
-            "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format(
-                parameter_file_name
-            )
-        )
-
-        var_name_list = []
-        if var_list is None and os.path.isfile(model_path):
-            raise ValueError(
-                "var_list can not be None when model_path is a file type"
-            )
-
-        for root, dirs, files in os.walk(model_path, topdown=False):
-            for f in files:
-                file_path = os.path.join(root, f)
-                var_temp_name = os.path.relpath(file_path, model_path)
-                var_temp_name = var_temp_name.replace("\\", "/")
-                var_name_list.append(var_temp_name)
-
-        with _load_program_scope():
-            load_prog = Program()
-            load_block = load_prog.global_block()
-
-            def clone_var_to_block(block, var):
-                if not isinstance(var, Variable):
-                    raise TypeError("value in var_list must be variable")
-                return block.create_var(
-                    name=var.name,
-                    shape=var.shape,
-                    dtype=var.dtype,
-                    type=var.type,
-                    lod_level=var.lod_level
-                    if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR
-                    else None,
-                    persistable=True,
-                )
-
-            def _load_vars_with_try_catch(
-                exe, dirname, vars, filename, raise_error=True
-            ):
-                try:
-                    load_vars(
-                        executor=exe,
-                        dirname=dirname,
-                        vars=vars,
-                        filename=filename,
-                    )
-                    return True
-                except:
-                    error_str = (
-                        "Failed to load model/variables `%s`, please make sure "
-                        "model/variables file is saved with the following APIs: "
-                        "save_params, save_persistables, save_vars."
-                    )
-                    filenames = (
-                        [var.name for var in vars]
-                        if filename is None
-                        else filename
-                    )
-                    if raise_error:
-                        raise RuntimeError(error_str % filenames)
-                    else:
-                        warnings.warn(error_str % filenames, RuntimeWarning)
-                return False
-
-            place = paddle.fluid.CPUPlace()
-            exe = paddle.fluid.Executor(place)
-
-            loaded_var_list = []
-
-            if os.path.isfile(model_path):
-                # when model_path is file, var_list cannot be None
-                dir_name, file_name = os.path.split(model_path)
-                for var in var_list:
-                    loaded_var_list.append(clone_var_to_block(load_block, var))
-                _load_vars_with_try_catch(
-                    exe, dir_name, loaded_var_list, file_name
-                )
-            else:
-                # var_list can be None or not None
-                if var_list is not None:
-                    for var in var_list:
-                        loaded_var_list.append(
-                            clone_var_to_block(load_block, var)
-                        )
-                    _load_vars_with_try_catch(
-                        exe, model_path, loaded_var_list, None
-                    )
-                else:
-                    for var_name in var_name_list:
-                        # NOTE(chenweihang): If identify which files the user wants
-                        # to load from the disk, we load these variables one by one.
-                        # If a file does not exist, we only warn the user that the
-                        # file may be an irrelevant file, but does not throw an error
-                        # to ensure that other legal variables can be loaded.
-                        temp_var = load_block.create_var(
-                            name=var_name, persistable=True
-                        )
-                        if _load_vars_with_try_catch(
-                            exe, model_path, [temp_var], None, False
-                        ):
-                            loaded_var_list.append(temp_var)
-
-            res_dict = {}
-            for var in loaded_var_list:
-                res_dict[var.name] = np.asarray(
-                    paddle.fluid.global_scope().find_var(var.name).get_tensor()
-                )
-
-            return res_dict
-
-    assert os.path.exists(
-        parameter_file_name
-    ), "Parameter file [{}] not exits".format(parameter_file_name)
-
-    with open(parameter_file_name, 'rb') as f:
-        # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
-        if sys.platform == 'darwin' and sys.version_info.major == 3:
-            para_dict = _pickle_loads_mac(parameter_file_name, f)
-        else:
-            para_dict = pickle.load(f, encoding='latin1')
-    para_dict = _pack_loaded_dict(para_dict)
-
-    opt_file_name = model_prefix + ".pdopt"
-    if os.path.exists(opt_file_name):
-        with open(opt_file_name, 'rb') as f:
-            opti_dict = pickle.load(f, encoding='latin1')
-
-        para_dict.update(opti_dict)
-
-    return para_dict
-
-
-@static_only
-def set_program_state(program, state_dict):
-    """
-    Set program parameter from state_dict
-
-    An exception will throw if shape or dtype of the parameters is not match.
-
-    NOTICE: This function MUST called after run start_up_program
-
-    Args:
-        program(Program): The program to be set
-        state_dict(dict): the dict store Parameter and optimizer information
-    Returns:
-        None
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.static as static
-
-            paddle.enable_static()
-
-            x = static.data(name="x", shape=[10, 10], dtype='float32')
-            y = static.nn.fc(x, 10)
-            z = static.nn.fc(y, 10)
-
-            place = paddle.CPUPlace()
-            exe = static.Executor(place)
-            exe.run(static.default_startup_program())
-            prog = static.default_main_program()
-
-            static.save(prog, "./temp")
-            program_state = static.load_program_state("./temp")
-
-            static.set_program_state(prog, program_state)
-    """
-    state_dict = _pack_loaded_dict(state_dict)
-    parameter_list = list(filter(is_persistable, program.list_vars()))
-
-    used_para_list = {}
-    for para in parameter_list:
-        var_temp = paddle.fluid.global_scope().find_var(para.name)
-        assert (
-            var_temp is not None
-        ), "Variable [ {} ] Not found, Please make sure run startup program".format(
-            para.name
-        )
-        if para.name in state_dict:
-            # set value from state dict
-            orig_para_np = np.array(var_temp.get_tensor())
-            new_para_np = state_dict[para.name]
-            assert orig_para_np.shape == new_para_np.shape, (
-                "Parameter's shape does not match, the Program requires a parameter with the shape of ({}), "
-                "while the loaded parameter (namely [ {} ]) has a shape of  ({}).".format(
-                    orig_para_np.shape, para.name, new_para_np.shape
-                )
-            )
-            assert orig_para_np.dtype == new_para_np.dtype, (
-                "Parameter's data type does not match, the Program requires a parameter with a dtype of ({}), "
-                "while the loaded parameter (namely [ {} ]) has a dtype of  ({}).".format(
-                    orig_para_np.dtype, para.name, new_para_np.dtype
-                )
-            )
-
-            ten = var_temp.get_tensor()
-            ten_place = ten._place()
-
-            # assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \
-            #    "Place not support, only support CPUPlace and GPUPlace, now is {}".format(str(ten_place))
-            py_place = paddle.fluid.CPUPlace()
-            if ten_place.is_cuda_pinned_place():
-                place = paddle.fluid.CUDAPinnedPlace()
-            elif ten_place.is_gpu_place():
-                p = paddle.fluid.core.Place()
-                p.set_place(ten_place)
-                py_place = paddle.fluid.CUDAPlace(p.gpu_device_id())
-            elif ten_place.is_xpu_place():
-                p = paddle.fluid.core.Place()
-                p.set_place(ten_place)
-                py_place = paddle.fluid.XPUPlace(p.xpu_device_id())
-            elif ten_place.is_npu_place():
-                p = paddle.fluid.core.Place()
-                p.set_place(ten_place)
-                py_place = paddle.fluid.NPUPlace(p.npu_device_id())
-            elif ten_place.is_mlu_place():
-                p = paddle.fluid.core.Place()
-                p.set_place(ten_place)
-                py_place = paddle.fluid.MLUPlace(p.mlu_device_id())
-
-            ten.set(new_para_np, py_place)
-
-            used_para_list[para.name] = 1
-
-    unused_para_list = []
-    for k, v in state_dict.items():
-        if k not in used_para_list:
-            unused_para_list.append(k)
-    if len(unused_para_list) > 0:
-        warnings.warn(
-            "This list is not set, Because of Paramerter not found in program. There are: {}".format(
-                " ".join(unused_para_list)
-            )
-        )

python/paddle/fluid/tests/unittests/dist_save_load.py

@@ -22,7 +22,7 @@ from test_dist_base import RUN_STEP, runtime_main
 
 import paddle
 import paddle.fluid as fluid
-from paddle.fluid import core, io
+from paddle.fluid import core
 
 
 class TestDistSaveLoad2x2(TestDistSimnetBow2x2):
@@ -56,7 +56,7 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2):
 
             return var.persistable
 
-        io.load_vars(
+        paddle.static.io.load_vars(
             executor,
             dirname=dirname,
             main_program=program,
@@ -89,7 +89,9 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2):
         exe.run(startup_prog)
 
         if need_load and model_dir:
-            fluid.io.load_persistables(exe, model_dir, pserver_prog)
+            paddle.distributed.io.load_persistables(
+                exe, model_dir, pserver_prog
+            )
 
         exe.run(pserver_prog)
 

python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py

@@ -172,7 +172,7 @@ def train_static(args, batch_generator):
                     model_path = os.path.join(
                         args.save_static_model_path, "transformer"
                     )
-                    fluid.save(train_prog, model_path)
+                    paddle.static.save(train_prog, model_path)
                 break
     return np.array(avg_loss)
 

python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py

@@ -20,7 +20,6 @@ import numpy as np
 
 import paddle
 import paddle.dataset.wmt16 as wmt16
-import paddle.fluid as fluid
 
 
 def get_input_descs(args, mode="train"):
@@ -310,7 +309,7 @@ def load(program, model_path, executor=None, var_list=None):
     To load python2 saved models in python3.
     """
     try:
-        fluid.load(program, model_path, executor, var_list)
+        paddle.static.load(program, model_path, executor, var_list)
     except UnicodeDecodeError:
         warnings.warn(
             "An UnicodeDecodeError is catched, which might be caused by loading "
@@ -319,7 +318,7 @@ def load(program, model_path, executor=None, var_list=None):
         )
         load_bak = pickle.load
         pickle.load = partial(load_bak, encoding="latin1")
-        fluid.load(program, model_path, executor, var_list)
+        paddle.static.load(program, model_path, executor, var_list)
         pickle.load = load_bak
 
 

python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py

@@ -42,7 +42,7 @@ def main_test_func(place, dtype):
             adam_optimizer.minimize(avg_cost)
 
             fetch_list = [avg_cost]
-            train_reader = fluid.io.batch(
+            train_reader = paddle.batch(
                 paddle.dataset.uci_housing.train(), batch_size=1
             )
             feeder = fluid.DataFeeder(place=place, feed_list=[x, y])

python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py

@@ -40,7 +40,7 @@ class TestClass(unittest.TestCase):
                 yield img, label
 
         reader = paddle.reader.cache(fake_reader)
-        batch_reader = fluid.io.batch(reader, batch_size=batch_size)
+        batch_reader = paddle.batch(reader, batch_size=batch_size)
 
         places = [fluid.CPUPlace()]
         if fluid.core.is_compiled_with_cuda():

python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py

@@ -264,7 +264,7 @@ class TestDistMnistAsync2x2WithGauss(TestFleetBase):
         feeder = fluid.DataFeeder(place=fluid.CPUPlace(), feed_list=datas)
         exe.run(fluid.default_startup_program())
 
-        fluid.io.load_persistables(exe, model_file)
+        paddle.distributed.io.load_persistables(exe, model_file)
 
         for batch_id, data in enumerate(reader()):
             score = exe.run(

python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py

@@ -150,12 +150,12 @@ class TestDygraphLoadStatic(unittest.TestCase):
         )
         out = exe.run(framework.default_startup_program())
 
-        fluid.save(
+        paddle.static.save(
             framework.default_main_program(),
             os.path.join(temp_dir.name, "test_1"),
         )
 
-        para_dict = fluid.load_program_state(
+        para_dict = paddle.static.load_program_state(
             os.path.join(temp_dir.name, "test_1")
         )
 

python/paddle/fluid/tests/unittests/test_inference_model_io.py

@@ -25,14 +25,13 @@ import paddle.fluid as fluid
 import paddle.fluid.core as core
 import paddle.fluid.executor as executor
 import paddle.fluid.optimizer as optimizer
-from paddle.distributed.io import load_inference_model_distributed
-from paddle.fluid.compiler import CompiledProgram
-from paddle.fluid.framework import Program, program_guard
-from paddle.fluid.io import (
-    load_inference_model,
-    save_inference_model,
+from paddle.distributed.io import (
+    load_inference_model_distributed,
     save_persistables,
 )
+from paddle.fluid.compiler import CompiledProgram
+from paddle.fluid.framework import Program, program_guard
+from paddle.fluid.io import load_inference_model, save_inference_model
 
 paddle.enable_static()
 

python/paddle/fluid/tests/unittests/test_io_save_load.py

@@ -18,6 +18,7 @@ import unittest
 
 import paddle
 import paddle.fluid as fluid
+import paddle.static as static
 from paddle.fluid import core
 
 
@@ -34,24 +35,24 @@ class TestSaveLoadAPIError(unittest.TestCase):
         graph = core.Graph(core.ProgramDesc())
         compiled_program = fluid.CompiledProgram(graph)
         with self.assertRaises(TypeError):
-            fluid.io._get_valid_program(compiled_program)
+            paddle.static.io._get_valid_program(compiled_program)
 
         # case 2: main_program type error
         with self.assertRaises(TypeError):
-            fluid.io._get_valid_program("program")
+            paddle.static.io._get_valid_program("program")
 
     def test_load_vars_error(self):
         place = fluid.CPUPlace()
         exe = fluid.Executor(place)
         # case 1: main_program type error when vars None
         with self.assertRaises(TypeError):
-            fluid.io.load_vars(
+            static.io.load_vars(
                 executor=exe, dirname=self.save_dir, main_program="program"
             )
 
         # case 2: main_program type error when vars not None
         with self.assertRaises(TypeError):
-            fluid.io.load_vars(
+            static.io.load_vars(
                 executor=exe,
                 dirname=self.save_dir,
                 main_program="program",

python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py

@@ -75,7 +75,7 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
     def tearDown(self):
         self.temp_dir.cleanup()
 
-    def train_and_save_model(self, only_params=False):
+    def train_and_save_model(self):
         with new_program_scope():
             startup_program = fluid.default_startup_program()
             main_program = fluid.default_main_program()
@@ -122,19 +122,14 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
                     param.name
                 )
 
-            if only_params:
-                fluid.io.save_params(
-                    exe, self.save_dirname, filename=self.params_filename
-                )
-            else:
-                fluid.io.save_inference_model(
-                    self.save_dirname,
-                    ["img"],
-                    [prediction],
-                    exe,
-                    model_filename=self.model_filename,
-                    params_filename=self.params_filename,
-                )
+            fluid.io.save_inference_model(
+                self.save_dirname,
+                ["img"],
+                [prediction],
+                exe,
+                model_filename=self.model_filename,
+                params_filename=self.params_filename,
+            )
 
         return static_param_dict
 
@@ -195,16 +190,6 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
         )
         self.check_load_state_dict(orig_param_dict, new_load_param_dict)
 
-    def test_load_state_dict_from_save_params(self):
-        self.save_dirname = os.path.join(
-            self.temp_dir.name, "static_mnist.load_state_dict.save_params"
-        )
-        self.params_filename = None
-        orig_param_dict = self.train_and_save_model(True)
-
-        new_load_param_dict = paddle.load(self.save_dirname)
-        self.check_load_state_dict(orig_param_dict, new_load_param_dict)
-
 
 if __name__ == '__main__':
     unittest.main()

python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py

-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import shutil
-import unittest
-
-import paddle
-import paddle.fluid as fluid
-from paddle.fluid.executor import Executor
-
-
-class TestLoadVarsShapeCheck(unittest.TestCase):
-    def setUp(self):
-        self.model_path = "./model_temp/"
-
-    def test_shape_check_save_load(self):
-        program_1 = fluid.Program()
-        startup_program_1 = fluid.Program()
-
-        with fluid.program_guard(program_1, startup_program_1):
-            input = paddle.static.data(
-                name="x", shape=[-1, 10], dtype='float32'
-            )
-            out = paddle.static.nn.fc(input, 20)
-        place = fluid.CPUPlace()
-        exe = Executor(place)
-        exe.run(startup_program_1)
-
-        fluid.io.save_params(exe, self.model_path, main_program=program_1)
-        fluid.io.load_params(exe, self.model_path, main_program=program_1)
-
-    def tearDown(self):
-        if os.path.exists(self.model_path):
-            shutil.rmtree(self.model_path)
-
-
-if __name__ == "__main__":
-    unittest.main()

python/paddle/fluid/tests/unittests/test_paddle_save_load.py

@@ -235,7 +235,7 @@ class TestSaveLoadAny(unittest.TestCase):
             # paddle.save, legacy paddle.fluid.load
             self.replace_static_save(prog, path)
             self.set_zero(prog, place)
-            paddle.fluid.io.load(prog, path)
+            paddle.static.load(prog, path)
             for var in prog.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
                     new_t = np.array(
@@ -244,7 +244,7 @@ class TestSaveLoadAny(unittest.TestCase):
                     base_t = base_map[var.name]
                     np.testing.assert_array_equal(new_t, np.array(base_t))
             # legacy paddle.fluid.save, paddle.load
-            paddle.fluid.io.save(prog, path)
+            paddle.static.save(prog, path)
             self.set_zero(prog, place)
             self.replace_static_load(prog, path)
             for var in prog.list_vars():
@@ -904,7 +904,7 @@ class TestSaveLoadToMemory(unittest.TestCase):
         with self.assertRaises(ValueError):
             paddle.save(state_dict, '')
         with self.assertRaises(ValueError):
-            paddle.fluid.io._open_file_buffer('temp', 'b')
+            paddle.framework.io_utils._open_file_buffer('temp', 'b')
 
     def test_static_save_to_memory(self):
         paddle.enable_static()

python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py

@@ -107,7 +107,7 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
             var_list = list(
                 filter(lambda var: var.persistable, prog.list_vars())
             )
-            fluid.io.load_vars(
+            paddle.static.io.load_vars(
                 exe, path_vars1, main_program=prog, vars=var_list
             )
 
@@ -123,7 +123,7 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
             path_vars2 = os.path.join(
                 self.temp_dir.name, 'test_replace_save_load_vars_binary2/model/'
             )
-            fluid.io.save_vars(
+            paddle.static.io.save_vars(
                 exe, path_vars2, main_program=prog, vars=var_list
             )
             self.set_zero(prog, place)

python/paddle/fluid/tests/unittests/test_parameter.py

@@ -19,7 +19,6 @@ import numpy as np
 
 import paddle
 import paddle.fluid.core as core
-import paddle.fluid.io as io
 from paddle.fluid.dygraph import guard
 from paddle.fluid.executor import Executor
 from paddle.fluid.framework import ParamBase, Variable, default_main_program
@@ -47,8 +46,6 @@ class ParameterChecks(unittest.TestCase):
         exe = Executor(paddle.CPUPlace())
         p = exe.run(main_program, fetch_list=[param])[0]
         np.testing.assert_array_equal(p, np.ones(shape) * val)
-        p = io.get_parameter_value_by_name('fc.w', exe, main_program)
-        np.testing.assert_array_equal(p, np.ones(shape) * val)
 
         zero_dim_param = b.create_parameter(name='x', shape=[], dtype='float32')
         self.assertEqual(zero_dim_param.shape, ())

python/paddle/fluid/tests/unittests/test_static_save_load.py

@@ -346,7 +346,9 @@ class TestSaveLoadBase(unittest.TestCase):
                     self.assertTrue(np.sum(np.abs(t)) != 0)
                     base_map[var.name] = t
 
-            fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
+            paddle.static.save(
+                main_program, os.path.join(temp_dir.name, "test_1")
+            )
 
             # set var to zero
             for var in main_program.list_vars():
@@ -360,7 +362,7 @@ class TestSaveLoadBase(unittest.TestCase):
                     # make sure all the paramerter or optimizer var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.load(
+            paddle.static.load(
                 main_program,
                 os.path.join(temp_dir.name, "test_1.pdparams"),
                 exe,
@@ -485,7 +487,9 @@ class TestSaveLoadPartial(unittest.TestCase):
                     self.assertTrue(np.sum(np.abs(t)) != 0)
                     base_map[var.name] = t
 
-            fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
+            paddle.static.save(
+                main_program, os.path.join(temp_dir.name, "test_1")
+            )
 
             # set var to zero
             for var in main_program.list_vars():
@@ -499,7 +503,7 @@ class TestSaveLoadPartial(unittest.TestCase):
                     # make sure all the paramerter or optimizer var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.load(
+            paddle.static.load(
                 test_program, os.path.join(temp_dir.name, "test_1.pdopt"), None
             )
 
@@ -510,7 +514,7 @@ class TestSaveLoadPartial(unittest.TestCase):
                     )
                     base_t = base_map[var.name]
                     np.testing.assert_array_equal(new_t, base_t)
-            fluid.load(
+            paddle.static.load(
                 test_program,
                 os.path.join(temp_dir.name, "test_1.pdmodel"),
                 None,
@@ -617,7 +621,9 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
                     self.assertTrue(np.sum(np.abs(t)) != 0)
                     base_map[var.name] = t
 
-            fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
+            paddle.static.save(
+                main_program, os.path.join(temp_dir.name, "test_1")
+            )
 
             # set var to zero
             for var in main_program.list_vars():
@@ -631,7 +637,9 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
                     # make sure all the paramerter or optimizer var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.load(main_program, os.path.join(temp_dir.name, "test_1"), exe)
+            paddle.static.load(
+                main_program, os.path.join(temp_dir.name, "test_1"), exe
+            )
 
             for var in main_program.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
@@ -752,7 +760,9 @@ class TestProgramStatePartial(unittest.TestCase):
                     self.assertTrue(np.sum(np.abs(t)) != 0)
                     base_map[var.name] = t
 
-            fluid.save(main_program, os.path.join(temp_dir.name, 'test_1'))
+            paddle.static.save(
+                main_program, os.path.join(temp_dir.name, 'test_1')
+            )
 
             # set var to zero
             for var in main_program.list_vars():
@@ -767,23 +777,23 @@ class TestProgramStatePartial(unittest.TestCase):
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
             # fluid.load(test_program, "./test_1", None )
-            program_state = fluid.load_program_state(
+            program_state = paddle.static.load_program_state(
                 os.path.join(temp_dir.name, 'test_1')
             )
 
-            program_state_1 = fluid.load_program_state(
+            program_state_1 = paddle.static.load_program_state(
                 os.path.join(temp_dir.name, 'test_1.pdparams')
             )
 
-            program_state_2 = fluid.load_program_state(
+            program_state_2 = paddle.static.load_program_state(
                 os.path.join(temp_dir.name, 'test_1.pdopt')
             )
 
-            program_state_3 = fluid.load_program_state(
+            program_state_3 = paddle.static.load_program_state(
                 os.path.join(temp_dir.name, 'test_1.pdmodel')
             )
 
-            fluid.set_program_state(test_program, program_state)
+            paddle.static.set_program_state(test_program, program_state)
 
             for var in test_program.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
@@ -805,7 +815,7 @@ class TestProgramStatePartial(unittest.TestCase):
                     # make sure all the paramerter or optimizer var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.set_program_state(test_program, program_state_1)
+            paddle.static.set_program_state(test_program, program_state_1)
 
             for var in test_program.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
@@ -827,7 +837,7 @@ class TestProgramStatePartial(unittest.TestCase):
                     # make sure all the paramerter or optimizer var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.set_program_state(test_program, program_state_2)
+            paddle.static.set_program_state(test_program, program_state_2)
 
             for var in test_program.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
@@ -849,7 +859,7 @@ class TestProgramStatePartial(unittest.TestCase):
                     # make sure all the paramerter or optimizer var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.set_program_state(test_program, program_state_3)
+            paddle.static.set_program_state(test_program, program_state_3)
 
             for var in test_program.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
@@ -880,7 +890,7 @@ class TestVariableInit(unittest.TestCase):
         exe.run(fluid.default_startup_program())
 
         temp_dir = tempfile.TemporaryDirectory()
-        fluid.save(
+        paddle.static.save(
             fluid.default_main_program(),
             os.path.join(temp_dir.name, "test_path"),
         )
@@ -905,7 +915,7 @@ class TestVariableInit(unittest.TestCase):
         place = self.set_place()
         exe = fluid.Executor(place)
         parameter_list = list(
-            filter(fluid.io.is_parameter, program.list_vars())
+            filter(paddle.framework.is_parameter, program.list_vars())
         )
 
         fluid.core._create_loaded_parameter(
@@ -925,7 +935,10 @@ class TestVariableInit(unittest.TestCase):
             set_var(new_v, load_dict[v.name])
 
         opt_list = list(
-            filter(fluid.io.is_belong_to_optimizer, program.list_vars())
+            filter(
+                paddle.framework.io_utils.is_belong_to_optimizer,
+                program.list_vars(),
+            )
         )
 
         fluid.core._create_loaded_parameter(
@@ -1092,7 +1105,7 @@ class TestLoadFromOldInterface(unittest.TestCase):
                     # make sure all the paramerter or optimizer var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.load(
+            paddle.static.load(
                 main_program, os.path.join(self.temp_dir.name, "test_path"), exe
             )
 
@@ -1112,7 +1125,7 @@ class TestLoadFromOldInterface(unittest.TestCase):
 
                     var.desc.set_shape(new_shape)
             with self.assertRaises(RuntimeError):
-                fluid.load(
+                paddle.static.load(
                     main_program,
                     os.path.join(self.temp_dir.name, "test_path"),
                     exe,
@@ -1120,7 +1133,7 @@ class TestLoadFromOldInterface(unittest.TestCase):
 
             # check unused parameter
 
-            fluid.load(
+            paddle.static.load(
                 test_clone_program,
                 os.path.join(self.temp_dir.name, "test_path"),
                 exe,
@@ -1239,7 +1252,7 @@ class TestLoadFromOldInterface(unittest.TestCase):
                     # make sure all the paramerter or optimizer var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.load(
+            paddle.static.load(
                 main_program,
                 os.path.join(self.temp_dir.name, "test_static_load_var_list"),
                 exe,
@@ -1377,11 +1390,11 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
             file_model_path = os.path.join(save_dir, "model_single")
-            fluid.load(
+            paddle.static.load(
                 main_program,
                 file_model_path,
                 exe,
-                fluid.io.get_program_persistable_vars(main_program),
+                paddle.static.io.get_program_persistable_vars(main_program),
             )
 
             for var in main_program.list_vars():
@@ -1403,36 +1416,33 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
                     var.desc.set_shape(new_shape)
 
             with self.assertRaises(RuntimeError):
-                fluid.load(
+                paddle.static.load(
                     main_program,
                     file_model_path,
                     exe,
-                    fluid.io.get_program_persistable_vars(main_program),
+                    paddle.static.io.get_program_persistable_vars(main_program),
                 )
 
-            fluid.io.save_params(
-                exe, "test_path", main_program, filename="model_single"
-            )
             with self.assertRaises(RuntimeError):
-                fluid.load(
+                paddle.static.load(
                     main_program,
                     file_model_path,
                     exe,
-                    fluid.io.get_program_persistable_vars(main_program),
+                    paddle.static.io.get_program_persistable_vars(main_program),
                 )
 
             # check when executor is None
             with self.assertRaises(ValueError):
-                fluid.load(
+                paddle.static.load(
                     main_program,
                     file_model_path,
                     None,
-                    fluid.io.get_program_persistable_vars(main_program),
+                    paddle.static.io.get_program_persistable_vars(main_program),
                 )
 
             # check when var list is None
             with self.assertRaises(ValueError):
-                fluid.load(main_program, file_model_path, exe, None)
+                paddle.static.load(main_program, file_model_path, exe, None)
 
             # check save params, load var_list = get_program_persistable_vars
             with self.assertRaises(RuntimeError):
@@ -1440,7 +1450,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
                     main_program.global_block(), shape=[1], name="test_temp_var"
                 )
                 all_var_list = list(main_program.list_vars())
-                fluid.load(
+                paddle.static.load(
                     main_program,
                     file_model_path,
                     exe,
@@ -1579,8 +1589,8 @@ class TestProgramStateOldSave(unittest.TestCase):
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
             # case 1: load basic
-            program_state = fluid.load_program_state(save_dir)
-            fluid.set_program_state(main_program, program_state)
+            program_state = paddle.static.load_program_state(save_dir)
+            paddle.static.set_program_state(main_program, program_state)
             self.check_in_static(main_program, base_map)
 
             # case 2: load with no need file
@@ -1594,21 +1604,21 @@ class TestProgramStateOldSave(unittest.TestCase):
                     else:
                         raise e
 
-            program_state = fluid.load_program_state(save_dir)
-            fluid.set_program_state(main_program, program_state)
+            program_state = paddle.static.load_program_state(save_dir)
+            paddle.static.set_program_state(main_program, program_state)
             self.check_in_static(main_program, base_map)
 
             # case 3: load with var_list
-            program_state = fluid.load_program_state(
+            program_state = paddle.static.load_program_state(
                 save_dir, main_program.all_parameters()
             )
-            fluid.set_program_state(main_program, program_state)
+            paddle.static.set_program_state(main_program, program_state)
             self.check_in_static(main_program, base_map)
 
         if self.test_dygraph:
             # make sure `load_program_state` can be used in dynamic graph mode
             with fluid.dygraph.guard(place):
-                load_state = fluid.load_program_state(save_dir)
+                load_state = paddle.static.load_program_state(save_dir)
                 for k, v in load_state.items():
                     np.testing.assert_array_equal(base_map[k], v)
 
@@ -1758,11 +1768,13 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
             # fluid.load(test_program, "./test_1", None )
-            program_state = fluid.load_program_state(
+            program_state = paddle.static.load_program_state(
                 os.path.join(save_dir, "model_1"),
-                var_list=fluid.io.get_program_persistable_vars(main_program),
+                var_list=paddle.static.io.get_program_persistable_vars(
+                    main_program
+                ),
             )
-            fluid.set_program_state(main_program, program_state)
+            paddle.static.set_program_state(main_program, program_state)
 
             for var in main_program.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
@@ -1773,15 +1785,17 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
                     np.testing.assert_array_equal(new_t, base_t)
 
             with self.assertRaises(ValueError):
-                fluid.load_program_state(os.path.join(save_dir, "model_1"))
+                paddle.static.load_program_state(
+                    os.path.join(save_dir, "model_1")
+                )
 
             with self.assertRaises(TypeError):
-                fluid.load_program_state(
+                paddle.static.load_program_state(
                     os.path.join(save_dir, "model_1"), var_list=["str"]
                 )
 
             with self.assertRaises(RuntimeError):
-                fluid.load_program_state(
+                paddle.static.load_program_state(
                     os.path.join(save_dir, "model_1"),
                     var_list=[
                         main_program.global_block().create_var(
@@ -1827,17 +1841,17 @@ class TestStaticSaveLoadPickle(unittest.TestCase):
             )
 
             with self.assertRaises(ValueError):
-                paddle.fluid.save(prog, path, 2.0)
+                paddle.static.save(prog, path, 2.0)
 
             with self.assertRaises(ValueError):
-                paddle.fluid.save(prog, path, 1)
+                paddle.static.save(prog, path, 1)
 
             with self.assertRaises(ValueError):
-                paddle.fluid.save(prog, path, 5)
+                paddle.static.save(prog, path, 5)
 
             protocols = [2, 3, 4]
             for protocol in protocols:
-                paddle.fluid.save(prog, path, protocol)
+                paddle.static.save(prog, path, protocol)
                 # set var to zero
                 for var in prog.list_vars():
                     if isinstance(var, framework.Parameter) or var.persistable:
@@ -1851,7 +1865,7 @@ class TestStaticSaveLoadPickle(unittest.TestCase):
                         )
                         self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-                paddle.fluid.load(prog, path)
+                paddle.static.load(prog, path)
 
                 for var in prog.list_vars():
                     if isinstance(var, framework.Parameter) or var.persistable:

python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py

@@ -134,7 +134,7 @@ class TestSaveLoadBF16(unittest.TestCase):
                     self.assertTrue(np.sum(np.abs(t)) != 0)
                     base_map[var.name] = t
             save_dir = os.path.join(self.temp_dir.name, "test_1")
-            fluid.save(main_program, save_dir)
+            paddle.static.save(main_program, save_dir)
 
             # set var to zero
             for var in main_program.list_vars():
@@ -148,7 +148,7 @@ class TestSaveLoadBF16(unittest.TestCase):
                     # make sure all the paramerter or optimizer var have been set to zero
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            fluid.load(
+            paddle.static.load(
                 main_program,
                 os.path.join(self.temp_dir.name, "test_1.pdparams"),
                 exe,

python/paddle/fluid/tests/unittests/test_static_save_load_large.py

@@ -58,7 +58,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
             )
             path = os.path.join(path, "static_save")
             protocol = 4
-            paddle.fluid.save(prog, path, pickle_protocol=protocol)
+            paddle.static.save(prog, path, pickle_protocol=protocol)
             # set var to zero
             for var in prog.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
@@ -70,7 +70,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
                     )
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            paddle.fluid.load(prog, path)
+            paddle.static.load(prog, path)
 
             for var in prog.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
@@ -91,8 +91,8 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
                     )
                     self.assertTrue(np.sum(np.abs(new_t)) == 0)
 
-            program_state = fluid.load_program_state(path)
-            fluid.set_program_state(prog, program_state)
+            program_state = paddle.static.load_program_state(path)
+            paddle.static.set_program_state(prog, program_state)
             for var in prog.list_vars():
                 if isinstance(var, framework.Parameter) or var.persistable:
                     new_t = np.array(

python/paddle/framework/__init__.py

@@ -36,6 +36,16 @@ from ..fluid.dygraph.base import grad  # noqa: F401
 from .io import save  # noqa: F401
 from .io import load  # noqa: F401
 
+from .io_utils import _open_file_buffer  # noqa: F401
+from .io_utils import is_parameter  # noqa: F401
+from .io_utils import is_persistable  # noqa: F401
+from .io_utils import is_belong_to_optimizer  # noqa: F401
+from .io_utils import _clone_var_in_block_  # noqa: F401
+from .io_utils import _pickle_loads_mac
+from .io_utils import _pack_loaded_dict
+from .io_utils import _unpack_saved_dict
+from .io_utils import _load_program_scope
+
 from ..fluid import monkey_patch_variable
 from ..fluid.dygraph import monkey_patch_math_varbase
 from ..fluid.framework import disable_signal_handler  # noqa: F401

python/paddle/framework/io.py

@@ -37,14 +37,6 @@ from paddle.fluid.framework import (
     _non_static_mode,
     _varbase_creator,
 )
-from paddle.fluid.io import _is_file_path, _is_memory_buffer
-from paddle.fluid.io import _legacy_save as _legacy_static_save
-from paddle.fluid.io import (
-    _open_file_buffer,
-    _pack_loaded_dict,
-    _pickle_loads_mac,
-    _unpack_saved_dict,
-)
 from paddle.jit.api import _SaveLoadConfig
 from paddle.jit.translated_layer import (
     INFER_MODEL_SUFFIX,
@@ -53,6 +45,16 @@ from paddle.jit.translated_layer import (
     _construct_program_holders,
 )
 
+from .io_utils import (
+    _is_file_path,
+    _is_memory_buffer,
+    _legacy_static_save,
+    _open_file_buffer,
+    _pack_loaded_dict,
+    _pickle_loads_mac,
+    _unpack_saved_dict,
+)
+
 __all__ = []
 
 

python/paddle/framework/io_utils.py

+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import math
+import os
+import pickle
+import sys
+from io import BytesIO
+
+import numpy as np
+
+import paddle
+from paddle.fluid import core
+from paddle.fluid.framework import Parameter, Variable, static_only
+from paddle.fluid.log_helper import get_logger
+from paddle.fluid.wrapped_decorator import signature_safe_contextmanager
+
+_logger = get_logger(
+    __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
+)
+
+# This file contains various utility functions that are used in static.io(io related api that used in static graph)
+# and framework.io(io related api that used in dygraph)
+
+
+class _open_buffer:
+    def __init__(self, buffer):
+        self.buffer = buffer
+
+    def __enter__(self):
+        return self.buffer
+
+
+class _buffer_reader(_open_buffer):
+    def __init__(self, buffer):
+        super().__init__(buffer)
+        self.initial_tell = self.buffer.tell()
+
+    def __exit__(self, *args):
+        # `args[0]` is type of exception. When the `read` is abnormal, the file pointer returns to the initial position.
+        if args[0] is not None:
+            self.buffer.seek(self.initial_tell)
+
+
+class _buffer_writer(_open_buffer):
+    def __exit__(self, *args):
+        self.buffer.flush()
+
+
+def _is_file_path(path):
+    return isinstance(path, str)
+
+
+def _open_file_buffer(path_or_buffer, mode):
+
+    if _is_file_path(path_or_buffer):
+        return open(path_or_buffer, mode)
+    else:
+        if 'w' in mode:
+            return _buffer_writer(path_or_buffer)
+        elif 'r' in mode:
+            return _buffer_reader(path_or_buffer)
+        else:
+            raise ValueError(
+                "Expected 'r' or 'w' in mode but got {}".format(mode)
+            )
+
+
+def _is_memory_buffer(buffer):
+    return isinstance(buffer, BytesIO)
+
+
+def is_persistable(var):
+    """
+
+    Check whether the given variable is persistable.
+
+    Args:
+        var(Variable): The variable to be checked.
+
+    Returns:
+        bool: True if the given `var` is persistable
+        False if not.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.fluid as fluid
+
+            paddle.enable_static()
+            param = fluid.default_main_program().global_block().var('fc.b')
+            res = fluid.io.is_persistable(param)
+    """
+    if (
+        var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH
+        or var.desc.type() == core.VarDesc.VarType.FETCH_LIST
+        or var.desc.type() == core.VarDesc.VarType.READER
+    ):
+        return False
+    return var.persistable
+
+
+def is_parameter(var):
+    """
+    Check whether the given variable is an instance of Parameter.
+
+    Args:
+        var(Variable): The variable to be checked.
+
+    Returns:
+        bool: True if the given `var` is an instance of Parameter,
+        False if not.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.fluid as fluid
+
+            paddle.enable_static()
+            param = fluid.default_main_program().global_block().var('fc.w')
+            res = fluid.io.is_parameter(param)
+    """
+    return isinstance(var, Parameter)
+
+
+def is_belong_to_optimizer(var):
+    if not (isinstance(var, Parameter) or var.desc.need_check_feed()):
+        return is_persistable(var)
+
+    return False
+
+
+def _clone_var_in_block_(block, var):
+    assert isinstance(var, Variable)
+    if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR:
+        return block.create_var(
+            name=var.name,
+            shape=var.shape,
+            dtype=var.dtype,
+            type=var.type,
+            lod_level=var.lod_level,
+            persistable=True,
+        )
+    else:
+        return block.create_var(
+            name=var.name,
+            shape=var.shape,
+            dtype=var.dtype,
+            type=var.type,
+            persistable=True,
+        )
+
+
+@signature_safe_contextmanager
+def _load_program_scope(main=None, startup=None, scope=None):
+    prog = main if main else paddle.fluid.Program()
+    startup_prog = startup if startup else paddle.fluid.Program()
+    scope = scope if scope else paddle.fluid.core.Scope()
+    with paddle.fluid.scope_guard(scope):
+        with paddle.fluid.program_guard(prog, startup_prog):
+            with paddle.fluid.unique_name.guard():
+                with paddle.fluid.framework._dygraph_guard(None):
+                    yield
+
+
+@static_only
+def _legacy_static_save(param_dict, model_path, protocol=2):
+    def get_tensor(var):
+        if isinstance(var, (core.VarBase, core.eager.Tensor)):
+            return var.numpy()
+        elif isinstance(var, core.LoDTensor):
+            return np.array(var)
+        return var
+
+    param_dict = {name: get_tensor(param_dict[name]) for name in param_dict}
+
+    # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
+    if (
+        _is_file_path(model_path)
+        and sys.platform == 'darwin'
+        and sys.version_info.major == 3
+    ):
+        pickle_bytes = pickle.dumps(param_dict, protocol=protocol)
+        with open(model_path, 'wb') as f:
+            max_bytes = 2**30
+            for i in range(0, len(pickle_bytes), max_bytes):
+                f.write(pickle_bytes[i : i + max_bytes])
+    else:
+        with _open_file_buffer(model_path, 'wb') as f:
+            pickle.dump(param_dict, f, protocol=protocol)
+
+
+def _pickle_loads_mac(path, f):
+    pickle_bytes = bytearray(0)
+    file_size = os.path.getsize(path)
+    max_bytes = 2**30
+    for _ in range(0, file_size, max_bytes):
+        pickle_bytes += f.read(max_bytes)
+    load_result = pickle.loads(pickle_bytes, encoding='latin1')
+    return load_result
+
+
+def _pack_loaded_dict(load_obj):
+    if isinstance(load_obj, dict):
+        unpack_info = 'UnpackBigParamInfor@@'
+        if unpack_info in load_obj:
+            removes = []
+            for key, value in load_obj[unpack_info].items():
+                slices = [load_obj[part] for part in value["slices"]]
+                load_obj[key] = np.concatenate(slices).reshape(
+                    value["OriginShape"]
+                )
+                removes += value["slices"]
+            for key in removes:
+                load_obj.pop(key)
+            load_obj.pop(unpack_info)
+
+    return load_obj
+
+
+def _unpack_saved_dict(saved_obj, protocol):
+    temp_saved_obj = {}
+    unpack_infor = {}
+    # When pickle protocol=2 or protocol=3 the serialized object cannot be larger than 4G.
+    if 1 < protocol < 4:
+        if isinstance(saved_obj, dict):
+            for key, value in saved_obj.items():
+                if isinstance(value, np.ndarray):
+                    MAX_NUMBER_OF_ELEMENT = int(
+                        (2**30 - 1) / value.dtype.itemsize
+                    )
+                    num_element = np.prod(value.shape)
+                    if num_element > MAX_NUMBER_OF_ELEMENT:
+                        unpack_infor[key] = {}
+                        unpack_infor[key]["OriginShape"] = value.shape
+                        unpack_infor[key]["slices"] = []
+                        value = value.flatten()
+                        for i in range(
+                            int(
+                                math.ceil(
+                                    num_element * 1.0 / MAX_NUMBER_OF_ELEMENT
+                                )
+                            )
+                        ):
+                            part_name = key + "@@." + str(i)
+                            unpack_infor[key]["slices"].append(part_name)
+                            temp_saved_obj[part_name] = value[
+                                i
+                                * MAX_NUMBER_OF_ELEMENT : MAX_NUMBER_OF_ELEMENT
+                                * (i + 1)
+                            ]
+
+    if unpack_infor:
+        for key, value in unpack_infor.items():
+            if key in saved_obj:
+                saved_obj.pop(key)
+                for part in value['slices']:
+                    saved_obj[part] = temp_saved_obj[part]
+        saved_obj['UnpackBigParamInfor@@'] = unpack_infor
+    return saved_obj

python/paddle/hapi/model.py

@@ -34,9 +34,9 @@ from paddle.fluid.executor import global_scope
 from paddle.fluid.framework import Variable
 from paddle.fluid.framework import _current_expected_place as _get_device
 from paddle.fluid.framework import _get_paddle_place, _non_static_mode
-from paddle.fluid.io import is_belong_to_optimizer
 from paddle.fluid.layers import collective
 from paddle.fluid.layers.utils import flatten
+from paddle.framework.io_utils import is_belong_to_optimizer
 from paddle.io import DataLoader, Dataset, DistributedBatchSampler
 from paddle.jit.translated_layer import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX
 from paddle.metric import Metric

python/paddle/incubate/distributed/fleet/fleet_util.py

@@ -23,6 +23,7 @@ import time
 
 import numpy as np
 
+import paddle
 import paddle.fluid as fluid
 from paddle.distributed.fleet.utils.fs import HDFSClient
 from paddle.fluid.log_helper import get_logger
@@ -1087,11 +1088,13 @@ class FleetUtil:
             vars = [program.global_block().var(i) for i in var_names]
             with fluid.scope_guard(scope):
                 if save_combine:
-                    fluid.io.save_vars(
+                    paddle.static.io.save_vars(
                         executor, "./", program, vars=vars, filename=model_name
                     )
                 else:
-                    fluid.io.save_vars(executor, model_name, program, vars=vars)
+                    paddle.static.io.save_vars(
+                        executor, model_name, program, vars=vars
+                    )
 
             configs = {
                 "fs.default.name": hadoop_fs_name,

python/paddle/incubate/distributed/fleet/utils.py

@@ -22,6 +22,7 @@ from google.protobuf import text_format
 
 import paddle
 import paddle.fluid as fluid
+import paddle.framework.io_utils as io_utils
 from paddle.fluid import core, debugger
 from paddle.fluid.framework import Program
 from paddle.fluid.proto import framework_pb2
@@ -92,7 +93,7 @@ def check_pruned_program_vars(train_prog, pruned_prog):
     pruned_vars = [
         (v.name, v)
         for v in pruned_prog.list_vars()
-        if fluid.io.is_persistable(v)
+        if io_utils.is_persistable(v)
     ]
     pruned_vars = OrderedDict(pruned_vars)
     pruned_vars_name = [name for name in pruned_vars]
@@ -451,7 +452,7 @@ def check_saved_vars_try_dump(
         os.path.join(dump_dir, dump_prog_fn), is_text_dump_program
     )
     saved_params = [
-        v for v in dump_prog.list_vars() if fluid.io.is_persistable(v)
+        v for v in dump_prog.list_vars() if io_utils.is_persistable(v)
     ]
     logger.info(
         "persistable vars in dump program: {}".format(
@@ -477,7 +478,7 @@ def parse_program(program, output_dir):
     # persistable vars
     output = {}
     persistable_vars = [
-        v for v in program.list_vars() if fluid.io.is_persistable(v)
+        v for v in program.list_vars() if io_utils.is_persistable(v)
     ]
     output["persistable_vars"] = [
         {

python/paddle/jit/api.py

@@ -1209,7 +1209,11 @@ def save(layer, path, input_spec=None, **configs):
             paddle.static.save_vars(
                 Executor(_current_expected_place()),
                 dirname=model_path,
-                vars=list(filter(paddle.fluid.io.is_persistable, ordered_vars)),
+                vars=list(
+                    filter(
+                        paddle.framework.io_utils.is_persistable, ordered_vars
+                    )
+                ),
                 filename=params_filename,
             )
         # save property

python/paddle/static/__init__.py

@@ -27,6 +27,13 @@ from .io import serialize_program  # noqa: F401
 from .io import load_from_file  # noqa: F401
 from .io import save_to_file  # noqa: F401
 from .io import normalize_program  # noqa: F401
+from .io import is_persistable  # noqa: F401
+from .io import save_vars  # noqa: F401
+from .io import load_vars  # noqa: F401
+from .io import save  # noqa: F401
+from .io import load  # noqa: F401
+from .io import load_program_state  # noqa: F401
+from .io import set_program_state  # noqa: F401
 from ..fluid import Scope  # noqa: F401
 from .input import data  # noqa: F401
 from .input import InputSpec  # noqa: F401
@@ -66,13 +73,6 @@ from ..fluid.param_attr import WeightNormParamAttr  # noqa: F401
 from ..fluid.optimizer import Optimizer  # noqa: F401
 from ..fluid.optimizer import Adam  # noqa: F401
 from ..fluid.optimizer import ExponentialMovingAverage  # noqa: F401
-from ..fluid.io import save  # noqa: F401
-from ..fluid.io import load  # noqa: F401
-from ..fluid.io import load_program_state  # noqa: F401
-from ..fluid.io import set_program_state  # noqa: F401
-from ..fluid.io import load_vars  # noqa: F401
-from ..fluid.io import save_vars  # noqa: F401
-from ..fluid.io import batch  # noqa: F401
 
 from ..fluid.contrib.layers import ctr_metric_bundle  # noqa: F401
 from ..fluid.layers import exponential_decay  # noqa: F401

python/paddle/static/io.py

@@ -16,6 +16,8 @@ import errno
 import inspect
 import logging
 import os
+import pickle
+import sys
 import warnings
 
 import numpy as np
@@ -30,10 +32,20 @@ from paddle.fluid import (
     program_guard,
     unique_name,
 )
-from paddle.fluid.executor import global_scope
-from paddle.fluid.framework import Parameter, static_only
+from paddle.fluid.executor import Executor, global_scope
+from paddle.fluid.framework import Parameter, dygraph_not_support, static_only
 from paddle.fluid.io import append_fetch_ops, prepend_feed_ops
 from paddle.fluid.log_helper import get_logger
+from paddle.framework.io_utils import (
+    _clone_var_in_block_,
+    _load_program_scope,
+    _pack_loaded_dict,
+    _pickle_loads_mac,
+    _unpack_saved_dict,
+    is_belong_to_optimizer,
+    is_parameter,
+    is_persistable,
+)
 
 __all__ = []
 
@@ -230,37 +242,6 @@ def normalize_program(program, feed_vars, fetch_vars):
     return copy_program
 
 
-def is_persistable(var):
-    """
-
-    Check whether the given variable is persistable.
-
-    Args:
-        var(Variable): The variable to be checked.
-
-    Returns:
-        bool: True if the given `var` is persistable
-        False if not.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.fluid as fluid
-
-            paddle.enable_static()
-            param = fluid.default_main_program().global_block().var('fc.b')
-            res = fluid.io.is_persistable(param)
-    """
-    if (
-        var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH
-        or var.desc.type() == core.VarDesc.VarType.FETCH_LIST
-        or var.desc.type() == core.VarDesc.VarType.READER
-    ):
-        return False
-    return var.persistable
-
-
 @static_only
 def serialize_program(feed_vars, fetch_vars, **kwargs):
     """
@@ -882,3 +863,1048 @@ def load_inference_model(path_prefix, executor, **kwargs):
     ]
 
     return [program, feed_target_names, fetch_targets]
+
+
+@dygraph_not_support
+def save_vars(
+    executor,
+    dirname,
+    main_program=None,
+    vars=None,
+    predicate=None,
+    filename=None,
+):
+    """
+    Save specific variables in the `Program` to files.
+
+    There are two ways to specify the variables to be saved: set variables in
+    a list and assign it to the `vars`, or use the `predicate` function to select
+    variables that make `predicate(variable) == True`. The first way has a higher priority.
+
+    The `dirname` is used to specify the folder where to save variables.
+    If you prefer to save variables in separate files in the `dirname` folder,
+    do not set `filename`. If you prefer to save all variables in a single file,
+    use `filename` to specify it.
+
+    Args:
+        executor(Executor): The executor to run for saving variables.
+        dirname(str, optional): The folder where to save variables.
+                            When you need to save the parameter to the memory, set it to None.
+        main_program(Program, optional): The program whose variables will be saved.
+                                    If it is None, the default main program will
+                                    be used automatically.
+                                    Default: None
+        vars(list[Variable], optional): The list contains all variables to be saved.
+                                        Default: None
+        predicate(function, optional): The function selects the variables that make
+                                       `predicate(variable) == True`.
+                                       Default: None
+        filename(str, optional): If you prefer to save all variables in a single file,
+                                 use `filename` to specify it. Otherwise, let `filename` be None.
+                                 Default: None
+
+    Returns:
+        str: When saving parameters to a file, returns None.
+             When saving parameters to memory, returns a binary string containing parameters.
+
+    Raises:
+        TypeError: If `main_program` is not an instance of Program nor None.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.fluid as fluid
+
+            paddle.enable_static()
+            main_prog = fluid.Program()
+            startup_prog = fluid.Program()
+            with fluid.program_guard(main_prog, startup_prog):
+                data = paddle.static.data(name="img", shape=[64, 784])
+                w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
+                b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
+                hidden_w = paddle.matmul(x=data, y=w)
+                hidden_b = paddle.add(hidden_w, b)
+            place = fluid.CPUPlace()
+            exe = fluid.Executor(place)
+            exe.run(startup_prog)
+
+            # The first usage: use `vars` to set the saved variables.
+            var_list = [w, b]
+            path = "./my_paddle_vars"
+            fluid.io.save_vars(executor=exe, dirname=path, vars=var_list,
+                            filename="vars_file")
+            # w and b will be save in a file named "var_file".
+
+            # The second usage: use `predicate` to select the saved variable.
+            def name_has_fc(var):
+                res = "fc" in var.name
+                return res
+            param_path = "./my_paddle_model"
+            fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog, vars=None, predicate = name_has_fc)
+            # all variables whose names contain "fc " are saved.
+    """
+    save_to_memory = False
+    if dirname is None and filename is None:
+        save_to_memory = True
+
+    main_program = paddle.static.io._get_valid_program(main_program)
+
+    if vars is None:
+        return save_vars(
+            executor,
+            main_program=main_program,
+            dirname=dirname,
+            vars=list(filter(predicate, main_program.list_vars())),
+            filename=filename,
+        )
+    else:
+        params_var_name = "saved_params"
+        # give warning when there is no var in model
+        if len(list(vars)) == 0:
+            warnings.warn(
+                "no variable in your model, please ensure there are any variables in your model to save"
+            )
+            return None
+
+        save_program = Program()
+        save_block = save_program.global_block()
+
+        save_var_map = {}
+        for each_var in vars:
+            # NOTE: don't save the variable which type is RAW
+            if each_var.type == core.VarDesc.VarType.RAW:
+                continue
+            new_var = _clone_var_in_block_(save_block, each_var)
+            if filename is None and save_to_memory is False:
+                save_file_path = os.path.join(
+                    os.path.normpath(dirname), new_var.name
+                )
+                save_block.append_op(
+                    type='save',
+                    inputs={'X': [new_var]},
+                    outputs={},
+                    attrs={'file_path': os.path.normpath(save_file_path)},
+                )
+            else:
+                save_var_map[new_var.name] = new_var
+
+        if filename is not None or save_to_memory:
+            save_var_list = []
+            for name in sorted(save_var_map.keys()):
+                save_var_list.append(save_var_map[name])
+
+            save_path = str()
+            if save_to_memory is False:
+                save_path = os.path.join(os.path.normpath(dirname), filename)
+
+            saved_params = save_block.create_var(
+                type=core.VarDesc.VarType.RAW, name=params_var_name
+            )
+            saved_params.desc.set_persistable(True)
+            save_block.append_op(
+                type='save_combine',
+                inputs={'X': save_var_list},
+                outputs={'Y': saved_params},
+                attrs={
+                    'file_path': save_path,
+                    'save_to_memory': save_to_memory,
+                },
+            )
+
+        # NOTE(zhiqiu): save op will add variable kLookupTablePath in save_program.desc,
+        # which leads to diff on save_program and its desc. Call _sync_with_cpp
+        # to keep consistency.
+        save_program._sync_with_cpp()
+        executor.run(save_program)
+        if save_to_memory:
+            return global_scope().find_var(params_var_name).get_bytes()
+
+
+def load_vars(
+    executor,
+    dirname,
+    main_program=None,
+    vars=None,
+    predicate=None,
+    filename=None,
+):
+    """
+    :api_attr: Static Graph
+
+    This API loads variables from files by executor.
+
+    There are two ways to specify the variables to be loaded: the first way, set
+    variables in a list and assign it to the `vars`; the second way, use the
+    `predicate` function to select variables that make `predicate(variable) == True`.
+    The first way has a higher priority.
+
+    The `dirname` is used to specify the folder where to load variables.
+    If variables were saved in separate files in the folder `dirname`,
+    set `filename` None. If all variables were saved in a single file,
+    use `filename` to specify it.
+
+    Args:
+        executor(Executor): The executor to run for loading variables.
+        dirname(str): The folder where to load the variables.
+        main_program(Program, optional): The program whose variables will be loaded.
+                                    If it is None, the default main program will
+                                    be used automatically.
+                                    Default: None
+        vars(list[Variable], optional): The list that contains all variables to be loaded.
+                                   Default: None
+        predicate(function, optional): The function selects variables that make
+                                        `predicate(variable) == True`.
+                                        Default: None
+        filename(str, optional): The file which saved all required variables. If variables
+                                were saved in separate files, set it to be None.
+                                Default: None
+
+    Returns:
+        None
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.fluid as fluid
+
+            paddle.enable_static()
+            main_prog = fluid.Program()
+            startup_prog = fluid.Program()
+            with fluid.program_guard(main_prog, startup_prog):
+                data = paddle.static.data(name="img", shape=[64, 784])
+                w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
+                b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
+                hidden_w = paddle.matmul(x=data, y=w)
+                hidden_b = paddle.add(hidden_w, b)
+            place = fluid.CPUPlace()
+            exe = fluid.Executor(place)
+            exe.run(startup_prog)
+
+            # The first usage: using `vars` to specify the variables.
+            path = "./my_paddle_vars"
+            var_list = [w, b]
+            fluid.io.save_vars(executor=exe, dirname=path, vars=var_list,
+                               filename="vars_file")
+            fluid.io.load_vars(executor=exe, dirname=path, vars=var_list,
+                               filename="vars_file")
+            # w and b will be loaded, and they are supposed to
+            # be saved in the same file named 'var_file' in the path "./my_paddle_vars".
+
+            # The second usage: using the `predicate` function to select variables
+            param_path = "./my_paddle_model"
+            def name_has_fc(var):
+                res = "fc" in var.name
+                return res
+            fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog,
+                              vars=None, predicate=name_has_fc)
+            fluid.io.load_vars(executor=exe, dirname=param_path, main_program=main_prog,
+                               vars=None, predicate=name_has_fc)
+            # Load All variables in the `main_program` whose name includes "fc".
+            # And all the variables are supposed to be saved in separate files.
+
+    """
+    vars_from_memory = False
+    if dirname is not None:
+        dirname = os.path.normpath(dirname)
+    else:
+        vars_from_memory = True
+
+    if vars is None:
+        if main_program is None:
+            main_program = default_main_program()
+        if not isinstance(main_program, Program):
+            raise TypeError(
+                "The type of input main_program is invalid, expected type is fluid.Program, but received %s"
+                % type(main_program)
+            )
+
+        load_vars(
+            executor,
+            dirname=dirname,
+            main_program=main_program,
+            vars=list(filter(predicate, main_program.list_vars())),
+            filename=filename,
+        )
+    else:
+        load_prog = Program()
+        load_block = load_prog.global_block()
+
+        if main_program is None:
+            main_program = default_main_program()
+
+        if not isinstance(main_program, Program):
+            raise TypeError(
+                "The type of input main_program is invalid, expected type is fluid.Program, but received %s"
+                % type(main_program)
+            )
+
+        # save origin param shape
+        orig_para_shape = {}
+        load_var_map = {}
+
+        check_vars = []
+        sparse_vars = []
+
+        for each_var in vars:
+            assert isinstance(each_var, Variable)
+
+            if each_var.type == core.VarDesc.VarType.RAW:
+                continue
+
+            if isinstance(each_var, Parameter):
+                orig_para_shape[each_var.name] = tuple(
+                    each_var.desc.get_shape()
+                )
+
+            if each_var.type == core.VarDesc.VarType.SELECTED_ROWS:
+                sparse_vars.append(each_var)
+                continue
+
+            new_var = _clone_var_in_block_(load_block, each_var)
+            check_vars.append(each_var)
+
+            if filename is None:
+                if dirname is None:
+                    raise ValueError(
+                        "The directory path and params cannot be None at the same time."
+                    )
+                load_block.append_op(
+                    type='load',
+                    inputs={},
+                    outputs={'Out': [new_var]},
+                    attrs={'file_path': os.path.join(dirname, new_var.name)},
+                )
+            else:
+                load_var_map[new_var.name] = new_var
+
+        for each_var in sparse_vars:
+            assert isinstance(each_var, Variable)
+
+            if filename is not None:
+                raise ValueError(
+                    "SelectedRows can not be load with load_combine"
+                )
+
+            new_var = _clone_var_in_block_(load_block, each_var)
+
+            var_path = os.path.join(dirname, new_var.name)
+            if not os.path.exists(var_path):
+                raise ValueError(
+                    "SelectedRows var {} can not find at {}".format(
+                        new_var.name, var_path
+                    )
+                )
+
+            if os.path.isfile(var_path):
+                load_block.append_op(
+                    type='load',
+                    inputs={},
+                    outputs={'Out': [new_var]},
+                    attrs={'file_path': os.path.join(dirname, new_var.name)},
+                )
+            else:
+                blocks = []
+                block_paths = os.listdir(var_path)
+
+                for block in block_paths:
+                    if block.startswith(new_var.name):
+                        blocks.append(block)
+
+                slices = []
+                for block in blocks:
+                    slice = load_block.create_var(
+                        name=block,
+                        type=new_var.type,
+                        shape=new_var.shape,
+                        dtype=new_var.dtype,
+                        persistable=False,
+                    )
+                    slices.append(slice)
+
+                    file_path = os.path.join(var_path, block, "Param")
+                    load_block.append_op(
+                        type='load',
+                        inputs={},
+                        outputs={'Out': [slice]},
+                        attrs={'file_path': file_path},
+                    )
+
+                load_block.append_op(
+                    type='lookup_sparse_table_merge',
+                    inputs={'X': slices},
+                    outputs={'Out': new_var},
+                    attrs={},
+                )
+
+        if filename is not None:
+            load_var_list = []
+            for name in sorted(load_var_map.keys()):
+                load_var_list.append(load_var_map[name])
+
+            if vars_from_memory is False:
+                filename = os.path.join(dirname, filename)
+
+            load_block.append_op(
+                type='load_combine',
+                inputs={},
+                outputs={"Out": load_var_list},
+                attrs={
+                    'file_path': filename,
+                    'model_from_memory': vars_from_memory,
+                },
+            )
+        executor.run(load_prog)
+
+        # check var shape
+        for each_var in check_vars:
+            if not isinstance(each_var, Parameter):
+                continue
+            var_temp = paddle.fluid.global_scope().find_var(each_var.name)
+            assert var_temp is not None, "can't not find var: " + each_var.name
+            new_shape = (np.array(var_temp.get_tensor())).shape
+            assert each_var.name in orig_para_shape, (
+                each_var.name + "MUST in var list"
+            )
+            orig_shape = orig_para_shape.get(each_var.name)
+            if new_shape != orig_shape:
+                raise RuntimeError(
+                    "Variable's shape does not match, the Program requires a parameter with the shape of ({}), "
+                    "while the loaded parameter (namely [ {} ]) has a shape of  ({}).".format(
+                        orig_shape, each_var.name, new_shape
+                    )
+                )
+
+
+@static_only
+def save(program, model_path, protocol=4, **configs):
+    """
+
+    This function save parameters, optimizer information and network description to model_path.
+
+    The parameters contains all the trainable Tensor, will save to a file with suffix ".pdparams".
+    The optimizer information contains all the Tensor used by optimizer. For Adam optimizer, contains beta1, beta2, momentum etc. All the information will save to a file with suffix ".pdopt". (If the optimizer have no Tensor need to save (like SGD), the fill will not generated).
+    The network description is the description of the program. It's only used for deployment. The description  will save to a file with a suffix ".pdmodel".
+
+    Args:
+        program(Program) : The program to saved.
+        model_path(str): the file prefix to save the program. The format is "dirname/file_prefix". If file_prefix is empty str. A exception will be raised
+        protocol(int, optional): The protocol version of pickle module must be greater than 1 and less than 5.
+                                 Default: 4
+        configs(dict, optional) : optional keyword arguments.
+
+    Returns:
+        None
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.static as static
+
+            paddle.enable_static()
+
+            x = static.data(name="x", shape=[10, 10], dtype='float32')
+            y = static.nn.fc(x, 10)
+            z = static.nn.fc(y, 10)
+
+            place = paddle.CPUPlace()
+            exe = static.Executor(place)
+            exe.run(static.default_startup_program())
+            prog = static.default_main_program()
+
+            static.save(prog, "./temp")
+    """
+
+    base_name = os.path.basename(model_path)
+    assert (
+        base_name != ""
+    ), "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received model_path is empty string."
+    if 'pickle_protocol' in configs:
+        protocol = configs['pickle_protocol']
+        warnings.warn(
+            "'pickle_protocol' is a deprecated argument. Please use 'protocol' instead."
+        )
+
+    if not isinstance(protocol, int):
+        raise ValueError(
+            "The 'protocol' MUST be `int`, but received {}".format(
+                type(protocol)
+            )
+        )
+
+    if protocol < 2 or protocol > 4:
+        raise ValueError(
+            "Expected 1<'protocol'<5, but received protocol={}".format(protocol)
+        )
+
+    dir_name = os.path.dirname(model_path)
+    if dir_name and not os.path.exists(dir_name):
+        os.makedirs(dir_name)
+
+    def get_tensor(var):
+        t = global_scope().find_var(var.name).get_tensor()
+        return np.array(t)
+
+    parameter_list = list(filter(is_parameter, program.list_vars()))
+    param_dict = {p.name: get_tensor(p) for p in parameter_list}
+
+    param_dict = _unpack_saved_dict(param_dict, protocol)
+
+    # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
+    if sys.platform == 'darwin' and sys.version_info.major == 3:
+        pickle_bytes = pickle.dumps(param_dict, protocol=protocol)
+        with open(model_path + ".pdparams", 'wb') as f:
+            max_bytes = 2**30
+            for i in range(0, len(pickle_bytes), max_bytes):
+                f.write(pickle_bytes[i : i + max_bytes])
+    else:
+        with open(model_path + ".pdparams", 'wb') as f:
+            pickle.dump(param_dict, f, protocol=protocol)
+
+    optimizer_var_list = list(
+        filter(is_belong_to_optimizer, program.list_vars())
+    )
+
+    opt_dict = {p.name: get_tensor(p) for p in optimizer_var_list}
+    with open(model_path + ".pdopt", 'wb') as f:
+        pickle.dump(opt_dict, f, protocol=protocol)
+
+    main_program = program.clone()
+    program.desc.flush()
+    main_program.desc._set_version()
+    paddle.fluid.core.save_op_version_info(program.desc)
+
+    with open(model_path + ".pdmodel", "wb") as f:
+        f.write(program.desc.serialize_to_string())
+
+
+@static_only
+def load(program, model_path, executor=None, var_list=None):
+    """
+    :api_attr: Static Graph
+
+    This function get parameters and optimizer information from program, and then get corresponding value from file.
+    An exception will throw if shape or dtype of the parameters is not match.
+
+    This function can also load model file saved with [ save_params, save_persistables, save_vars ].
+    var_list can not be None  when load single model file
+    ( filename is not None When save_params, save_persistables or save_vars is called ).
+
+    Args:
+        program(Program): The program will be loaded
+        model_path(str): The file prefix store the program
+        executor(Executor, optional): The executor used for initialize the parameter
+                                      When startup program is not run.
+        var_list(list|tuple, optional): The Tensor list/tuple to load single model file saved with
+                                  [ save_params, save_persistables, save_vars ].
+                                  Default: None
+
+    Returns:
+        None
+
+     Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.static as static
+
+            paddle.enable_static()
+
+            x = static.data(name="x", shape=[10, 10], dtype='float32')
+            y = static.nn.fc(x, 10)
+            z = static.nn.fc(y, 10)
+
+            place = paddle.CPUPlace()
+            exe = static.Executor(place)
+            exe.run(static.default_startup_program())
+            prog = static.default_main_program()
+
+            static.save(prog, "./temp")
+            static.load(prog, "./temp")
+    """
+
+    assert executor is None or isinstance(executor, Executor)
+
+    model_prefix = model_path
+    if model_prefix.endswith(".pdparams"):
+        model_prefix = model_prefix[:-9]
+    elif model_prefix.endswith(".pdopt"):
+        model_prefix = model_prefix[:-6]
+    elif model_prefix.endswith(".pdmodel"):
+        model_prefix = model_prefix[:-8]
+
+    parameter_file_name = model_prefix + ".pdparams"
+
+    if not os.path.exists(parameter_file_name):
+        # model file save by fluid.save not found, try to load model file saved with
+        # [save_vars, save_params, save_persistables]
+        _logger.debug(
+            "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format(
+                parameter_file_name
+            )
+        )
+        if executor is None:
+            raise ValueError(
+                "executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
+            )
+
+        if var_list is not None:
+            var_list_names = [var.name for var in var_list]
+        else:
+            var_list_names = None
+
+        if os.path.isdir(model_path):
+            binary_file_set = set()
+            for root, dirs, files in os.walk(model_path, topdown=False):
+                for f in files:
+                    binary_file_set.add(
+                        os.path.join(root, f).replace("\\", "/")
+                    )
+            program_var_list = list(program.list_vars())
+            loaded_var_list = []
+            for var in program_var_list:
+                var_path = os.path.join(model_path, var.name).replace("\\", "/")
+                load_condition = (
+                    var_list_names is None or var.name in var_list_names
+                )
+                if var_path in binary_file_set and load_condition:
+                    loaded_var_list.append(var)
+                    binary_file_set.remove(var_path)
+            if len(binary_file_set) > 0:
+                unused_var_list = " ".join(list(binary_file_set))
+                _logger.warning(
+                    "variable file [ %s ] not used"
+                    % (" ".join(list(binary_file_set)))
+                )
+            try:
+                load_vars(
+                    executor=executor, dirname=model_path, vars=loaded_var_list
+                )
+            except RuntimeError as e:
+                _logger.error(e)
+                raise e
+            except:
+                raise RuntimeError(
+                    "Failed to load model file, please make sure model file is saved with the "
+                    "following APIs: save_params, save_persistables, save_vars"
+                )
+
+            return
+        elif os.path.isfile(model_path):
+            if var_list is None:
+                raise ValueError(
+                    "var_list is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
+                )
+            program_var_list = program.list_vars()
+            program_var_name_set = set([var.name for var in program_var_list])
+
+            # check all the variable inlcuded in program
+            for var in var_list:
+                if var.name not in program_var_name_set:
+                    raise LookupError(
+                        "loaded var [{}] is not in program variable list"
+                    )
+
+            dir_name, file_name = os.path.split(model_path)
+            try:
+                load_vars(
+                    executor=executor,
+                    dirname=dir_name,
+                    vars=var_list,
+                    filename=file_name,
+                )
+            except RuntimeError as e:
+                _logger.error(e)
+                raise e
+            except:
+                raise RuntimeError(
+                    "Failed to load model file , please make sure model file is saved with the "
+                    "the following APIs: [ save_params, save_persistables, save_vars ]. "
+                    "When these API called, filename CANNOT be None"
+                )
+
+            return
+
+    def set_var(var, ndarray):
+        t = global_scope().find_var(var.name).get_tensor()
+        p = t._place()
+        if p.is_cpu_place():
+            place = paddle.fluid.CPUPlace()
+        elif p.is_cuda_pinned_place():
+            place = paddle.fluid.CUDAPinnedPlace()
+        elif p.is_xpu_place():
+            p = paddle.fluid.core.Place()
+            p.set_place(t._place())
+            place = paddle.fluid.XPUPlace(p.xpu_device_id())
+        elif p.is_npu_place():
+            p = paddle.fluid.core.Place()
+            p.set_place(t._place())
+            place = paddle.fluid.NPUPlace(p.npu_device_id())
+        elif p.is_mlu_place():
+            p = paddle.fluid.core.Place()
+            p.set_place(t._place())
+            place = paddle.fluid.MLUPlace(p.mlu_device_id())
+        else:
+            p = paddle.fluid.core.Place()
+            p.set_place(t._place())
+            place = paddle.fluid.CUDAPlace(p.gpu_device_id())
+
+        t.set(ndarray, place)
+
+    parameter_list = list(filter(is_parameter, program.list_vars()))
+
+    if executor:
+        paddle.fluid.core._create_loaded_parameter(
+            parameter_list, global_scope(), executor._default_executor
+        )
+    with open(parameter_file_name, 'rb') as f:
+
+        # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
+        if sys.platform == 'darwin' and sys.version_info.major == 3:
+            load_dict = _pickle_loads_mac(parameter_file_name, f)
+        else:
+            load_dict = pickle.load(f, encoding='latin1')
+        load_dict = _pack_loaded_dict(load_dict)
+    for v in parameter_list:
+        assert (
+            v.name in load_dict
+        ), "Can not find [{}] in model file [{}]".format(
+            v.name, parameter_file_name
+        )
+        set_var(v, load_dict[v.name])
+
+    optimizer_var_list = list(
+        filter(is_belong_to_optimizer, program.list_vars())
+    )
+
+    if len(optimizer_var_list) > 0:
+        opt_file_name = model_prefix + ".pdopt"
+        assert os.path.exists(
+            opt_file_name
+        ), "Optimizer file [{}] not exits".format(opt_file_name)
+
+        if executor:
+            paddle.fluid.core._create_loaded_parameter(
+                optimizer_var_list, global_scope(), executor._default_executor
+            )
+
+        with open(opt_file_name, 'rb') as f:
+            load_dict = pickle.load(f, encoding='latin1')
+        for v in optimizer_var_list:
+            assert (
+                v.name in load_dict
+            ), "Can not find [{}] in model file [{}]".format(
+                v.name, opt_file_name
+            )
+            set_var(v, load_dict[v.name])
+
+
+@static_only
+def set_program_state(program, state_dict):
+    """
+    Set program parameter from state_dict
+
+    An exception will throw if shape or dtype of the parameters is not match.
+
+    NOTICE: This function MUST called after run start_up_program
+
+    Args:
+        program(Program): The program to be set
+        state_dict(dict): the dict store Parameter and optimizer information
+    Returns:
+        None
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.static as static
+
+            paddle.enable_static()
+
+            x = static.data(name="x", shape=[10, 10], dtype='float32')
+            y = static.nn.fc(x, 10)
+            z = static.nn.fc(y, 10)
+
+            place = paddle.CPUPlace()
+            exe = static.Executor(place)
+            exe.run(static.default_startup_program())
+            prog = static.default_main_program()
+
+            static.save(prog, "./temp")
+            program_state = static.load_program_state("./temp")
+
+            static.set_program_state(prog, program_state)
+    """
+    state_dict = _pack_loaded_dict(state_dict)
+    parameter_list = list(filter(is_persistable, program.list_vars()))
+
+    used_para_list = {}
+    for para in parameter_list:
+        var_temp = paddle.fluid.global_scope().find_var(para.name)
+        assert (
+            var_temp is not None
+        ), "Variable [ {} ] Not found, Please make sure run startup program".format(
+            para.name
+        )
+        if para.name in state_dict:
+            # set value from state dict
+            orig_para_np = np.array(var_temp.get_tensor())
+            new_para_np = state_dict[para.name]
+            assert orig_para_np.shape == new_para_np.shape, (
+                "Parameter's shape does not match, the Program requires a parameter with the shape of ({}), "
+                "while the loaded parameter (namely [ {} ]) has a shape of  ({}).".format(
+                    orig_para_np.shape, para.name, new_para_np.shape
+                )
+            )
+            assert orig_para_np.dtype == new_para_np.dtype, (
+                "Parameter's data type does not match, the Program requires a parameter with a dtype of ({}), "
+                "while the loaded parameter (namely [ {} ]) has a dtype of  ({}).".format(
+                    orig_para_np.dtype, para.name, new_para_np.dtype
+                )
+            )
+
+            ten = var_temp.get_tensor()
+            ten_place = ten._place()
+
+            # assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \
+            #    "Place not support, only support CPUPlace and GPUPlace, now is {}".format(str(ten_place))
+            py_place = paddle.fluid.CPUPlace()
+            if ten_place.is_cuda_pinned_place():
+                place = paddle.fluid.CUDAPinnedPlace()
+            elif ten_place.is_gpu_place():
+                p = paddle.fluid.core.Place()
+                p.set_place(ten_place)
+                py_place = paddle.fluid.CUDAPlace(p.gpu_device_id())
+            elif ten_place.is_xpu_place():
+                p = paddle.fluid.core.Place()
+                p.set_place(ten_place)
+                py_place = paddle.fluid.XPUPlace(p.xpu_device_id())
+            elif ten_place.is_npu_place():
+                p = paddle.fluid.core.Place()
+                p.set_place(ten_place)
+                py_place = paddle.fluid.NPUPlace(p.npu_device_id())
+            elif ten_place.is_mlu_place():
+                p = paddle.fluid.core.Place()
+                p.set_place(ten_place)
+                py_place = paddle.fluid.MLUPlace(p.mlu_device_id())
+
+            ten.set(new_para_np, py_place)
+
+            used_para_list[para.name] = 1
+
+    unused_para_list = []
+    for k, v in state_dict.items():
+        if k not in used_para_list:
+            unused_para_list.append(k)
+    if len(unused_para_list) > 0:
+        warnings.warn(
+            "This list is not set, Because of Paramerter not found in program. There are: {}".format(
+                " ".join(unused_para_list)
+            )
+        )
+
+
+@dygraph_not_support
+def get_program_persistable_vars(program):
+    """
+    Get all the persistable vars from Program.
+    Args:
+        var(Program): The Program to get persistable vars
+    Returns:
+        list: The list contains all persistable vars in the program
+    Examples:
+        .. code-block:: python
+            import paddle
+            import paddle.static.io as io
+            import paddle.fluid as fluid
+            paddle.enable_static()
+            data = fluid.data(name="img", shape=[64, 784])
+            w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
+            b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
+            list_para  = io.get_program_persistable_vars(  fluid.default_main_program() )
+    """
+    return list(filter(is_persistable, program.list_vars()))
+
+
+def load_program_state(model_path, var_list=None):
+    """
+
+    Load program state from local file
+
+    Args:
+        model_path(str): The file prefix store the program
+        var_list(list|tuple, optional): The Tensor list/tuple to load saved with
+                                  [ save_params, save_persistables, save_vars ].
+                                  Default: None.
+                                  The var_list is only used to get name,
+                                  will not be modified.
+    Returns:
+        state_dict(dict): the dict store Parameter and optimizer information
+
+    Examples:
+
+        .. code-block:: python
+
+            import paddle
+            import paddle.static as static
+
+            paddle.enable_static()
+
+            x = static.data(name="x", shape=[10, 10], dtype='float32')
+            y = static.nn.fc(x, 10)
+            z = static.nn.fc(y, 10)
+
+            place = paddle.CPUPlace()
+            exe = static.Executor(place)
+            exe.run(static.default_startup_program())
+            prog = static.default_main_program()
+
+            static.save(prog, "./temp")
+            program_state = static.load_program_state("./temp")
+    """
+    model_prefix = model_path
+    if model_prefix.endswith(".pdparams"):
+        model_prefix = model_prefix[:-9]
+    elif model_prefix.endswith(".pdopt"):
+        model_prefix = model_prefix[:-6]
+    elif model_prefix.endswith(".pdmodel"):
+        model_prefix = model_prefix[:-8]
+
+    parameter_file_name = model_prefix + ".pdparams"
+    if not os.path.exists(parameter_file_name):
+        # model file saved with fluid.save is not found, try to load model file saved with
+        # [save_vars, save_params, save_persistables]
+        _logger.debug(
+            "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format(
+                parameter_file_name
+            )
+        )
+
+        var_name_list = []
+        if var_list is None and os.path.isfile(model_path):
+            raise ValueError(
+                "var_list can not be None when model_path is a file type"
+            )
+
+        for root, dirs, files in os.walk(model_path, topdown=False):
+            for f in files:
+                file_path = os.path.join(root, f)
+                var_temp_name = os.path.relpath(file_path, model_path)
+                var_temp_name = var_temp_name.replace("\\", "/")
+                var_name_list.append(var_temp_name)
+
+        with _load_program_scope():
+            load_prog = Program()
+            load_block = load_prog.global_block()
+
+            def clone_var_to_block(block, var):
+                if not isinstance(var, Variable):
+                    raise TypeError("value in var_list must be variable")
+                return block.create_var(
+                    name=var.name,
+                    shape=var.shape,
+                    dtype=var.dtype,
+                    type=var.type,
+                    lod_level=var.lod_level
+                    if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR
+                    else None,
+                    persistable=True,
+                )
+
+            def _load_vars_with_try_catch(
+                exe, dirname, vars, filename, raise_error=True
+            ):
+                try:
+                    load_vars(
+                        executor=exe,
+                        dirname=dirname,
+                        vars=vars,
+                        filename=filename,
+                    )
+                    return True
+                except:
+                    error_str = (
+                        "Failed to load model/variables `%s`, please make sure "
+                        "model/variables file is saved with the following APIs: "
+                        "save_params, save_persistables, save_vars."
+                    )
+                    filenames = (
+                        [var.name for var in vars]
+                        if filename is None
+                        else filename
+                    )
+                    if raise_error:
+                        raise RuntimeError(error_str % filenames)
+                    else:
+                        warnings.warn(error_str % filenames, RuntimeWarning)
+                return False
+
+            place = paddle.fluid.CPUPlace()
+            exe = paddle.fluid.Executor(place)
+
+            loaded_var_list = []
+
+            if os.path.isfile(model_path):
+                # when model_path is file, var_list cannot be None
+                dir_name, file_name = os.path.split(model_path)
+                for var in var_list:
+                    loaded_var_list.append(clone_var_to_block(load_block, var))
+                _load_vars_with_try_catch(
+                    exe, dir_name, loaded_var_list, file_name
+                )
+            else:
+                # var_list can be None or not None
+                if var_list is not None:
+                    for var in var_list:
+                        loaded_var_list.append(
+                            clone_var_to_block(load_block, var)
+                        )
+                    _load_vars_with_try_catch(
+                        exe, model_path, loaded_var_list, None
+                    )
+                else:
+                    for var_name in var_name_list:
+                        # NOTE(chenweihang): If identify which files the user wants
+                        # to load from the disk, we load these variables one by one.
+                        # If a file does not exist, we only warn the user that the
+                        # file may be an irrelevant file, but does not throw an error
+                        # to ensure that other legal variables can be loaded.
+                        temp_var = load_block.create_var(
+                            name=var_name, persistable=True
+                        )
+                        if _load_vars_with_try_catch(
+                            exe, model_path, [temp_var], None, False
+                        ):
+                            loaded_var_list.append(temp_var)
+
+            res_dict = {}
+            for var in loaded_var_list:
+                res_dict[var.name] = np.asarray(
+                    paddle.fluid.global_scope().find_var(var.name).get_tensor()
+                )
+
+            return res_dict
+
+    assert os.path.exists(
+        parameter_file_name
+    ), "Parameter file [{}] not exits".format(parameter_file_name)
+
+    with open(parameter_file_name, 'rb') as f:
+        # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
+        if sys.platform == 'darwin' and sys.version_info.major == 3:
+            para_dict = _pickle_loads_mac(parameter_file_name, f)
+        else:
+            para_dict = pickle.load(f, encoding='latin1')
+    para_dict = _pack_loaded_dict(para_dict)
+
+    opt_file_name = model_prefix + ".pdopt"
+    if os.path.exists(opt_file_name):
+        with open(opt_file_name, 'rb') as f:
+            opti_dict = pickle.load(f, encoding='latin1')
+
+        para_dict.update(opti_dict)
+
+    return para_dict

python/paddle/static/quantization/tests/test_graph.py

@@ -15,8 +15,6 @@
 import os
 import unittest
 
-import numpy as np
-
 import paddle
 from paddle.fluid.framework import IrGraph
 from paddle.framework import core
@@ -103,40 +101,6 @@ class TestGraph(unittest.TestCase):
         _train(origin_binary)
         _train(backup_binary)
 
-        checkponit_dir = "checkpoint_gpu" if use_cuda else "checkpoint_cpu"
-
-        def _set_zero(var_name, scope, place):
-            var = scope.find_var(var_name).get_tensor()
-            var_array = np.zeros(var._get_dims()).astype("float32")
-            var.set(var_array, place)
-
-        sum_before = np.sum(
-            np.array(
-                paddle.static.global_scope()
-                .find_var('conv2d_1.w_0')
-                .get_tensor()
-            )
-        )
-        paddle.fluid.io._save_persistable_nodes(exe, checkponit_dir, graph)
-        _set_zero('conv2d_1.w_0', paddle.static.global_scope(), place)
-        set_after = np.sum(
-            np.array(
-                paddle.static.global_scope()
-                .find_var('conv2d_1.w_0')
-                .get_tensor()
-            )
-        )
-        self.assertEqual(set_after, 0)
-        paddle.fluid.io._load_persistable_nodes(exe, checkponit_dir, graph)
-        sum_after = np.sum(
-            np.array(
-                paddle.static.global_scope()
-                .find_var('conv2d_1.w_0')
-                .get_tensor()
-            )
-        )
-        self.assertEqual(sum_before, sum_after)
-
         marked_nodes = set()
         for op in graph.all_op_nodes():
             if op.name().find('conv2d') > -1: