diff --git a/python/paddle/distributed/auto_parallel/utils.py b/python/paddle/distributed/auto_parallel/utils.py index c4e64ee269fb8dcead1affea20e10081112f8460..070ea91bf75843d86ab63710ae31a17326826c15 100644 --- a/python/paddle/distributed/auto_parallel/utils.py +++ b/python/paddle/distributed/auto_parallel/utils.py @@ -22,8 +22,8 @@ from functools import reduce import numpy as np import paddle -from paddle.fluid.io import is_belong_to_optimizer, is_parameter from paddle.framework import core +from paddle.framework.io_utils import is_belong_to_optimizer, is_parameter from paddle.static import Variable from .dist_attribute import OperatorDistAttr, TensorDistAttr diff --git a/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py b/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py index 0d4c3944c72a2edaae4d97be1b7587969285cba1..ce3c6209eb3643c822a84c54f3c8a22ff2f88cc5 100644 --- a/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py +++ b/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py @@ -109,7 +109,6 @@ class ParameterServerRuntime(RuntimeBase): assert isinstance(each_var, Variable) origin_varname, _, _ = _get_varname_parts(each_var.name) - new_var = paddle.static.io._clone_var_in_block(load_block, each_var) var_path = os.path.join(dirname, origin_varname) if not os.path.exists(var_path): diff --git a/python/paddle/distributed/io.py b/python/paddle/distributed/io.py index 2af170d07aad9a3c3bae4a7d49690f432db86c56..7255e96c405b2589c45430efec423b8e3821b697 100644 --- a/python/paddle/distributed/io.py +++ b/python/paddle/distributed/io.py @@ -16,10 +16,175 @@ import os import paddle from paddle.fluid.framework import Program, static_only -from paddle.fluid.io import load_persistables from paddle.framework import core, dygraph_not_support +def _load_distributed_persistables(executor, dirname, main_program=None): + """ + customized load_persistables for distributed training. + it should be used on parameter server, + + Args: + executor(Executor): The executor to run for saving parameters. + dirname(str): The load directory path. + main_program(Program): The program whose parameters will be + loaded. the main_program must be the pserver_program + get after transpiler. + + Returns: + None + + Examples: + .. code-block:: python + + import paddle + import paddle.fluid as fluid + + paddle.enable_static() + exe = fluid.Executor(fluid.CPUPlace()) + param_path = "./my_paddle_model" + t = paddle.distributed.transpiler.DistributeTranspiler() + t.transpile(...) + pserver_prog = t.get_pserver_program(...) + _load_distributed_persistables(executor=exe, dirname=param_path, main_program=pserver_prog) + """ + + def __is_distributed_part_var(varname): + trainer_idx = varname.find(".trainer_") + block_idx = varname.find(".block") + return trainer_idx or block_idx + + def __load_persistable_vars(executor, dirname, need_load_vars): + load_prog = Program() + load_block = load_prog.global_block() + need_delete_vars = [] + + for param in need_load_vars: + origin_var = param.origin + slice_var = param.slice + is_slice = param.is_slice + offset = param.offset + + if is_slice: + slice = load_block.create_var( + name=slice_var.name, + type=slice_var.type, + shape=slice_var.shape, + dtype=slice_var.dtype, + persistable=True, + ) + + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [slice]}, + attrs={ + 'file_path': os.path.join(dirname, origin_var.name), + 'seek': offset, + 'shape': slice.shape, + }, + ) + else: + origin = load_block.create_var( + name="{}".format(origin_var.name), + type=origin_var.type, + shape=origin_var.shape, + dtype=origin_var.dtype, + persistable=True, + ) + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [origin]}, + attrs={'file_path': os.path.join(dirname, origin_var.name)}, + ) + + load_block.append_op( + type='delete_var', + inputs={'X': need_delete_vars}, + ) + + executor.run(load_prog) + + if not isinstance(main_program, Program): + raise TypeError("'main_program' should be an instance of Program.") + + if not main_program._is_distributed: + raise ValueError( + "'_load_distributed_persistables' just be designed for distributed training." + ) + + if not main_program._ps_endpoint: + raise ValueError( + "'_load_distributed_persistables' need current_endpoint set in DistributeTranspiler.transpile" + ) + + need_load_vars = ( + main_program._parameters_on_pservers.get_distributed_vars_by_ep( + main_program._ps_endpoint + ) + ) + __load_persistable_vars(executor, dirname, need_load_vars) + + +@dygraph_not_support +def load_persistables(executor, dirname, main_program=None, filename=None): + """ + :api_attr: Static Graph + + This API filters out all variables with ``persistable==True`` from the + given ``main_program`` and then tries to load these variables from the + directory ``dirname`` or the file ``filename``. + + Use the ``dirname`` to specify the directory where persistable variables + (refer to :ref:`api_guide_model_save_reader_en`) were saved. If variables + were saved in separate files, set ``filename`` as None; if all variables + were saved in a single file, use ``filename`` to specify the file name. + + Args: + executor(Executor): The executor used for loading persistable variables. + See :ref:`api_guide_executor_en` for more details about it. + dirname(str): The directory path. + main_program(Program, optional): The program whose persistable variables will + be loaded. If it is None, the ``default_main_program`` + will be used automatically. See :ref:`api_guide_Program_en` + for more about ``Program``. + Default: None. + filename(str, optional): The file which saved all persistable variables. If variables + were saved in separated files, set it to None. + Default: None. + + Returns: + None + + Examples: + .. code-block:: python + + import paddle + import paddle.fluid as fluid + + paddle.enable_static() + exe = fluid.Executor(fluid.CPUPlace()) + param_path = "./my_paddle_model" + prog = fluid.default_main_program() + paddle.distributed.io.load_persistables(executor=exe, dirname=param_path, + main_program=None) + """ + + if main_program and main_program._is_distributed: + _load_distributed_persistables( + executor, dirname=dirname, main_program=main_program + ) + else: + paddle.static.io.load_vars( + executor, + dirname=dirname, + main_program=main_program, + predicate=is_persistable, + filename=filename, + ) + + def _save_distributed_persistables(executor, dirname, main_program): """ save_persistables for distributed training. diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index 10293a4178653f26c628084c7c7a2c0ff184432a..82bd5cd621e916054a10895fc04d6044e6f68576 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -89,7 +89,6 @@ from .compiler import * from paddle.fluid.layers.math_op_patch import monkey_patch_variable from .dygraph.layers import * from .dygraph.base import enable_dygraph, disable_dygraph -from .io import save, load, load_program_state, set_program_state from .dygraph.varbase_patch_methods import monkey_patch_varbase from .core import _cuda_synchronize from .trainer_desc import ( @@ -146,8 +145,6 @@ __all__ = ( 'profiler', 'unique_name', 'Scope', - 'save', - 'load', '_cuda_synchronize', ] ) diff --git a/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py b/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py index 79161fe0fa0203165b8611c59493765ac988e53e..0b113c2b87fc865f7e0e09585ff97ade3adef5ce 100644 --- a/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py +++ b/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py @@ -34,7 +34,7 @@ class PaddleModel(SerializableBase): self._file_name = "_paddle_fleet_param__" def serialize(self, path): - from ...io import save_persistables + from paddle.distributed.io import save_persistables save_persistables( executor=self._exe, @@ -44,7 +44,7 @@ class PaddleModel(SerializableBase): ) def deserialize(self, path): - from ...io import load_persistables + from paddle.distributed.io import load_persistables load_persistables( executor=self._exe, diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index ac9563fb9217e98365f0d3e7af0caac7487c5842..bd5049e74f29a99f71249160a5410923a1260bc5 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -62,1205 +62,15 @@ from . import core from paddle.utils import deprecated from paddle.fluid.framework import static_only -batch = paddle.batch - -__all__ = [ - 'save_vars', - 'save_params', - 'save_persistables', - 'load_vars', - 'load_params', - 'load_persistables', - 'save_inference_model', - 'load_inference_model', - 'batch', - 'save', - 'load', - 'load_program_state', - 'set_program_state', - 'get_program_parameter', - 'get_program_persistable_vars', -] + reader.__all__ - -_logger = get_logger( - __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' -) - - -class _open_buffer: - def __init__(self, buffer): - self.buffer = buffer - - def __enter__(self): - return self.buffer - - -class _buffer_reader(_open_buffer): - def __init__(self, buffer): - super().__init__(buffer) - self.initial_tell = self.buffer.tell() - - def __exit__(self, *args): - # `args[0]` is type of exception. When the `read` is abnormal, the file pointer returns to the initial position. - if args[0] is not None: - self.buffer.seek(self.initial_tell) - - -class _buffer_writer(_open_buffer): - def __exit__(self, *args): - self.buffer.flush() - - -def _is_file_path(path): - return isinstance(path, str) - - -def _open_file_buffer(path_or_buffer, mode): - - if _is_file_path(path_or_buffer): - return open(path_or_buffer, mode) - else: - if 'w' in mode: - return _buffer_writer(path_or_buffer) - elif 'r' in mode: - return _buffer_reader(path_or_buffer) - else: - raise ValueError( - "Expected 'r' or 'w' in mode but got {}".format(mode) - ) - - -def _is_memory_buffer(buffer): - return isinstance(buffer, BytesIO) - - -def is_parameter(var): - """ - Check whether the given variable is an instance of Parameter. - - Args: - var(Variable): The variable to be checked. - - Returns: - bool: True if the given `var` is an instance of Parameter, - False if not. - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - param = fluid.default_main_program().global_block().var('fc.w') - res = fluid.io.is_parameter(param) - """ - return isinstance(var, Parameter) - - -def is_persistable(var): - """ - Check whether the given variable is persistable. - - Args: - var(Variable): The variable to be checked. - - Returns: - bool: True if the given `var` is persistable - False if not. - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - param = fluid.default_main_program().global_block().var('fc.b') - res = fluid.io.is_persistable(param) - """ - if ( - var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH - or var.desc.type() == core.VarDesc.VarType.FETCH_LIST - or var.desc.type() == core.VarDesc.VarType.READER - ): - return False - return var.persistable - - -def is_belong_to_optimizer(var): - if not (isinstance(var, Parameter) or var.desc.need_check_feed()): - return is_persistable(var) - - return False - - -@dygraph_not_support -def get_program_parameter(program): - """ - Get all the parameters from Program. - - Args: - var(Program): The Program to get parameters - - Returns: - list: The list contains all parameters in the program - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - data = fluid.data(name="img", shape=[64, 784]) - w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w') - b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b') - list_para = fluid.io.get_program_parameter( fluid.default_main_program() ) - """ - return list(filter(is_parameter, program.list_vars())) - - -@dygraph_not_support -def get_program_persistable_vars(program): - """ - Get all the persistable vars from Program. - - Args: - var(Program): The Program to get persistable vars - - Returns: - list: The list contains all persistable vars in the program - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - data = fluid.data(name="img", shape=[64, 784]) - w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w') - b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b') - list_para = fluid.io.get_program_persistable_vars( fluid.default_main_program() ) - """ - return list(filter(is_persistable, program.list_vars())) - - -def _clone_var_in_block_(block, var): - assert isinstance(var, Variable) - if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR: - return block.create_var( - name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=True, - ) - else: - return block.create_var( - name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - persistable=True, - ) - - -@signature_safe_contextmanager -def _load_program_scope(main=None, startup=None, scope=None): - prog = main if main else paddle.fluid.Program() - startup_prog = startup if startup else paddle.fluid.Program() - scope = scope if scope else paddle.fluid.core.Scope() - with paddle.fluid.scope_guard(scope): - with paddle.fluid.program_guard(prog, startup_prog): - with paddle.fluid.unique_name.guard(): - with paddle.fluid.framework._dygraph_guard(None): - yield - - -def _get_valid_program(main_program=None): - if main_program is None: - main_program = default_main_program() - elif isinstance(main_program, CompiledProgram): - main_program = main_program._program - if main_program is None: - raise TypeError( - "The type of input main_program is invalid, expected tyep is Program, but received None" - ) - warnings.warn( - "The input is a CompiledProgram, this is not recommended." - ) - if not isinstance(main_program, Program): - raise TypeError( - "The type of input main_program is invalid, expected type is fluid.Program, but received %s" - % type(main_program) - ) - return main_program - - -@dygraph_not_support -def save_vars( - executor, - dirname, - main_program=None, - vars=None, - predicate=None, - filename=None, -): - """ - Save specific variables in the `Program` to files. - - There are two ways to specify the variables to be saved: set variables in - a list and assign it to the `vars`, or use the `predicate` function to select - variables that make `predicate(variable) == True`. The first way has a higher priority. - - The `dirname` is used to specify the folder where to save variables. - If you prefer to save variables in separate files in the `dirname` folder, - do not set `filename`. If you prefer to save all variables in a single file, - use `filename` to specify it. - - Args: - executor(Executor): The executor to run for saving variables. - dirname(str, optional): The folder where to save variables. - When you need to save the parameter to the memory, set it to None. - main_program(Program, optional): The program whose variables will be saved. - If it is None, the default main program will - be used automatically. - Default: None - vars(list[Variable], optional): The list contains all variables to be saved. - Default: None - predicate(function, optional): The function selects the variables that make - `predicate(variable) == True`. - Default: None - filename(str, optional): If you prefer to save all variables in a single file, - use `filename` to specify it. Otherwise, let `filename` be None. - Default: None - - Returns: - str: When saving parameters to a file, returns None. - When saving parameters to memory, returns a binary string containing parameters. - - Raises: - TypeError: If `main_program` is not an instance of Program nor None. - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - main_prog = fluid.Program() - startup_prog = fluid.Program() - with fluid.program_guard(main_prog, startup_prog): - data = paddle.static.data(name="img", shape=[64, 784]) - w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w') - b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b') - hidden_w = paddle.matmul(x=data, y=w) - hidden_b = paddle.add(hidden_w, b) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(startup_prog) - - # The first usage: use `vars` to set the saved variables. - var_list = [w, b] - path = "./my_paddle_vars" - fluid.io.save_vars(executor=exe, dirname=path, vars=var_list, - filename="vars_file") - # w and b will be save in a file named "var_file". - - # The second usage: use `predicate` to select the saved variable. - def name_has_fc(var): - res = "fc" in var.name - return res - param_path = "./my_paddle_model" - fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog, vars=None, predicate = name_has_fc) - # all variables whose names contain "fc " are saved. - """ - save_to_memory = False - if dirname is None and filename is None: - save_to_memory = True - - main_program = _get_valid_program(main_program) - - if vars is None: - return save_vars( - executor, - main_program=main_program, - dirname=dirname, - vars=list(filter(predicate, main_program.list_vars())), - filename=filename, - ) - else: - params_var_name = "saved_params" - # give warning when there is no var in model - if len(list(vars)) == 0: - warnings.warn( - "no variable in your model, please ensure there are any variables in your model to save" - ) - return None - - save_program = Program() - save_block = save_program.global_block() - - save_var_map = {} - for each_var in vars: - # NOTE: don't save the variable which type is RAW - if each_var.type == core.VarDesc.VarType.RAW: - continue - new_var = _clone_var_in_block_(save_block, each_var) - if filename is None and save_to_memory is False: - save_file_path = os.path.join( - os.path.normpath(dirname), new_var.name - ) - save_block.append_op( - type='save', - inputs={'X': [new_var]}, - outputs={}, - attrs={'file_path': os.path.normpath(save_file_path)}, - ) - else: - save_var_map[new_var.name] = new_var - - if filename is not None or save_to_memory: - save_var_list = [] - for name in sorted(save_var_map.keys()): - save_var_list.append(save_var_map[name]) - - save_path = str() - if save_to_memory is False: - save_path = os.path.join(os.path.normpath(dirname), filename) - - saved_params = save_block.create_var( - type=core.VarDesc.VarType.RAW, name=params_var_name - ) - saved_params.desc.set_persistable(True) - save_block.append_op( - type='save_combine', - inputs={'X': save_var_list}, - outputs={'Y': saved_params}, - attrs={ - 'file_path': save_path, - 'save_to_memory': save_to_memory, - }, - ) - - # NOTE(zhiqiu): save op will add variable kLookupTablePath in save_program.desc, - # which leads to diff on save_program and its desc. Call _sync_with_cpp - # to keep consistency. - save_program._sync_with_cpp() - executor.run(save_program) - if save_to_memory: - return global_scope().find_var(params_var_name).get_bytes() - - -@dygraph_not_support -def save_params(executor, dirname, main_program=None, filename=None): - """ - Save all parameters from the :code:`main_program` to - the folder :code:`dirname` or file :code:`filename`. You can refer to - :ref:`api_guide_model_save_reader_en` for more details. - - Use the :code:`dirname` to specify the saving folder. If you would like to - save parameters in separate files, set :code:`filename` None; if you would - like to save all parameters in a single file, use :code:`filename` to specify - the file name. - - Note: - Some variables are not Parameter while they are necessary for - training, such as learning rate, global step, etc. So you can NOT save - and continue your training just by :ref:`api_fluid_io_save_params` - and :ref:`api_fluid_io_load_params`. Please use :ref:`api_fluid_io_save_persistables` - and :ref:`api_fluid_io_load_persistables` instead. - - If you want to save your model for the inference, please use the - :ref:`api_fluid_io_save_inference_model`. You can refer to - :ref:`api_guide_model_save_reader_en` for more details. - - Args: - executor(Executor): The executor to run for saving parameters, You can - refer to :ref:`api_guide_executor_en`. - dirname(str, optional): The saving directory path. - When you need to save the parameter to the memory, set it to None. - main_program(Program, optional): The program whose parameters will be - saved. You can refer to - :ref:`api_guide_Program_en` for more - details. If it is None, the default main - program will be used. - Default: None - filename(str, optional): The file to save all parameters. If you prefer - to save parameters in different files, set it - to None. - Default: None - - Returns: - str: When saving parameters to a file, returns None. - When saving parameters to memory, returns a binary string containing parameters. - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - - paddle.enable_static() - params_path = "./my_paddle_model" - image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32') - label = fluid.data(name='label', shape=[None, 1], dtype='int64') - feeder = fluid.DataFeeder(feed_list=[image, label], place=fluid.CPUPlace()) - predict = paddle.static.nn.fc(x=image, size=10, activation='softmax') - - loss = paddle.nn.functional.cross_entropy( - input=predict, label=label, - reduction='none', use_softmax=False - ) - avg_loss = paddle.mean(loss) - - exe = fluid.Executor(fluid.CPUPlace()) - exe.run(fluid.default_startup_program()) - fluid.io.save_params(executor=exe, dirname=params_path) - # The parameters weights and bias of the fc layer in the network are going to - # be saved in different files in the path "./my_paddle_model" - """ - return save_vars( - executor, - dirname=dirname, - main_program=main_program, - vars=None, - predicate=is_parameter, - filename=filename, - ) - - -def _save_distributed_persistables(executor, dirname, main_program): - """ - save_persistables for distributed training. - the method will do things listed below: - 1.save part of persistable variables on trainer. - 2.receive "remote prefetch variables" from parameter servers and merge them. - 3.save "distributed lookup table" on parameter servers. - 4.receive "optimizer variables" from parameter servers and merge them. - - Args: - executor(Executor): The executor to run for saving parameters. - dirname(str): The saving directory path. - main_program(Program): The program whose parameters will be - saved. the main_program must be the trainer_program - get after transpiler. - - Returns: - None - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - exe = fluid.Executor(fluid.CPUPlace()) - param_path = "./my_paddle_model" - t = paddle.distributed.transpiler.DistributeTranspiler() - t.transpile(...) - train_program = t.get_trainer_program() - _save_distributed_persistables(executor=exe, dirname=param_path, main_program=train_program) - """ - - def __save_remote_params(executor, dirname, remote_params_map): - """ - receive params on pserver through rpc. - if the params are be sliced, will concat them to one, then save it. - """ - if not remote_params_map: - return - - prog = Program() - block = prog.global_block() - - # recv optimize vars from pserver - for name, remote_params in remote_params_map.items(): - origin = remote_params[0].origin - is_slice = remote_params[0].is_slice - - slices = [None] * len(remote_params) - slice_varnames = [None] * len(remote_params) - remote_varnames = [None] * len(remote_params) - endpoints = [None] * len(remote_params) - - for idx, optimizer in enumerate(remote_params): - block_id = optimizer.block_id - slice = optimizer.slice - endpoint = optimizer.endpoint - - index = block_id if is_slice else idx - slices[index] = slice - slice_varnames[index] = "{}.slice.{}".format(slice.name, idx) - remote_varnames[index] = slice.name - endpoints[index] = endpoint - - slice_shapes = [] - for slice in slices: - tmp = [str(dim) for dim in slice.shape] - slice_shapes.append(",".join(tmp)) - - block.append_op( - type='recv_save', - attrs={ - "trainer_id": 0, - "shape": origin.shape, - "slice_shapes": slice_shapes, - "slice_varnames": slice_varnames, - "remote_varnames": remote_varnames, - "endpoints": endpoints, - "file_path": os.path.join(dirname, origin.name), - }, - ) - - executor.run(prog) - - def __save_distributed_lookup_tables( - executor, dirname, distributed_lookup_table, endpoints - ): - """ - because the distributed lookup table may too huge to merge and save at one place, - it will be saved at parameter server independent respectively. - - the save directory is dirname/"__lookup_table__". - - """ - prog = Program() - block = prog.global_block() - - # if there is lookup table, the trainer 0 will notify all pserver to save. - lookup_table_filename = os.path.join(dirname, "__lookup_table__") - attrs = {} - attrs['epmap'] = endpoints - attrs['dir'] = lookup_table_filename - attrs['lookup_table'] = distributed_lookup_table - block.append_op( - type='checkpoint_notify', inputs={}, outputs={}, attrs=attrs - ) - executor.run(prog) - - def __exclude_vars(exclude_var_names=[]): - def is_valid(var): - if var.name in exclude_var_names: - return False - if ( - var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH - or var.desc.type() == core.VarDesc.VarType.FETCH_LIST - or var.desc.type() == core.VarDesc.VarType.READER - ): - return False - return var.persistable - - return is_valid - - if not isinstance(main_program, Program): - raise TypeError("'main_program' should be an instance of Program.") - - if not main_program._is_distributed: - raise ValueError( - "'_save_distributed_persistables' just be designed for distributed training." - ) - - remote_params_map = ( - main_program._parameters_on_pservers.get_distributed_vars_by_vtypes( - ["Optimizer", "RemotePrefetch"], groupby=True - ) - ) - - exclude_var_names = [] - if remote_params_map: - exclude_var_names.extend(remote_params_map.keys()) - - if main_program._distributed_lookup_table: - if isinstance(main_program._distributed_lookup_table, list): - exclude_var_names.extend(main_program._distributed_lookup_table) - else: - exclude_var_names.append(main_program._distributed_lookup_table) - - local_vars = list( - filter(__exclude_vars(exclude_var_names), main_program.list_vars()) - ) - save_vars( - executor, main_program=main_program, dirname=dirname, vars=local_vars - ) - - if main_program._is_chief: - if remote_params_map: - __save_remote_params(executor, dirname, remote_params_map) - if main_program._distributed_lookup_table: - __save_distributed_lookup_tables( - executor, - dirname, - main_program._distributed_lookup_table, - main_program._endpoints, - ) - - -@dygraph_not_support -def save_persistables(executor, dirname, main_program=None, filename=None): - """ - Save all persistable variables from :code:`main_program` to - the folder :code:`dirname` or file :code:`filename`. You can refer to - :ref:`api_guide_model_save_reader_en` for more details. And then - saves these persistables variables to the folder :code:`dirname` or file - :code:`filename`. - - The :code:`dirname` is used to specify the folder where persistable variables - are going to be saved. If you would like to save variables in separate - files, set :code:`filename` None; if you would like to save all variables in a - single file, use :code:`filename` to specify the file name. - - Args: - executor(Executor): The executor to run for saving persistable variables. - You can refer to :ref:`api_guide_executor_en` for - more details. - - dirname(str, optional): The saving directory path. - When you need to save the parameter to the memory, set it to None. - main_program(Program, optional): The program whose persistbale variables will - be saved. You can refer to - :ref:`api_guide_Program_en` for more details. - If it is None, the default main program will - be used. - Default: None. - filename(str, optional): The file to save all variables. If you prefer to - save variables in different files, set it to None. - Default: None. - - Returns: - str: When saving parameters to a file, returns None. - When saving parameters to memory, returns a binary string containing parameters. - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - dir_path = "./my_paddle_model" - file_name = "persistables" - image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32') - label = fluid.data(name='label', shape=[None, 1], dtype='int64') - feeder = fluid.DataFeeder(feed_list=[image, label], place=fluid.CPUPlace()) - - predict = paddle.static.nn.fc(x=image, size=10, activation='softmax') - loss = paddle.nn.functional.cross_entropy( - input=predict, label=label, - reduction='none', use_softmax=False - ) - avg_loss = paddle.mean(loss) - exe = fluid.Executor(fluid.CPUPlace()) - exe.run(fluid.default_startup_program()) - fluid.io.save_persistables(executor=exe, dirname=dir_path, filename=file_name) - # The persistables variables weights and bias in the fc layer of the network - # are going to be saved in the same file named "persistables" in the path - # "./my_paddle_model" - """ - if main_program and main_program._is_distributed: - return _save_distributed_persistables( - executor, dirname=dirname, main_program=main_program - ) - else: - return save_vars( - executor, - dirname=dirname, - main_program=main_program, - vars=None, - predicate=is_persistable, - filename=filename, - ) - - -def load_vars( - executor, - dirname, - main_program=None, - vars=None, - predicate=None, - filename=None, -): - """ - :api_attr: Static Graph - - This API loads variables from files by executor. - - There are two ways to specify the variables to be loaded: the first way, set - variables in a list and assign it to the `vars`; the second way, use the - `predicate` function to select variables that make `predicate(variable) == True`. - The first way has a higher priority. - - The `dirname` is used to specify the folder where to load variables. - If variables were saved in separate files in the folder `dirname`, - set `filename` None. If all variables were saved in a single file, - use `filename` to specify it. - - Args: - executor(Executor): The executor to run for loading variables. - dirname(str): The folder where to load the variables. - main_program(Program, optional): The program whose variables will be loaded. - If it is None, the default main program will - be used automatically. - Default: None - vars(list[Variable], optional): The list that contains all variables to be loaded. - Default: None - predicate(function, optional): The function selects variables that make - `predicate(variable) == True`. - Default: None - filename(str, optional): The file which saved all required variables. If variables - were saved in separate files, set it to be None. - Default: None - - Returns: - None - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - main_prog = fluid.Program() - startup_prog = fluid.Program() - with fluid.program_guard(main_prog, startup_prog): - data = paddle.static.data(name="img", shape=[64, 784]) - w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w') - b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b') - hidden_w = paddle.matmul(x=data, y=w) - hidden_b = paddle.add(hidden_w, b) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(startup_prog) - - # The first usage: using `vars` to specify the variables. - path = "./my_paddle_vars" - var_list = [w, b] - fluid.io.save_vars(executor=exe, dirname=path, vars=var_list, - filename="vars_file") - fluid.io.load_vars(executor=exe, dirname=path, vars=var_list, - filename="vars_file") - # w and b will be loaded, and they are supposed to - # be saved in the same file named 'var_file' in the path "./my_paddle_vars". - - # The second usage: using the `predicate` function to select variables - param_path = "./my_paddle_model" - def name_has_fc(var): - res = "fc" in var.name - return res - fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog, - vars=None, predicate=name_has_fc) - fluid.io.load_vars(executor=exe, dirname=param_path, main_program=main_prog, - vars=None, predicate=name_has_fc) - # Load All variables in the `main_program` whose name includes "fc". - # And all the variables are supposed to be saved in separate files. - - """ - vars_from_memory = False - if dirname is not None: - dirname = os.path.normpath(dirname) - else: - vars_from_memory = True - - if vars is None: - if main_program is None: - main_program = default_main_program() - if not isinstance(main_program, Program): - raise TypeError( - "The type of input main_program is invalid, expected type is fluid.Program, but received %s" - % type(main_program) - ) - - load_vars( - executor, - dirname=dirname, - main_program=main_program, - vars=list(filter(predicate, main_program.list_vars())), - filename=filename, - ) - else: - load_prog = Program() - load_block = load_prog.global_block() - - if main_program is None: - main_program = default_main_program() - - if not isinstance(main_program, Program): - raise TypeError( - "The type of input main_program is invalid, expected type is fluid.Program, but received %s" - % type(main_program) - ) - - # save origin param shape - orig_para_shape = {} - load_var_map = {} - - check_vars = [] - sparse_vars = [] - - for each_var in vars: - assert isinstance(each_var, Variable) - - if each_var.type == core.VarDesc.VarType.RAW: - continue - - if isinstance(each_var, Parameter): - orig_para_shape[each_var.name] = tuple( - each_var.desc.get_shape() - ) - - if each_var.type == core.VarDesc.VarType.SELECTED_ROWS: - sparse_vars.append(each_var) - continue - - new_var = _clone_var_in_block_(load_block, each_var) - check_vars.append(each_var) - - if filename is None: - if dirname is None: - raise ValueError( - "The directory path and params cannot be None at the same time." - ) - load_block.append_op( - type='load', - inputs={}, - outputs={'Out': [new_var]}, - attrs={'file_path': os.path.join(dirname, new_var.name)}, - ) - else: - load_var_map[new_var.name] = new_var - - for each_var in sparse_vars: - assert isinstance(each_var, Variable) - - if filename is not None: - raise ValueError( - "SelectedRows can not be load with load_combine" - ) - - new_var = _clone_var_in_block_(load_block, each_var) - - var_path = os.path.join(dirname, new_var.name) - if not os.path.exists(var_path): - raise ValueError( - "SelectedRows var {} can not find at {}".format( - new_var.name, var_path - ) - ) - - if os.path.isfile(var_path): - load_block.append_op( - type='load', - inputs={}, - outputs={'Out': [new_var]}, - attrs={'file_path': os.path.join(dirname, new_var.name)}, - ) - else: - blocks = [] - block_paths = os.listdir(var_path) - - for block in block_paths: - if block.startswith(new_var.name): - blocks.append(block) - - slices = [] - for block in blocks: - slice = load_block.create_var( - name=block, - type=new_var.type, - shape=new_var.shape, - dtype=new_var.dtype, - persistable=False, - ) - slices.append(slice) - - file_path = os.path.join(var_path, block, "Param") - load_block.append_op( - type='load', - inputs={}, - outputs={'Out': [slice]}, - attrs={'file_path': file_path}, - ) - - load_block.append_op( - type='lookup_sparse_table_merge', - inputs={'X': slices}, - outputs={'Out': new_var}, - attrs={}, - ) - - if filename is not None: - load_var_list = [] - for name in sorted(load_var_map.keys()): - load_var_list.append(load_var_map[name]) - - if vars_from_memory is False: - filename = os.path.join(dirname, filename) - - load_block.append_op( - type='load_combine', - inputs={}, - outputs={"Out": load_var_list}, - attrs={ - 'file_path': filename, - 'model_from_memory': vars_from_memory, - }, - ) - executor.run(load_prog) - - # check var shape - for each_var in check_vars: - if not isinstance(each_var, Parameter): - continue - var_temp = paddle.fluid.global_scope().find_var(each_var.name) - assert var_temp is not None, "can't not find var: " + each_var.name - new_shape = (np.array(var_temp.get_tensor())).shape - assert each_var.name in orig_para_shape, ( - each_var.name + "MUST in var list" - ) - orig_shape = orig_para_shape.get(each_var.name) - if new_shape != orig_shape: - raise RuntimeError( - "Variable's shape does not match, the Program requires a parameter with the shape of ({}), " - "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( - orig_shape, each_var.name, new_shape - ) - ) - - -@dygraph_not_support -def load_params(executor, dirname, main_program=None, filename=None): - """ - :api_attr: Static Graph - - This API filters out all parameters from the give ``main_program`` - and then tries to load these parameters from the directory ``dirname`` or - the file ``filename``. - - Use the ``dirname`` to specify the directory where parameters were saved. If - parameters were saved in separate files under the directory `dirname`, set - ``filename`` as None; if all parameters were saved in a single file, use - ``filename`` to specify the file name. - - **Note**: - Some variables are not Parameter while they are necessary for - training, such as learning rate, global step, etc. So you cannot save and - continue your training just by using :ref:`api_fluid_io_save_params` and - :ref:`api_fluid_io_load_params`. Please use :ref:`api_fluid_io_save_persistables` - and :ref:`api_fluid_io_load_persistables` instead. - - If you want to load the pre-trained model structure and parameters - for the inference, please use the :ref:`api_fluid_io_load_inference_model` API. You can - refer to :ref:`api_guide_model_save_reader_en` for more details. - - Args: - executor(Executor): The executor used for loading parameters. - See :ref:`api_guide_executor_en` for more details about it. - dirname(str): The directory path. - main_program(Program, optional): The program whose parameters will be - loaded. If it is None, the ``default_main_program`` - will be used automatically. See :ref:`api_guide_Program_en` - for more about ``Program``. - Default: None. - filename(str, optional): The file which saved all parameters. If parameters - were saved in separated files, set it to None. - Default: None. - - Returns: - None - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - exe = fluid.Executor(fluid.CPUPlace()) - param_path = "./my_paddle_model" - prog = fluid.default_main_program() - fluid.io.load_params(executor=exe, dirname=param_path, - main_program=None) - """ - load_vars( - executor, - dirname=dirname, - main_program=main_program, - predicate=is_parameter, - filename=filename, - ) - - -@dygraph_not_support -def load_persistables(executor, dirname, main_program=None, filename=None): - """ - :api_attr: Static Graph - - This API filters out all variables with ``persistable==True`` from the - given ``main_program`` and then tries to load these variables from the - directory ``dirname`` or the file ``filename``. - - Use the ``dirname`` to specify the directory where persistable variables - (refer to :ref:`api_guide_model_save_reader_en`) were saved. If variables - were saved in separate files, set ``filename`` as None; if all variables - were saved in a single file, use ``filename`` to specify the file name. - - Args: - executor(Executor): The executor used for loading persistable variables. - See :ref:`api_guide_executor_en` for more details about it. - dirname(str): The directory path. - main_program(Program, optional): The program whose persistable variables will - be loaded. If it is None, the ``default_main_program`` - will be used automatically. See :ref:`api_guide_Program_en` - for more about ``Program``. - Default: None. - filename(str, optional): The file which saved all persistable variables. If variables - were saved in separated files, set it to None. - Default: None. - - Returns: - None - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - exe = fluid.Executor(fluid.CPUPlace()) - param_path = "./my_paddle_model" - prog = fluid.default_main_program() - fluid.io.load_persistables(executor=exe, dirname=param_path, - main_program=None) - """ - - if main_program and main_program._is_distributed: - _load_distributed_persistables( - executor, dirname=dirname, main_program=main_program - ) - else: - load_vars( - executor, - dirname=dirname, - main_program=main_program, - predicate=is_persistable, - filename=filename, - ) - - -def _load_distributed_persistables(executor, dirname, main_program=None): - """ - customized load_persistables for distributed training. - it should be used on parameter server, - - Args: - executor(Executor): The executor to run for saving parameters. - dirname(str): The load directory path. - main_program(Program): The program whose parameters will be - loaded. the main_program must be the pserver_program - get after transpiler. - - Returns: - None - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - exe = fluid.Executor(fluid.CPUPlace()) - param_path = "./my_paddle_model" - t = paddle.distributed.transpiler.DistributeTranspiler() - t.transpile(...) - pserver_prog = t.get_pserver_program(...) - _load_distributed_persistables(executor=exe, dirname=param_path, main_program=pserver_prog) - """ - - def __is_distributed_part_var(varname): - trainer_idx = varname.find(".trainer_") - block_idx = varname.find(".block") - return trainer_idx or block_idx - - def __load_persistable_vars(executor, dirname, need_load_vars): - load_prog = Program() - load_block = load_prog.global_block() - need_delete_vars = [] - - for param in need_load_vars: - origin_var = param.origin - slice_var = param.slice - is_slice = param.is_slice - offset = param.offset - - if is_slice: - slice = load_block.create_var( - name=slice_var.name, - type=slice_var.type, - shape=slice_var.shape, - dtype=slice_var.dtype, - persistable=True, - ) - - load_block.append_op( - type='load', - inputs={}, - outputs={'Out': [slice]}, - attrs={ - 'file_path': os.path.join(dirname, origin_var.name), - 'seek': offset, - 'shape': slice.shape, - }, - ) - else: - origin = load_block.create_var( - name="{}".format(origin_var.name), - type=origin_var.type, - shape=origin_var.shape, - dtype=origin_var.dtype, - persistable=True, - ) - load_block.append_op( - type='load', - inputs={}, - outputs={'Out': [origin]}, - attrs={'file_path': os.path.join(dirname, origin_var.name)}, - ) - - load_block.append_op( - type='delete_var', - inputs={'X': need_delete_vars}, - ) - - executor.run(load_prog) - - if not isinstance(main_program, Program): - raise TypeError("'main_program' should be an instance of Program.") - - if not main_program._is_distributed: - raise ValueError( - "'_load_distributed_persistables' just be designed for distributed training." - ) +__all__ = [ + 'save_inference_model', + 'load_inference_model', +] + reader.__all__ - if not main_program._ps_endpoint: - raise ValueError( - "'_load_distributed_persistables' need current_endpoint set in DistributeTranspiler.transpile" - ) - need_load_vars = ( - main_program._parameters_on_pservers.get_distributed_vars_by_ep( - main_program._ps_endpoint - ) - ) - __load_persistable_vars(executor, dirname, need_load_vars) +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def prepend_feed_ops( @@ -1430,7 +240,7 @@ def save_inference_model( ): raise ValueError("'target_vars' should be a list of Variable.") - main_program = _get_valid_program(main_program) + main_program = paddle.static.io._get_valid_program(main_program) # remind user to set auc_states to zeros if the program contains auc op all_ops = main_program.global_block().ops @@ -1534,7 +344,9 @@ def save_inference_model( if params_filename is not None: params_filename = os.path.basename(params_filename) - save_persistables(executor, save_dirname, main_program, params_filename) + paddle.distributed.io.save_persistables( + executor, save_dirname, main_program, params_filename + ) return target_var_name_list @@ -1671,7 +483,9 @@ def load_inference_model( "Unsupported program version: %d\n" % program._version() ) # Binary data also need versioning. - load_persistables(executor, load_dirname, program, params_filename) + paddle.distributed.io.load_persistables( + executor, load_dirname, program, params_filename + ) if pserver_endpoints: program = _endpoints_replacement(program, pserver_endpoints) @@ -1692,859 +506,3 @@ def _endpoints_replacement(program, endpoints): op.set_attr(ENDPOINT_MAP, endpoints) program._sync_with_cpp() return program - - -def get_parameter_value(para, executor): - """ - Get the LoDTensor value of the given parameter. - - Args: - para(Parameter): The parameter to get value from. - executor(Executor): The executor to run for retrieving the value. - - Returns: - numpy.array: The given parameter's values. - - Raises: - AssertionError: If the `para` is not an instance of Parameter. - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - exe = fluid.Executor(fluid.CPUPlace()) - param = fluid.default_main_program().global_block().var('fc.w') - p = fluid.io.get_parameter_value(param, exe) - - """ - assert is_parameter(para), "The input variable is not parameter." - - get_program = Program() - block = get_program.global_block() - new_var = _clone_var_in_block_(block, para) - return executor.run(get_program, feed={}, fetch_list=[new_var])[0] - - -def get_parameter_value_by_name(name, executor, program=None): - """ - Get the LoDTensor value of a certain parameter by its name. - - Args: - name(str): The parameter's name. - executor(Executor): The executor to run for retrieving the value. - program(Program | None): The program where to find the parameter. - If it's set to be None, the function will - try to find the parameter in the default - main program. - - Returns: - numpy.array: The parameter's values. - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - exe = fluid.Executor(fluid.CPUPlace()) - p = fluid.io.get_parameter_value('fc.w', exe) - """ - if program is None: - program = default_main_program() - var = program.global_block().var(name) - return get_parameter_value(var, executor) - - -def _save_persistable_nodes(executor, dirname, graph): - """ - Save persistable nodes to the given directory by the executor. - - Args: - executor(Executor): The executor to run for saving node values. - dirname(str): The directory path. - graph(IrGraph): All the required persistable nodes in the graph will be saved. - """ - persistable_node_names = set() - persistable_nodes = [] - all_persistable_nodes = graph.all_persistable_nodes() - for node in all_persistable_nodes: - name = node.name() - if name not in persistable_node_names: - persistable_node_names.add(name) - persistable_nodes.append(node) - program = Program() - var_list = [] - for node in persistable_nodes: - var_desc = node.var() - if ( - var_desc.type() == core.VarDesc.VarType.RAW - or var_desc.type() == core.VarDesc.VarType.READER - ): - continue - var = program.global_block().create_var( - name=var_desc.name(), - shape=var_desc.shape(), - dtype=var_desc.dtype(), - type=var_desc.type(), - lod_level=var_desc.lod_level(), - persistable=var_desc.persistable(), - ) - var_list.append(var) - save_vars(executor=executor, dirname=dirname, vars=var_list) - - -def _load_persistable_nodes(executor, dirname, graph): - """ - Load persistable node values from the given directory by the executor. - - Args: - executor(Executor): The executor to run for loading node values. - dirname(str): The directory path. - graph(IrGraph): All the required persistable nodes in the graph will be loaded. - """ - persistable_node_names = set() - persistable_nodes = [] - all_persistable_nodes = graph.all_persistable_nodes() - for node in all_persistable_nodes: - name = node.name() - if name not in persistable_node_names: - persistable_node_names.add(name) - persistable_nodes.append(node) - program = Program() - var_list = [] - - def _exist(var): - return os.path.exists(os.path.join(dirname, var.name)) - - for node in persistable_nodes: - var_desc = node.var() - if ( - var_desc.type() == core.VarDesc.VarType.RAW - or var_desc.type() == core.VarDesc.VarType.READER - ): - continue - var = program.global_block().create_var( - name=var_desc.name(), - shape=var_desc.shape(), - dtype=var_desc.dtype(), - type=var_desc.type(), - lod_level=var_desc.lod_level(), - persistable=var_desc.persistable(), - ) - if _exist(var): - var_list.append(var) - else: - _logger.warn("Cannot find the var %s!!!" % (node.name())) - load_vars(executor=executor, dirname=dirname, vars=var_list) - - -def _unpack_saved_dict(saved_obj, protocol): - temp_saved_obj = {} - unpack_infor = {} - # When pickle protocol=2 or protocol=3 the serialized object cannot be larger than 4G. - if 1 < protocol < 4: - if isinstance(saved_obj, dict): - for key, value in saved_obj.items(): - if isinstance(value, np.ndarray): - MAX_NUMBER_OF_ELEMENT = int( - (2**30 - 1) / value.dtype.itemsize - ) - num_element = np.prod(value.shape) - if num_element > MAX_NUMBER_OF_ELEMENT: - unpack_infor[key] = {} - unpack_infor[key]["OriginShape"] = value.shape - unpack_infor[key]["slices"] = [] - value = value.flatten() - for i in range( - int( - math.ceil( - num_element * 1.0 / MAX_NUMBER_OF_ELEMENT - ) - ) - ): - part_name = key + "@@." + str(i) - unpack_infor[key]["slices"].append(part_name) - temp_saved_obj[part_name] = value[ - i - * MAX_NUMBER_OF_ELEMENT : MAX_NUMBER_OF_ELEMENT - * (i + 1) - ] - - if unpack_infor: - for key, value in unpack_infor.items(): - if key in saved_obj: - saved_obj.pop(key) - for part in value['slices']: - saved_obj[part] = temp_saved_obj[part] - saved_obj['UnpackBigParamInfor@@'] = unpack_infor - return saved_obj - - -def _pack_loaded_dict(load_obj): - if isinstance(load_obj, dict): - unpack_info = 'UnpackBigParamInfor@@' - if unpack_info in load_obj: - removes = [] - for key, value in load_obj[unpack_info].items(): - slices = [load_obj[part] for part in value["slices"]] - load_obj[key] = np.concatenate(slices).reshape( - value["OriginShape"] - ) - removes += value["slices"] - for key in removes: - load_obj.pop(key) - load_obj.pop(unpack_info) - - return load_obj - - -@static_only -def _legacy_save(param_dict, model_path, protocol=2): - def get_tensor(var): - if isinstance(var, (core.VarBase, core.eager.Tensor)): - return var.numpy() - elif isinstance(var, core.LoDTensor): - return np.array(var) - return var - - param_dict = {name: get_tensor(param_dict[name]) for name in param_dict} - - # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' - if ( - _is_file_path(model_path) - and sys.platform == 'darwin' - and sys.version_info.major == 3 - ): - pickle_bytes = pickle.dumps(param_dict, protocol=protocol) - with open(model_path, 'wb') as f: - max_bytes = 2**30 - for i in range(0, len(pickle_bytes), max_bytes): - f.write(pickle_bytes[i : i + max_bytes]) - else: - with _open_file_buffer(model_path, 'wb') as f: - pickle.dump(param_dict, f, protocol=protocol) - - -@static_only -def save(program, model_path, protocol=4, **configs): - """ - - This function save parameters, optimizer information and network description to model_path. - - The parameters contains all the trainable Tensor, will save to a file with suffix ".pdparams". - The optimizer information contains all the Tensor used by optimizer. For Adam optimizer, contains beta1, beta2, momentum etc. All the information will save to a file with suffix ".pdopt". (If the optimizer have no Tensor need to save (like SGD), the fill will not generated). - The network description is the description of the program. It's only used for deployment. The description will save to a file with a suffix ".pdmodel". - - Args: - program(Program) : The program to saved. - model_path(str): the file prefix to save the program. The format is "dirname/file_prefix". If file_prefix is empty str. A exception will be raised - protocol(int, optional): The protocol version of pickle module must be greater than 1 and less than 5. - Default: 4 - configs(dict, optional) : optional keyword arguments. - - Returns: - None - - Examples: - .. code-block:: python - - import paddle - import paddle.static as static - - paddle.enable_static() - - x = static.data(name="x", shape=[10, 10], dtype='float32') - y = static.nn.fc(x, 10) - z = static.nn.fc(y, 10) - - place = paddle.CPUPlace() - exe = static.Executor(place) - exe.run(static.default_startup_program()) - prog = static.default_main_program() - - static.save(prog, "./temp") - """ - - base_name = os.path.basename(model_path) - assert ( - base_name != "" - ), "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received model_path is empty string." - if 'pickle_protocol' in configs: - protocol = configs['pickle_protocol'] - warnings.warn( - "'pickle_protocol' is a deprecated argument. Please use 'protocol' instead." - ) - - if not isinstance(protocol, int): - raise ValueError( - "The 'protocol' MUST be `int`, but received {}".format( - type(protocol) - ) - ) - - if protocol < 2 or protocol > 4: - raise ValueError( - "Expected 1<'protocol'<5, but received protocol={}".format(protocol) - ) - - dir_name = os.path.dirname(model_path) - if dir_name and not os.path.exists(dir_name): - os.makedirs(dir_name) - - def get_tensor(var): - t = global_scope().find_var(var.name).get_tensor() - return np.array(t) - - parameter_list = list(filter(is_parameter, program.list_vars())) - param_dict = {p.name: get_tensor(p) for p in parameter_list} - - param_dict = _unpack_saved_dict(param_dict, protocol) - - # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' - if sys.platform == 'darwin' and sys.version_info.major == 3: - pickle_bytes = pickle.dumps(param_dict, protocol=protocol) - with open(model_path + ".pdparams", 'wb') as f: - max_bytes = 2**30 - for i in range(0, len(pickle_bytes), max_bytes): - f.write(pickle_bytes[i : i + max_bytes]) - else: - with open(model_path + ".pdparams", 'wb') as f: - pickle.dump(param_dict, f, protocol=protocol) - - optimizer_var_list = list( - filter(is_belong_to_optimizer, program.list_vars()) - ) - - opt_dict = {p.name: get_tensor(p) for p in optimizer_var_list} - with open(model_path + ".pdopt", 'wb') as f: - pickle.dump(opt_dict, f, protocol=protocol) - - main_program = program.clone() - program.desc.flush() - main_program.desc._set_version() - paddle.fluid.core.save_op_version_info(program.desc) - - with open(model_path + ".pdmodel", "wb") as f: - f.write(program.desc.serialize_to_string()) - - -def _pickle_loads_mac(path, f): - pickle_bytes = bytearray(0) - file_size = os.path.getsize(path) - max_bytes = 2**30 - for _ in range(0, file_size, max_bytes): - pickle_bytes += f.read(max_bytes) - load_result = pickle.loads(pickle_bytes, encoding='latin1') - return load_result - - -@static_only -def load(program, model_path, executor=None, var_list=None): - """ - :api_attr: Static Graph - - This function get parameters and optimizer information from program, and then get corresponding value from file. - An exception will throw if shape or dtype of the parameters is not match. - - This function can also load model file saved with [ save_params, save_persistables, save_vars ]. - var_list can not be None when load single model file - ( filename is not None When save_params, save_persistables or save_vars is called ). - - Args: - program(Program): The program will be loaded - model_path(str): The file prefix store the program - executor(Executor, optional): The executor used for initialize the parameter - When startup program is not run. - var_list(list|tuple, optional): The Tensor list/tuple to load single model file saved with - [ save_params, save_persistables, save_vars ]. - Default: None - - Returns: - None - - Examples: - .. code-block:: python - - import paddle - import paddle.static as static - - paddle.enable_static() - - x = static.data(name="x", shape=[10, 10], dtype='float32') - y = static.nn.fc(x, 10) - z = static.nn.fc(y, 10) - - place = paddle.CPUPlace() - exe = static.Executor(place) - exe.run(static.default_startup_program()) - prog = static.default_main_program() - - static.save(prog, "./temp") - static.load(prog, "./temp") - """ - - assert executor is None or isinstance(executor, Executor) - - model_prefix = model_path - if model_prefix.endswith(".pdparams"): - model_prefix = model_prefix[:-9] - elif model_prefix.endswith(".pdopt"): - model_prefix = model_prefix[:-6] - elif model_prefix.endswith(".pdmodel"): - model_prefix = model_prefix[:-8] - - parameter_file_name = model_prefix + ".pdparams" - - if not os.path.exists(parameter_file_name): - # model file save by fluid.save not found, try to load model file saved with - # [save_vars, save_params, save_persistables] - _logger.debug( - "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format( - parameter_file_name - ) - ) - if executor is None: - raise ValueError( - "executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]" - ) - - if var_list is not None: - var_list_names = [var.name for var in var_list] - else: - var_list_names = None - - if os.path.isdir(model_path): - binary_file_set = set() - for root, dirs, files in os.walk(model_path, topdown=False): - for f in files: - binary_file_set.add( - os.path.join(root, f).replace("\\", "/") - ) - program_var_list = list(program.list_vars()) - loaded_var_list = [] - for var in program_var_list: - var_path = os.path.join(model_path, var.name).replace("\\", "/") - load_condition = ( - var_list_names is None or var.name in var_list_names - ) - if var_path in binary_file_set and load_condition: - loaded_var_list.append(var) - binary_file_set.remove(var_path) - if len(binary_file_set) > 0: - unused_var_list = " ".join(list(binary_file_set)) - _logger.warning( - "variable file [ %s ] not used" - % (" ".join(list(binary_file_set))) - ) - try: - load_vars( - executor=executor, dirname=model_path, vars=loaded_var_list - ) - except RuntimeError as e: - _logger.error(e) - raise e - except: - raise RuntimeError( - "Failed to load model file, please make sure model file is saved with the " - "following APIs: save_params, save_persistables, save_vars" - ) - - return - elif os.path.isfile(model_path): - if var_list is None: - raise ValueError( - "var_list is required when loading model file saved with [ save_params, save_persistables, save_vars ]" - ) - program_var_list = program.list_vars() - program_var_name_set = set([var.name for var in program_var_list]) - - # check all the variable inlcuded in program - for var in var_list: - if var.name not in program_var_name_set: - raise LookupError( - "loaded var [{}] is not in program variable list" - ) - - dir_name, file_name = os.path.split(model_path) - try: - load_vars( - executor=executor, - dirname=dir_name, - vars=var_list, - filename=file_name, - ) - except RuntimeError as e: - _logger.error(e) - raise e - except: - raise RuntimeError( - "Failed to load model file , please make sure model file is saved with the " - "the following APIs: [ save_params, save_persistables, save_vars ]. " - "When these API called, filename CANNOT be None" - ) - - return - - def set_var(var, ndarray): - t = global_scope().find_var(var.name).get_tensor() - p = t._place() - if p.is_cpu_place(): - place = paddle.fluid.CPUPlace() - elif p.is_cuda_pinned_place(): - place = paddle.fluid.CUDAPinnedPlace() - elif p.is_xpu_place(): - p = paddle.fluid.core.Place() - p.set_place(t._place()) - place = paddle.fluid.XPUPlace(p.xpu_device_id()) - elif p.is_npu_place(): - p = paddle.fluid.core.Place() - p.set_place(t._place()) - place = paddle.fluid.NPUPlace(p.npu_device_id()) - elif p.is_mlu_place(): - p = paddle.fluid.core.Place() - p.set_place(t._place()) - place = paddle.fluid.MLUPlace(p.mlu_device_id()) - else: - p = paddle.fluid.core.Place() - p.set_place(t._place()) - place = paddle.fluid.CUDAPlace(p.gpu_device_id()) - - t.set(ndarray, place) - - parameter_list = list(filter(is_parameter, program.list_vars())) - - if executor: - paddle.fluid.core._create_loaded_parameter( - parameter_list, global_scope(), executor._default_executor - ) - with open(parameter_file_name, 'rb') as f: - - # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' - if sys.platform == 'darwin' and sys.version_info.major == 3: - load_dict = _pickle_loads_mac(parameter_file_name, f) - else: - load_dict = pickle.load(f, encoding='latin1') - load_dict = _pack_loaded_dict(load_dict) - for v in parameter_list: - assert ( - v.name in load_dict - ), "Can not find [{}] in model file [{}]".format( - v.name, parameter_file_name - ) - set_var(v, load_dict[v.name]) - - optimizer_var_list = list( - filter(is_belong_to_optimizer, program.list_vars()) - ) - - if len(optimizer_var_list) > 0: - opt_file_name = model_prefix + ".pdopt" - assert os.path.exists( - opt_file_name - ), "Optimizer file [{}] not exits".format(opt_file_name) - - if executor: - paddle.fluid.core._create_loaded_parameter( - optimizer_var_list, global_scope(), executor._default_executor - ) - - with open(opt_file_name, 'rb') as f: - load_dict = pickle.load(f, encoding='latin1') - for v in optimizer_var_list: - assert ( - v.name in load_dict - ), "Can not find [{}] in model file [{}]".format( - v.name, opt_file_name - ) - set_var(v, load_dict[v.name]) - - -def load_program_state(model_path, var_list=None): - """ - - Load program state from local file - - Args: - model_path(str): The file prefix store the program - var_list(list|tuple, optional): The Tensor list/tuple to load saved with - [ save_params, save_persistables, save_vars ]. - Default: None. - The var_list is only used to get name, - will not be modified. - Returns: - state_dict(dict): the dict store Parameter and optimizer information - - Examples: - - .. code-block:: python - - import paddle - import paddle.static as static - - paddle.enable_static() - - x = static.data(name="x", shape=[10, 10], dtype='float32') - y = static.nn.fc(x, 10) - z = static.nn.fc(y, 10) - - place = paddle.CPUPlace() - exe = static.Executor(place) - exe.run(static.default_startup_program()) - prog = static.default_main_program() - - static.save(prog, "./temp") - program_state = static.load_program_state("./temp") - """ - model_prefix = model_path - if model_prefix.endswith(".pdparams"): - model_prefix = model_prefix[:-9] - elif model_prefix.endswith(".pdopt"): - model_prefix = model_prefix[:-6] - elif model_prefix.endswith(".pdmodel"): - model_prefix = model_prefix[:-8] - - parameter_file_name = model_prefix + ".pdparams" - if not os.path.exists(parameter_file_name): - # model file saved with fluid.save is not found, try to load model file saved with - # [save_vars, save_params, save_persistables] - _logger.debug( - "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format( - parameter_file_name - ) - ) - - var_name_list = [] - if var_list is None and os.path.isfile(model_path): - raise ValueError( - "var_list can not be None when model_path is a file type" - ) - - for root, dirs, files in os.walk(model_path, topdown=False): - for f in files: - file_path = os.path.join(root, f) - var_temp_name = os.path.relpath(file_path, model_path) - var_temp_name = var_temp_name.replace("\\", "/") - var_name_list.append(var_temp_name) - - with _load_program_scope(): - load_prog = Program() - load_block = load_prog.global_block() - - def clone_var_to_block(block, var): - if not isinstance(var, Variable): - raise TypeError("value in var_list must be variable") - return block.create_var( - name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level - if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR - else None, - persistable=True, - ) - - def _load_vars_with_try_catch( - exe, dirname, vars, filename, raise_error=True - ): - try: - load_vars( - executor=exe, - dirname=dirname, - vars=vars, - filename=filename, - ) - return True - except: - error_str = ( - "Failed to load model/variables `%s`, please make sure " - "model/variables file is saved with the following APIs: " - "save_params, save_persistables, save_vars." - ) - filenames = ( - [var.name for var in vars] - if filename is None - else filename - ) - if raise_error: - raise RuntimeError(error_str % filenames) - else: - warnings.warn(error_str % filenames, RuntimeWarning) - return False - - place = paddle.fluid.CPUPlace() - exe = paddle.fluid.Executor(place) - - loaded_var_list = [] - - if os.path.isfile(model_path): - # when model_path is file, var_list cannot be None - dir_name, file_name = os.path.split(model_path) - for var in var_list: - loaded_var_list.append(clone_var_to_block(load_block, var)) - _load_vars_with_try_catch( - exe, dir_name, loaded_var_list, file_name - ) - else: - # var_list can be None or not None - if var_list is not None: - for var in var_list: - loaded_var_list.append( - clone_var_to_block(load_block, var) - ) - _load_vars_with_try_catch( - exe, model_path, loaded_var_list, None - ) - else: - for var_name in var_name_list: - # NOTE(chenweihang): If identify which files the user wants - # to load from the disk, we load these variables one by one. - # If a file does not exist, we only warn the user that the - # file may be an irrelevant file, but does not throw an error - # to ensure that other legal variables can be loaded. - temp_var = load_block.create_var( - name=var_name, persistable=True - ) - if _load_vars_with_try_catch( - exe, model_path, [temp_var], None, False - ): - loaded_var_list.append(temp_var) - - res_dict = {} - for var in loaded_var_list: - res_dict[var.name] = np.asarray( - paddle.fluid.global_scope().find_var(var.name).get_tensor() - ) - - return res_dict - - assert os.path.exists( - parameter_file_name - ), "Parameter file [{}] not exits".format(parameter_file_name) - - with open(parameter_file_name, 'rb') as f: - # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' - if sys.platform == 'darwin' and sys.version_info.major == 3: - para_dict = _pickle_loads_mac(parameter_file_name, f) - else: - para_dict = pickle.load(f, encoding='latin1') - para_dict = _pack_loaded_dict(para_dict) - - opt_file_name = model_prefix + ".pdopt" - if os.path.exists(opt_file_name): - with open(opt_file_name, 'rb') as f: - opti_dict = pickle.load(f, encoding='latin1') - - para_dict.update(opti_dict) - - return para_dict - - -@static_only -def set_program_state(program, state_dict): - """ - Set program parameter from state_dict - - An exception will throw if shape or dtype of the parameters is not match. - - NOTICE: This function MUST called after run start_up_program - - Args: - program(Program): The program to be set - state_dict(dict): the dict store Parameter and optimizer information - Returns: - None - - Examples: - .. code-block:: python - - import paddle - import paddle.static as static - - paddle.enable_static() - - x = static.data(name="x", shape=[10, 10], dtype='float32') - y = static.nn.fc(x, 10) - z = static.nn.fc(y, 10) - - place = paddle.CPUPlace() - exe = static.Executor(place) - exe.run(static.default_startup_program()) - prog = static.default_main_program() - - static.save(prog, "./temp") - program_state = static.load_program_state("./temp") - - static.set_program_state(prog, program_state) - """ - state_dict = _pack_loaded_dict(state_dict) - parameter_list = list(filter(is_persistable, program.list_vars())) - - used_para_list = {} - for para in parameter_list: - var_temp = paddle.fluid.global_scope().find_var(para.name) - assert ( - var_temp is not None - ), "Variable [ {} ] Not found, Please make sure run startup program".format( - para.name - ) - if para.name in state_dict: - # set value from state dict - orig_para_np = np.array(var_temp.get_tensor()) - new_para_np = state_dict[para.name] - assert orig_para_np.shape == new_para_np.shape, ( - "Parameter's shape does not match, the Program requires a parameter with the shape of ({}), " - "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( - orig_para_np.shape, para.name, new_para_np.shape - ) - ) - assert orig_para_np.dtype == new_para_np.dtype, ( - "Parameter's data type does not match, the Program requires a parameter with a dtype of ({}), " - "while the loaded parameter (namely [ {} ]) has a dtype of ({}).".format( - orig_para_np.dtype, para.name, new_para_np.dtype - ) - ) - - ten = var_temp.get_tensor() - ten_place = ten._place() - - # assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \ - # "Place not support, only support CPUPlace and GPUPlace, now is {}".format(str(ten_place)) - py_place = paddle.fluid.CPUPlace() - if ten_place.is_cuda_pinned_place(): - place = paddle.fluid.CUDAPinnedPlace() - elif ten_place.is_gpu_place(): - p = paddle.fluid.core.Place() - p.set_place(ten_place) - py_place = paddle.fluid.CUDAPlace(p.gpu_device_id()) - elif ten_place.is_xpu_place(): - p = paddle.fluid.core.Place() - p.set_place(ten_place) - py_place = paddle.fluid.XPUPlace(p.xpu_device_id()) - elif ten_place.is_npu_place(): - p = paddle.fluid.core.Place() - p.set_place(ten_place) - py_place = paddle.fluid.NPUPlace(p.npu_device_id()) - elif ten_place.is_mlu_place(): - p = paddle.fluid.core.Place() - p.set_place(ten_place) - py_place = paddle.fluid.MLUPlace(p.mlu_device_id()) - - ten.set(new_para_np, py_place) - - used_para_list[para.name] = 1 - - unused_para_list = [] - for k, v in state_dict.items(): - if k not in used_para_list: - unused_para_list.append(k) - if len(unused_para_list) > 0: - warnings.warn( - "This list is not set, Because of Paramerter not found in program. There are: {}".format( - " ".join(unused_para_list) - ) - ) diff --git a/python/paddle/fluid/tests/unittests/dist_save_load.py b/python/paddle/fluid/tests/unittests/dist_save_load.py index 0c7072c27da3e50287a59e43f0ec1d86c595064f..faff1ea1e78cba479aa8d2d3ba99b28176e98b9c 100644 --- a/python/paddle/fluid/tests/unittests/dist_save_load.py +++ b/python/paddle/fluid/tests/unittests/dist_save_load.py @@ -22,7 +22,7 @@ from test_dist_base import RUN_STEP, runtime_main import paddle import paddle.fluid as fluid -from paddle.fluid import core, io +from paddle.fluid import core class TestDistSaveLoad2x2(TestDistSimnetBow2x2): @@ -56,7 +56,7 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2): return var.persistable - io.load_vars( + paddle.static.io.load_vars( executor, dirname=dirname, main_program=program, @@ -89,7 +89,9 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2): exe.run(startup_prog) if need_load and model_dir: - fluid.io.load_persistables(exe, model_dir, pserver_prog) + paddle.distributed.io.load_persistables( + exe, model_dir, pserver_prog + ) exe.run(pserver_prog) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py index 0efc5445b6a0ecd91d842099eef169c782e9f66e..b90c5599bb015c040ba1819a2d24587ca6c148e3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py @@ -172,7 +172,7 @@ def train_static(args, batch_generator): model_path = os.path.join( args.save_static_model_path, "transformer" ) - fluid.save(train_prog, model_path) + paddle.static.save(train_prog, model_path) break return np.array(avg_loss) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py index 680d8afd4860ff61e924722e4bc73a303bb42f65..bfc485b2404eef66f40fef18aceb57af14497c26 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py @@ -20,7 +20,6 @@ import numpy as np import paddle import paddle.dataset.wmt16 as wmt16 -import paddle.fluid as fluid def get_input_descs(args, mode="train"): @@ -310,7 +309,7 @@ def load(program, model_path, executor=None, var_list=None): To load python2 saved models in python3. """ try: - fluid.load(program, model_path, executor, var_list) + paddle.static.load(program, model_path, executor, var_list) except UnicodeDecodeError: warnings.warn( "An UnicodeDecodeError is catched, which might be caused by loading " @@ -319,7 +318,7 @@ def load(program, model_path, executor=None, var_list=None): ) load_bak = pickle.load pickle.load = partial(load_bak, encoding="latin1") - fluid.load(program, model_path, executor, var_list) + paddle.static.load(program, model_path, executor, var_list) pickle.load = load_bak diff --git a/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py b/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py index a3d6c0cbfd992bb89275203f9c07c970e34172d2..bb863ad73e40a2cd632b5e5ac1ca9bc87371e005 100644 --- a/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py +++ b/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py @@ -42,7 +42,7 @@ def main_test_func(place, dtype): adam_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = fluid.io.batch( + train_reader = paddle.batch( paddle.dataset.uci_housing.train(), batch_size=1 ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) diff --git a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py index 53ef9b02ccb105c8fc63031731fcd168fde200b1..9fb7d8d31b4612a6b6de66f486473fd33bd3f5e7 100644 --- a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py +++ b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py @@ -40,7 +40,7 @@ class TestClass(unittest.TestCase): yield img, label reader = paddle.reader.cache(fake_reader) - batch_reader = fluid.io.batch(reader, batch_size=batch_size) + batch_reader = paddle.batch(reader, batch_size=batch_size) places = [fluid.CPUPlace()] if fluid.core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py index 8039e80e56eded696aef5d98d1ab2cc0bcad042e..8d86a74dbd4d9b876a833ab6378037ce3b811508 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py @@ -264,7 +264,7 @@ class TestDistMnistAsync2x2WithGauss(TestFleetBase): feeder = fluid.DataFeeder(place=fluid.CPUPlace(), feed_list=datas) exe.run(fluid.default_startup_program()) - fluid.io.load_persistables(exe, model_file) + paddle.distributed.io.load_persistables(exe, model_file) for batch_id, data in enumerate(reader()): score = exe.run( diff --git a/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py b/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py index d7a7e535991c5004a7fcac0ee6aaaa3d4b4683fb..6694d8f403d5ae6be2b65994b9635d548654843d 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py @@ -150,12 +150,12 @@ class TestDygraphLoadStatic(unittest.TestCase): ) out = exe.run(framework.default_startup_program()) - fluid.save( + paddle.static.save( framework.default_main_program(), os.path.join(temp_dir.name, "test_1"), ) - para_dict = fluid.load_program_state( + para_dict = paddle.static.load_program_state( os.path.join(temp_dir.name, "test_1") ) diff --git a/python/paddle/fluid/tests/unittests/test_inference_model_io.py b/python/paddle/fluid/tests/unittests/test_inference_model_io.py index 779d93d0560668137ae8bc1e3c30c9f7313f0b0d..85b4130521ce10d8730dbb4b77be8e0788e4a60f 100644 --- a/python/paddle/fluid/tests/unittests/test_inference_model_io.py +++ b/python/paddle/fluid/tests/unittests/test_inference_model_io.py @@ -25,14 +25,13 @@ import paddle.fluid as fluid import paddle.fluid.core as core import paddle.fluid.executor as executor import paddle.fluid.optimizer as optimizer -from paddle.distributed.io import load_inference_model_distributed -from paddle.fluid.compiler import CompiledProgram -from paddle.fluid.framework import Program, program_guard -from paddle.fluid.io import ( - load_inference_model, - save_inference_model, +from paddle.distributed.io import ( + load_inference_model_distributed, save_persistables, ) +from paddle.fluid.compiler import CompiledProgram +from paddle.fluid.framework import Program, program_guard +from paddle.fluid.io import load_inference_model, save_inference_model paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_io_save_load.py b/python/paddle/fluid/tests/unittests/test_io_save_load.py index 4acf3d3971c0ae7c0e5fb87d27b6dee3d2181707..5ae765af9db91260e2e51b329cdff49f06064149 100644 --- a/python/paddle/fluid/tests/unittests/test_io_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_io_save_load.py @@ -18,6 +18,7 @@ import unittest import paddle import paddle.fluid as fluid +import paddle.static as static from paddle.fluid import core @@ -34,24 +35,24 @@ class TestSaveLoadAPIError(unittest.TestCase): graph = core.Graph(core.ProgramDesc()) compiled_program = fluid.CompiledProgram(graph) with self.assertRaises(TypeError): - fluid.io._get_valid_program(compiled_program) + paddle.static.io._get_valid_program(compiled_program) # case 2: main_program type error with self.assertRaises(TypeError): - fluid.io._get_valid_program("program") + paddle.static.io._get_valid_program("program") def test_load_vars_error(self): place = fluid.CPUPlace() exe = fluid.Executor(place) # case 1: main_program type error when vars None with self.assertRaises(TypeError): - fluid.io.load_vars( + static.io.load_vars( executor=exe, dirname=self.save_dir, main_program="program" ) # case 2: main_program type error when vars not None with self.assertRaises(TypeError): - fluid.io.load_vars( + static.io.load_vars( executor=exe, dirname=self.save_dir, main_program="program", diff --git a/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py b/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py index 15317e7538aff019447fc11f57920d5a4a2c1e23..c52e534f8137a6e411abf2e9622bc44122463baf 100644 --- a/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py +++ b/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py @@ -75,7 +75,7 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): def tearDown(self): self.temp_dir.cleanup() - def train_and_save_model(self, only_params=False): + def train_and_save_model(self): with new_program_scope(): startup_program = fluid.default_startup_program() main_program = fluid.default_main_program() @@ -122,19 +122,14 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): param.name ) - if only_params: - fluid.io.save_params( - exe, self.save_dirname, filename=self.params_filename - ) - else: - fluid.io.save_inference_model( - self.save_dirname, - ["img"], - [prediction], - exe, - model_filename=self.model_filename, - params_filename=self.params_filename, - ) + fluid.io.save_inference_model( + self.save_dirname, + ["img"], + [prediction], + exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) return static_param_dict @@ -195,16 +190,6 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ) self.check_load_state_dict(orig_param_dict, new_load_param_dict) - def test_load_state_dict_from_save_params(self): - self.save_dirname = os.path.join( - self.temp_dir.name, "static_mnist.load_state_dict.save_params" - ) - self.params_filename = None - orig_param_dict = self.train_and_save_model(True) - - new_load_param_dict = paddle.load(self.save_dirname) - self.check_load_state_dict(orig_param_dict, new_load_param_dict) - if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py b/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py deleted file mode 100644 index 853b364f358ab1be9504cfd31a8483389ef0bbcb..0000000000000000000000000000000000000000 --- a/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import shutil -import unittest - -import paddle -import paddle.fluid as fluid -from paddle.fluid.executor import Executor - - -class TestLoadVarsShapeCheck(unittest.TestCase): - def setUp(self): - self.model_path = "./model_temp/" - - def test_shape_check_save_load(self): - program_1 = fluid.Program() - startup_program_1 = fluid.Program() - - with fluid.program_guard(program_1, startup_program_1): - input = paddle.static.data( - name="x", shape=[-1, 10], dtype='float32' - ) - out = paddle.static.nn.fc(input, 20) - place = fluid.CPUPlace() - exe = Executor(place) - exe.run(startup_program_1) - - fluid.io.save_params(exe, self.model_path, main_program=program_1) - fluid.io.load_params(exe, self.model_path, main_program=program_1) - - def tearDown(self): - if os.path.exists(self.model_path): - shutil.rmtree(self.model_path) - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py index aee128bd99ed5ed33131905ee50b007a66d3d069..ac8e185d22769a9ff4c1a5886ca045888c94e0a1 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py @@ -235,7 +235,7 @@ class TestSaveLoadAny(unittest.TestCase): # paddle.save, legacy paddle.fluid.load self.replace_static_save(prog, path) self.set_zero(prog, place) - paddle.fluid.io.load(prog, path) + paddle.static.load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: new_t = np.array( @@ -244,7 +244,7 @@ class TestSaveLoadAny(unittest.TestCase): base_t = base_map[var.name] np.testing.assert_array_equal(new_t, np.array(base_t)) # legacy paddle.fluid.save, paddle.load - paddle.fluid.io.save(prog, path) + paddle.static.save(prog, path) self.set_zero(prog, place) self.replace_static_load(prog, path) for var in prog.list_vars(): @@ -904,7 +904,7 @@ class TestSaveLoadToMemory(unittest.TestCase): with self.assertRaises(ValueError): paddle.save(state_dict, '') with self.assertRaises(ValueError): - paddle.fluid.io._open_file_buffer('temp', 'b') + paddle.framework.io_utils._open_file_buffer('temp', 'b') def test_static_save_to_memory(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py index ece27b82c5d84f60b04d4df059f0e87bd6fb4dd5..70fe5cc8d8a1af12271952eeb2da04ce72e3679b 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py @@ -107,7 +107,7 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): var_list = list( filter(lambda var: var.persistable, prog.list_vars()) ) - fluid.io.load_vars( + paddle.static.io.load_vars( exe, path_vars1, main_program=prog, vars=var_list ) @@ -123,7 +123,7 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): path_vars2 = os.path.join( self.temp_dir.name, 'test_replace_save_load_vars_binary2/model/' ) - fluid.io.save_vars( + paddle.static.io.save_vars( exe, path_vars2, main_program=prog, vars=var_list ) self.set_zero(prog, place) diff --git a/python/paddle/fluid/tests/unittests/test_parameter.py b/python/paddle/fluid/tests/unittests/test_parameter.py index 909feb2a48ff31c60269ae4f950a4348dbce209e..f80751cb7b7aa3f6e852729d4e528701732aa18d 100644 --- a/python/paddle/fluid/tests/unittests/test_parameter.py +++ b/python/paddle/fluid/tests/unittests/test_parameter.py @@ -19,7 +19,6 @@ import numpy as np import paddle import paddle.fluid.core as core -import paddle.fluid.io as io from paddle.fluid.dygraph import guard from paddle.fluid.executor import Executor from paddle.fluid.framework import ParamBase, Variable, default_main_program @@ -47,8 +46,6 @@ class ParameterChecks(unittest.TestCase): exe = Executor(paddle.CPUPlace()) p = exe.run(main_program, fetch_list=[param])[0] np.testing.assert_array_equal(p, np.ones(shape) * val) - p = io.get_parameter_value_by_name('fc.w', exe, main_program) - np.testing.assert_array_equal(p, np.ones(shape) * val) zero_dim_param = b.create_parameter(name='x', shape=[], dtype='float32') self.assertEqual(zero_dim_param.shape, ()) diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load.py b/python/paddle/fluid/tests/unittests/test_static_save_load.py index 14e10b9f269a93b9e77b1958cb6c5f27d1126c4e..ffd8fcc2f2e0b7535f352ee900a56575a46721d2 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load.py @@ -346,7 +346,9 @@ class TestSaveLoadBase(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, os.path.join(temp_dir.name, "test_1")) + paddle.static.save( + main_program, os.path.join(temp_dir.name, "test_1") + ) # set var to zero for var in main_program.list_vars(): @@ -360,7 +362,7 @@ class TestSaveLoadBase(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load( + paddle.static.load( main_program, os.path.join(temp_dir.name, "test_1.pdparams"), exe, @@ -485,7 +487,9 @@ class TestSaveLoadPartial(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, os.path.join(temp_dir.name, "test_1")) + paddle.static.save( + main_program, os.path.join(temp_dir.name, "test_1") + ) # set var to zero for var in main_program.list_vars(): @@ -499,7 +503,7 @@ class TestSaveLoadPartial(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load( + paddle.static.load( test_program, os.path.join(temp_dir.name, "test_1.pdopt"), None ) @@ -510,7 +514,7 @@ class TestSaveLoadPartial(unittest.TestCase): ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) - fluid.load( + paddle.static.load( test_program, os.path.join(temp_dir.name, "test_1.pdmodel"), None, @@ -617,7 +621,9 @@ class TestSaveLoadSetStateDict(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, os.path.join(temp_dir.name, "test_1")) + paddle.static.save( + main_program, os.path.join(temp_dir.name, "test_1") + ) # set var to zero for var in main_program.list_vars(): @@ -631,7 +637,9 @@ class TestSaveLoadSetStateDict(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, os.path.join(temp_dir.name, "test_1"), exe) + paddle.static.load( + main_program, os.path.join(temp_dir.name, "test_1"), exe + ) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -752,7 +760,9 @@ class TestProgramStatePartial(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, os.path.join(temp_dir.name, 'test_1')) + paddle.static.save( + main_program, os.path.join(temp_dir.name, 'test_1') + ) # set var to zero for var in main_program.list_vars(): @@ -767,23 +777,23 @@ class TestProgramStatePartial(unittest.TestCase): self.assertTrue(np.sum(np.abs(new_t)) == 0) # fluid.load(test_program, "./test_1", None ) - program_state = fluid.load_program_state( + program_state = paddle.static.load_program_state( os.path.join(temp_dir.name, 'test_1') ) - program_state_1 = fluid.load_program_state( + program_state_1 = paddle.static.load_program_state( os.path.join(temp_dir.name, 'test_1.pdparams') ) - program_state_2 = fluid.load_program_state( + program_state_2 = paddle.static.load_program_state( os.path.join(temp_dir.name, 'test_1.pdopt') ) - program_state_3 = fluid.load_program_state( + program_state_3 = paddle.static.load_program_state( os.path.join(temp_dir.name, 'test_1.pdmodel') ) - fluid.set_program_state(test_program, program_state) + paddle.static.set_program_state(test_program, program_state) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -805,7 +815,7 @@ class TestProgramStatePartial(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.set_program_state(test_program, program_state_1) + paddle.static.set_program_state(test_program, program_state_1) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -827,7 +837,7 @@ class TestProgramStatePartial(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.set_program_state(test_program, program_state_2) + paddle.static.set_program_state(test_program, program_state_2) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -849,7 +859,7 @@ class TestProgramStatePartial(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.set_program_state(test_program, program_state_3) + paddle.static.set_program_state(test_program, program_state_3) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -880,7 +890,7 @@ class TestVariableInit(unittest.TestCase): exe.run(fluid.default_startup_program()) temp_dir = tempfile.TemporaryDirectory() - fluid.save( + paddle.static.save( fluid.default_main_program(), os.path.join(temp_dir.name, "test_path"), ) @@ -905,7 +915,7 @@ class TestVariableInit(unittest.TestCase): place = self.set_place() exe = fluid.Executor(place) parameter_list = list( - filter(fluid.io.is_parameter, program.list_vars()) + filter(paddle.framework.is_parameter, program.list_vars()) ) fluid.core._create_loaded_parameter( @@ -925,7 +935,10 @@ class TestVariableInit(unittest.TestCase): set_var(new_v, load_dict[v.name]) opt_list = list( - filter(fluid.io.is_belong_to_optimizer, program.list_vars()) + filter( + paddle.framework.io_utils.is_belong_to_optimizer, + program.list_vars(), + ) ) fluid.core._create_loaded_parameter( @@ -1092,7 +1105,7 @@ class TestLoadFromOldInterface(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load( + paddle.static.load( main_program, os.path.join(self.temp_dir.name, "test_path"), exe ) @@ -1112,7 +1125,7 @@ class TestLoadFromOldInterface(unittest.TestCase): var.desc.set_shape(new_shape) with self.assertRaises(RuntimeError): - fluid.load( + paddle.static.load( main_program, os.path.join(self.temp_dir.name, "test_path"), exe, @@ -1120,7 +1133,7 @@ class TestLoadFromOldInterface(unittest.TestCase): # check unused parameter - fluid.load( + paddle.static.load( test_clone_program, os.path.join(self.temp_dir.name, "test_path"), exe, @@ -1239,7 +1252,7 @@ class TestLoadFromOldInterface(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load( + paddle.static.load( main_program, os.path.join(self.temp_dir.name, "test_static_load_var_list"), exe, @@ -1377,11 +1390,11 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): self.assertTrue(np.sum(np.abs(new_t)) == 0) file_model_path = os.path.join(save_dir, "model_single") - fluid.load( + paddle.static.load( main_program, file_model_path, exe, - fluid.io.get_program_persistable_vars(main_program), + paddle.static.io.get_program_persistable_vars(main_program), ) for var in main_program.list_vars(): @@ -1403,36 +1416,33 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): var.desc.set_shape(new_shape) with self.assertRaises(RuntimeError): - fluid.load( + paddle.static.load( main_program, file_model_path, exe, - fluid.io.get_program_persistable_vars(main_program), + paddle.static.io.get_program_persistable_vars(main_program), ) - fluid.io.save_params( - exe, "test_path", main_program, filename="model_single" - ) with self.assertRaises(RuntimeError): - fluid.load( + paddle.static.load( main_program, file_model_path, exe, - fluid.io.get_program_persistable_vars(main_program), + paddle.static.io.get_program_persistable_vars(main_program), ) # check when executor is None with self.assertRaises(ValueError): - fluid.load( + paddle.static.load( main_program, file_model_path, None, - fluid.io.get_program_persistable_vars(main_program), + paddle.static.io.get_program_persistable_vars(main_program), ) # check when var list is None with self.assertRaises(ValueError): - fluid.load(main_program, file_model_path, exe, None) + paddle.static.load(main_program, file_model_path, exe, None) # check save params, load var_list = get_program_persistable_vars with self.assertRaises(RuntimeError): @@ -1440,7 +1450,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): main_program.global_block(), shape=[1], name="test_temp_var" ) all_var_list = list(main_program.list_vars()) - fluid.load( + paddle.static.load( main_program, file_model_path, exe, @@ -1579,8 +1589,8 @@ class TestProgramStateOldSave(unittest.TestCase): self.assertTrue(np.sum(np.abs(new_t)) == 0) # case 1: load basic - program_state = fluid.load_program_state(save_dir) - fluid.set_program_state(main_program, program_state) + program_state = paddle.static.load_program_state(save_dir) + paddle.static.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) # case 2: load with no need file @@ -1594,21 +1604,21 @@ class TestProgramStateOldSave(unittest.TestCase): else: raise e - program_state = fluid.load_program_state(save_dir) - fluid.set_program_state(main_program, program_state) + program_state = paddle.static.load_program_state(save_dir) + paddle.static.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) # case 3: load with var_list - program_state = fluid.load_program_state( + program_state = paddle.static.load_program_state( save_dir, main_program.all_parameters() ) - fluid.set_program_state(main_program, program_state) + paddle.static.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) if self.test_dygraph: # make sure `load_program_state` can be used in dynamic graph mode with fluid.dygraph.guard(place): - load_state = fluid.load_program_state(save_dir) + load_state = paddle.static.load_program_state(save_dir) for k, v in load_state.items(): np.testing.assert_array_equal(base_map[k], v) @@ -1758,11 +1768,13 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): self.assertTrue(np.sum(np.abs(new_t)) == 0) # fluid.load(test_program, "./test_1", None ) - program_state = fluid.load_program_state( + program_state = paddle.static.load_program_state( os.path.join(save_dir, "model_1"), - var_list=fluid.io.get_program_persistable_vars(main_program), + var_list=paddle.static.io.get_program_persistable_vars( + main_program + ), ) - fluid.set_program_state(main_program, program_state) + paddle.static.set_program_state(main_program, program_state) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -1773,15 +1785,17 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): np.testing.assert_array_equal(new_t, base_t) with self.assertRaises(ValueError): - fluid.load_program_state(os.path.join(save_dir, "model_1")) + paddle.static.load_program_state( + os.path.join(save_dir, "model_1") + ) with self.assertRaises(TypeError): - fluid.load_program_state( + paddle.static.load_program_state( os.path.join(save_dir, "model_1"), var_list=["str"] ) with self.assertRaises(RuntimeError): - fluid.load_program_state( + paddle.static.load_program_state( os.path.join(save_dir, "model_1"), var_list=[ main_program.global_block().create_var( @@ -1827,17 +1841,17 @@ class TestStaticSaveLoadPickle(unittest.TestCase): ) with self.assertRaises(ValueError): - paddle.fluid.save(prog, path, 2.0) + paddle.static.save(prog, path, 2.0) with self.assertRaises(ValueError): - paddle.fluid.save(prog, path, 1) + paddle.static.save(prog, path, 1) with self.assertRaises(ValueError): - paddle.fluid.save(prog, path, 5) + paddle.static.save(prog, path, 5) protocols = [2, 3, 4] for protocol in protocols: - paddle.fluid.save(prog, path, protocol) + paddle.static.save(prog, path, protocol) # set var to zero for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -1851,7 +1865,7 @@ class TestStaticSaveLoadPickle(unittest.TestCase): ) self.assertTrue(np.sum(np.abs(new_t)) == 0) - paddle.fluid.load(prog, path) + paddle.static.load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py b/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py index 024f31fdf92cc80c434786d8826108010834212c..ca79579569198422e9386f6d57894b9b372ac28f 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py @@ -134,7 +134,7 @@ class TestSaveLoadBF16(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t save_dir = os.path.join(self.temp_dir.name, "test_1") - fluid.save(main_program, save_dir) + paddle.static.save(main_program, save_dir) # set var to zero for var in main_program.list_vars(): @@ -148,7 +148,7 @@ class TestSaveLoadBF16(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load( + paddle.static.load( main_program, os.path.join(self.temp_dir.name, "test_1.pdparams"), exe, diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py index 85c263e9791bf136d0189ce097e7e3ad88a72fc4..54ba994e321e6fc1c69632ff329c833a59c123ed 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py @@ -58,7 +58,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): ) path = os.path.join(path, "static_save") protocol = 4 - paddle.fluid.save(prog, path, pickle_protocol=protocol) + paddle.static.save(prog, path, pickle_protocol=protocol) # set var to zero for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -70,7 +70,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): ) self.assertTrue(np.sum(np.abs(new_t)) == 0) - paddle.fluid.load(prog, path) + paddle.static.load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -91,8 +91,8 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): ) self.assertTrue(np.sum(np.abs(new_t)) == 0) - program_state = fluid.load_program_state(path) - fluid.set_program_state(prog, program_state) + program_state = paddle.static.load_program_state(path) + paddle.static.set_program_state(prog, program_state) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: new_t = np.array( diff --git a/python/paddle/framework/__init__.py b/python/paddle/framework/__init__.py index 41b2999f814e8a88342caf51468dcecb34880000..267ee8e13a65e10271827099aa1926efe3fd81ff 100755 --- a/python/paddle/framework/__init__.py +++ b/python/paddle/framework/__init__.py @@ -36,6 +36,16 @@ from ..fluid.dygraph.base import grad # noqa: F401 from .io import save # noqa: F401 from .io import load # noqa: F401 +from .io_utils import _open_file_buffer # noqa: F401 +from .io_utils import is_parameter # noqa: F401 +from .io_utils import is_persistable # noqa: F401 +from .io_utils import is_belong_to_optimizer # noqa: F401 +from .io_utils import _clone_var_in_block_ # noqa: F401 +from .io_utils import _pickle_loads_mac +from .io_utils import _pack_loaded_dict +from .io_utils import _unpack_saved_dict +from .io_utils import _load_program_scope + from ..fluid import monkey_patch_variable from ..fluid.dygraph import monkey_patch_math_varbase from ..fluid.framework import disable_signal_handler # noqa: F401 diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index 432d1088b10471355a7fe74c9f9674763916a406..831c342433976ba2b6c37f297262f82ae74c445e 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -37,14 +37,6 @@ from paddle.fluid.framework import ( _non_static_mode, _varbase_creator, ) -from paddle.fluid.io import _is_file_path, _is_memory_buffer -from paddle.fluid.io import _legacy_save as _legacy_static_save -from paddle.fluid.io import ( - _open_file_buffer, - _pack_loaded_dict, - _pickle_loads_mac, - _unpack_saved_dict, -) from paddle.jit.api import _SaveLoadConfig from paddle.jit.translated_layer import ( INFER_MODEL_SUFFIX, @@ -53,6 +45,16 @@ from paddle.jit.translated_layer import ( _construct_program_holders, ) +from .io_utils import ( + _is_file_path, + _is_memory_buffer, + _legacy_static_save, + _open_file_buffer, + _pack_loaded_dict, + _pickle_loads_mac, + _unpack_saved_dict, +) + __all__ = [] diff --git a/python/paddle/framework/io_utils.py b/python/paddle/framework/io_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..752b938e7f4cc762ec843926fcaf78178fa592a6 --- /dev/null +++ b/python/paddle/framework/io_utils.py @@ -0,0 +1,274 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import math +import os +import pickle +import sys +from io import BytesIO + +import numpy as np + +import paddle +from paddle.fluid import core +from paddle.fluid.framework import Parameter, Variable, static_only +from paddle.fluid.log_helper import get_logger +from paddle.fluid.wrapped_decorator import signature_safe_contextmanager + +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) + +# This file contains various utility functions that are used in static.io(io related api that used in static graph) +# and framework.io(io related api that used in dygraph) + + +class _open_buffer: + def __init__(self, buffer): + self.buffer = buffer + + def __enter__(self): + return self.buffer + + +class _buffer_reader(_open_buffer): + def __init__(self, buffer): + super().__init__(buffer) + self.initial_tell = self.buffer.tell() + + def __exit__(self, *args): + # `args[0]` is type of exception. When the `read` is abnormal, the file pointer returns to the initial position. + if args[0] is not None: + self.buffer.seek(self.initial_tell) + + +class _buffer_writer(_open_buffer): + def __exit__(self, *args): + self.buffer.flush() + + +def _is_file_path(path): + return isinstance(path, str) + + +def _open_file_buffer(path_or_buffer, mode): + + if _is_file_path(path_or_buffer): + return open(path_or_buffer, mode) + else: + if 'w' in mode: + return _buffer_writer(path_or_buffer) + elif 'r' in mode: + return _buffer_reader(path_or_buffer) + else: + raise ValueError( + "Expected 'r' or 'w' in mode but got {}".format(mode) + ) + + +def _is_memory_buffer(buffer): + return isinstance(buffer, BytesIO) + + +def is_persistable(var): + """ + + Check whether the given variable is persistable. + + Args: + var(Variable): The variable to be checked. + + Returns: + bool: True if the given `var` is persistable + False if not. + + Examples: + .. code-block:: python + + import paddle + import paddle.fluid as fluid + + paddle.enable_static() + param = fluid.default_main_program().global_block().var('fc.b') + res = fluid.io.is_persistable(param) + """ + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): + return False + return var.persistable + + +def is_parameter(var): + """ + Check whether the given variable is an instance of Parameter. + + Args: + var(Variable): The variable to be checked. + + Returns: + bool: True if the given `var` is an instance of Parameter, + False if not. + + Examples: + .. code-block:: python + + import paddle + import paddle.fluid as fluid + + paddle.enable_static() + param = fluid.default_main_program().global_block().var('fc.w') + res = fluid.io.is_parameter(param) + """ + return isinstance(var, Parameter) + + +def is_belong_to_optimizer(var): + if not (isinstance(var, Parameter) or var.desc.need_check_feed()): + return is_persistable(var) + + return False + + +def _clone_var_in_block_(block, var): + assert isinstance(var, Variable) + if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR: + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=True, + ) + else: + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + persistable=True, + ) + + +@signature_safe_contextmanager +def _load_program_scope(main=None, startup=None, scope=None): + prog = main if main else paddle.fluid.Program() + startup_prog = startup if startup else paddle.fluid.Program() + scope = scope if scope else paddle.fluid.core.Scope() + with paddle.fluid.scope_guard(scope): + with paddle.fluid.program_guard(prog, startup_prog): + with paddle.fluid.unique_name.guard(): + with paddle.fluid.framework._dygraph_guard(None): + yield + + +@static_only +def _legacy_static_save(param_dict, model_path, protocol=2): + def get_tensor(var): + if isinstance(var, (core.VarBase, core.eager.Tensor)): + return var.numpy() + elif isinstance(var, core.LoDTensor): + return np.array(var) + return var + + param_dict = {name: get_tensor(param_dict[name]) for name in param_dict} + + # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' + if ( + _is_file_path(model_path) + and sys.platform == 'darwin' + and sys.version_info.major == 3 + ): + pickle_bytes = pickle.dumps(param_dict, protocol=protocol) + with open(model_path, 'wb') as f: + max_bytes = 2**30 + for i in range(0, len(pickle_bytes), max_bytes): + f.write(pickle_bytes[i : i + max_bytes]) + else: + with _open_file_buffer(model_path, 'wb') as f: + pickle.dump(param_dict, f, protocol=protocol) + + +def _pickle_loads_mac(path, f): + pickle_bytes = bytearray(0) + file_size = os.path.getsize(path) + max_bytes = 2**30 + for _ in range(0, file_size, max_bytes): + pickle_bytes += f.read(max_bytes) + load_result = pickle.loads(pickle_bytes, encoding='latin1') + return load_result + + +def _pack_loaded_dict(load_obj): + if isinstance(load_obj, dict): + unpack_info = 'UnpackBigParamInfor@@' + if unpack_info in load_obj: + removes = [] + for key, value in load_obj[unpack_info].items(): + slices = [load_obj[part] for part in value["slices"]] + load_obj[key] = np.concatenate(slices).reshape( + value["OriginShape"] + ) + removes += value["slices"] + for key in removes: + load_obj.pop(key) + load_obj.pop(unpack_info) + + return load_obj + + +def _unpack_saved_dict(saved_obj, protocol): + temp_saved_obj = {} + unpack_infor = {} + # When pickle protocol=2 or protocol=3 the serialized object cannot be larger than 4G. + if 1 < protocol < 4: + if isinstance(saved_obj, dict): + for key, value in saved_obj.items(): + if isinstance(value, np.ndarray): + MAX_NUMBER_OF_ELEMENT = int( + (2**30 - 1) / value.dtype.itemsize + ) + num_element = np.prod(value.shape) + if num_element > MAX_NUMBER_OF_ELEMENT: + unpack_infor[key] = {} + unpack_infor[key]["OriginShape"] = value.shape + unpack_infor[key]["slices"] = [] + value = value.flatten() + for i in range( + int( + math.ceil( + num_element * 1.0 / MAX_NUMBER_OF_ELEMENT + ) + ) + ): + part_name = key + "@@." + str(i) + unpack_infor[key]["slices"].append(part_name) + temp_saved_obj[part_name] = value[ + i + * MAX_NUMBER_OF_ELEMENT : MAX_NUMBER_OF_ELEMENT + * (i + 1) + ] + + if unpack_infor: + for key, value in unpack_infor.items(): + if key in saved_obj: + saved_obj.pop(key) + for part in value['slices']: + saved_obj[part] = temp_saved_obj[part] + saved_obj['UnpackBigParamInfor@@'] = unpack_infor + return saved_obj diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py index 6a24bf4cbf366cb1b14a7ecc2d3cf5b287197de6..1c3015e287507580a3e9acb8720297873721488c 100644 --- a/python/paddle/hapi/model.py +++ b/python/paddle/hapi/model.py @@ -34,9 +34,9 @@ from paddle.fluid.executor import global_scope from paddle.fluid.framework import Variable from paddle.fluid.framework import _current_expected_place as _get_device from paddle.fluid.framework import _get_paddle_place, _non_static_mode -from paddle.fluid.io import is_belong_to_optimizer from paddle.fluid.layers import collective from paddle.fluid.layers.utils import flatten +from paddle.framework.io_utils import is_belong_to_optimizer from paddle.io import DataLoader, Dataset, DistributedBatchSampler from paddle.jit.translated_layer import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX from paddle.metric import Metric diff --git a/python/paddle/incubate/distributed/fleet/fleet_util.py b/python/paddle/incubate/distributed/fleet/fleet_util.py index ce1cdd74de11f9034d297c3bc64edb36eb2df832..c9d457d92a282d47ce59b924ef89f41278ab9fb9 100644 --- a/python/paddle/incubate/distributed/fleet/fleet_util.py +++ b/python/paddle/incubate/distributed/fleet/fleet_util.py @@ -23,6 +23,7 @@ import time import numpy as np +import paddle import paddle.fluid as fluid from paddle.distributed.fleet.utils.fs import HDFSClient from paddle.fluid.log_helper import get_logger @@ -1087,11 +1088,13 @@ class FleetUtil: vars = [program.global_block().var(i) for i in var_names] with fluid.scope_guard(scope): if save_combine: - fluid.io.save_vars( + paddle.static.io.save_vars( executor, "./", program, vars=vars, filename=model_name ) else: - fluid.io.save_vars(executor, model_name, program, vars=vars) + paddle.static.io.save_vars( + executor, model_name, program, vars=vars + ) configs = { "fs.default.name": hadoop_fs_name, diff --git a/python/paddle/incubate/distributed/fleet/utils.py b/python/paddle/incubate/distributed/fleet/utils.py index a2838c25b56cf0ae553de40d07f1fb7f6a693802..278ab0fb61b686b0dbdae5e68b31f17051f92564 100644 --- a/python/paddle/incubate/distributed/fleet/utils.py +++ b/python/paddle/incubate/distributed/fleet/utils.py @@ -22,6 +22,7 @@ from google.protobuf import text_format import paddle import paddle.fluid as fluid +import paddle.framework.io_utils as io_utils from paddle.fluid import core, debugger from paddle.fluid.framework import Program from paddle.fluid.proto import framework_pb2 @@ -92,7 +93,7 @@ def check_pruned_program_vars(train_prog, pruned_prog): pruned_vars = [ (v.name, v) for v in pruned_prog.list_vars() - if fluid.io.is_persistable(v) + if io_utils.is_persistable(v) ] pruned_vars = OrderedDict(pruned_vars) pruned_vars_name = [name for name in pruned_vars] @@ -451,7 +452,7 @@ def check_saved_vars_try_dump( os.path.join(dump_dir, dump_prog_fn), is_text_dump_program ) saved_params = [ - v for v in dump_prog.list_vars() if fluid.io.is_persistable(v) + v for v in dump_prog.list_vars() if io_utils.is_persistable(v) ] logger.info( "persistable vars in dump program: {}".format( @@ -477,7 +478,7 @@ def parse_program(program, output_dir): # persistable vars output = {} persistable_vars = [ - v for v in program.list_vars() if fluid.io.is_persistable(v) + v for v in program.list_vars() if io_utils.is_persistable(v) ] output["persistable_vars"] = [ { diff --git a/python/paddle/jit/api.py b/python/paddle/jit/api.py index e928613d650b86481655d9d03d9f283a58cb6707..fc1311646059f1c1b6757af5c160bdd77fd43c14 100644 --- a/python/paddle/jit/api.py +++ b/python/paddle/jit/api.py @@ -1209,7 +1209,11 @@ def save(layer, path, input_spec=None, **configs): paddle.static.save_vars( Executor(_current_expected_place()), dirname=model_path, - vars=list(filter(paddle.fluid.io.is_persistable, ordered_vars)), + vars=list( + filter( + paddle.framework.io_utils.is_persistable, ordered_vars + ) + ), filename=params_filename, ) # save property diff --git a/python/paddle/static/__init__.py b/python/paddle/static/__init__.py index 6aff549a35acb42cc0e86b7a6db7f1fdf4961cc9..457d2f1d0768b65c9d149174e848f3ec9a3f98e8 100644 --- a/python/paddle/static/__init__.py +++ b/python/paddle/static/__init__.py @@ -27,6 +27,13 @@ from .io import serialize_program # noqa: F401 from .io import load_from_file # noqa: F401 from .io import save_to_file # noqa: F401 from .io import normalize_program # noqa: F401 +from .io import is_persistable # noqa: F401 +from .io import save_vars # noqa: F401 +from .io import load_vars # noqa: F401 +from .io import save # noqa: F401 +from .io import load # noqa: F401 +from .io import load_program_state # noqa: F401 +from .io import set_program_state # noqa: F401 from ..fluid import Scope # noqa: F401 from .input import data # noqa: F401 from .input import InputSpec # noqa: F401 @@ -66,13 +73,6 @@ from ..fluid.param_attr import WeightNormParamAttr # noqa: F401 from ..fluid.optimizer import Optimizer # noqa: F401 from ..fluid.optimizer import Adam # noqa: F401 from ..fluid.optimizer import ExponentialMovingAverage # noqa: F401 -from ..fluid.io import save # noqa: F401 -from ..fluid.io import load # noqa: F401 -from ..fluid.io import load_program_state # noqa: F401 -from ..fluid.io import set_program_state # noqa: F401 -from ..fluid.io import load_vars # noqa: F401 -from ..fluid.io import save_vars # noqa: F401 -from ..fluid.io import batch # noqa: F401 from ..fluid.contrib.layers import ctr_metric_bundle # noqa: F401 from ..fluid.layers import exponential_decay # noqa: F401 diff --git a/python/paddle/static/io.py b/python/paddle/static/io.py index a0a0b0c046a0304009ada876c8763b1551a218ee..3227dc6c67dd39c7c7ef30dd6d02e6b62fa1a3be 100644 --- a/python/paddle/static/io.py +++ b/python/paddle/static/io.py @@ -16,6 +16,8 @@ import errno import inspect import logging import os +import pickle +import sys import warnings import numpy as np @@ -30,10 +32,20 @@ from paddle.fluid import ( program_guard, unique_name, ) -from paddle.fluid.executor import global_scope -from paddle.fluid.framework import Parameter, static_only +from paddle.fluid.executor import Executor, global_scope +from paddle.fluid.framework import Parameter, dygraph_not_support, static_only from paddle.fluid.io import append_fetch_ops, prepend_feed_ops from paddle.fluid.log_helper import get_logger +from paddle.framework.io_utils import ( + _clone_var_in_block_, + _load_program_scope, + _pack_loaded_dict, + _pickle_loads_mac, + _unpack_saved_dict, + is_belong_to_optimizer, + is_parameter, + is_persistable, +) __all__ = [] @@ -230,37 +242,6 @@ def normalize_program(program, feed_vars, fetch_vars): return copy_program -def is_persistable(var): - """ - - Check whether the given variable is persistable. - - Args: - var(Variable): The variable to be checked. - - Returns: - bool: True if the given `var` is persistable - False if not. - - Examples: - .. code-block:: python - - import paddle - import paddle.fluid as fluid - - paddle.enable_static() - param = fluid.default_main_program().global_block().var('fc.b') - res = fluid.io.is_persistable(param) - """ - if ( - var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH - or var.desc.type() == core.VarDesc.VarType.FETCH_LIST - or var.desc.type() == core.VarDesc.VarType.READER - ): - return False - return var.persistable - - @static_only def serialize_program(feed_vars, fetch_vars, **kwargs): """ @@ -882,3 +863,1048 @@ def load_inference_model(path_prefix, executor, **kwargs): ] return [program, feed_target_names, fetch_targets] + + +@dygraph_not_support +def save_vars( + executor, + dirname, + main_program=None, + vars=None, + predicate=None, + filename=None, +): + """ + Save specific variables in the `Program` to files. + + There are two ways to specify the variables to be saved: set variables in + a list and assign it to the `vars`, or use the `predicate` function to select + variables that make `predicate(variable) == True`. The first way has a higher priority. + + The `dirname` is used to specify the folder where to save variables. + If you prefer to save variables in separate files in the `dirname` folder, + do not set `filename`. If you prefer to save all variables in a single file, + use `filename` to specify it. + + Args: + executor(Executor): The executor to run for saving variables. + dirname(str, optional): The folder where to save variables. + When you need to save the parameter to the memory, set it to None. + main_program(Program, optional): The program whose variables will be saved. + If it is None, the default main program will + be used automatically. + Default: None + vars(list[Variable], optional): The list contains all variables to be saved. + Default: None + predicate(function, optional): The function selects the variables that make + `predicate(variable) == True`. + Default: None + filename(str, optional): If you prefer to save all variables in a single file, + use `filename` to specify it. Otherwise, let `filename` be None. + Default: None + + Returns: + str: When saving parameters to a file, returns None. + When saving parameters to memory, returns a binary string containing parameters. + + Raises: + TypeError: If `main_program` is not an instance of Program nor None. + + Examples: + .. code-block:: python + + import paddle + import paddle.fluid as fluid + + paddle.enable_static() + main_prog = fluid.Program() + startup_prog = fluid.Program() + with fluid.program_guard(main_prog, startup_prog): + data = paddle.static.data(name="img", shape=[64, 784]) + w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w') + b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b') + hidden_w = paddle.matmul(x=data, y=w) + hidden_b = paddle.add(hidden_w, b) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(startup_prog) + + # The first usage: use `vars` to set the saved variables. + var_list = [w, b] + path = "./my_paddle_vars" + fluid.io.save_vars(executor=exe, dirname=path, vars=var_list, + filename="vars_file") + # w and b will be save in a file named "var_file". + + # The second usage: use `predicate` to select the saved variable. + def name_has_fc(var): + res = "fc" in var.name + return res + param_path = "./my_paddle_model" + fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog, vars=None, predicate = name_has_fc) + # all variables whose names contain "fc " are saved. + """ + save_to_memory = False + if dirname is None and filename is None: + save_to_memory = True + + main_program = paddle.static.io._get_valid_program(main_program) + + if vars is None: + return save_vars( + executor, + main_program=main_program, + dirname=dirname, + vars=list(filter(predicate, main_program.list_vars())), + filename=filename, + ) + else: + params_var_name = "saved_params" + # give warning when there is no var in model + if len(list(vars)) == 0: + warnings.warn( + "no variable in your model, please ensure there are any variables in your model to save" + ) + return None + + save_program = Program() + save_block = save_program.global_block() + + save_var_map = {} + for each_var in vars: + # NOTE: don't save the variable which type is RAW + if each_var.type == core.VarDesc.VarType.RAW: + continue + new_var = _clone_var_in_block_(save_block, each_var) + if filename is None and save_to_memory is False: + save_file_path = os.path.join( + os.path.normpath(dirname), new_var.name + ) + save_block.append_op( + type='save', + inputs={'X': [new_var]}, + outputs={}, + attrs={'file_path': os.path.normpath(save_file_path)}, + ) + else: + save_var_map[new_var.name] = new_var + + if filename is not None or save_to_memory: + save_var_list = [] + for name in sorted(save_var_map.keys()): + save_var_list.append(save_var_map[name]) + + save_path = str() + if save_to_memory is False: + save_path = os.path.join(os.path.normpath(dirname), filename) + + saved_params = save_block.create_var( + type=core.VarDesc.VarType.RAW, name=params_var_name + ) + saved_params.desc.set_persistable(True) + save_block.append_op( + type='save_combine', + inputs={'X': save_var_list}, + outputs={'Y': saved_params}, + attrs={ + 'file_path': save_path, + 'save_to_memory': save_to_memory, + }, + ) + + # NOTE(zhiqiu): save op will add variable kLookupTablePath in save_program.desc, + # which leads to diff on save_program and its desc. Call _sync_with_cpp + # to keep consistency. + save_program._sync_with_cpp() + executor.run(save_program) + if save_to_memory: + return global_scope().find_var(params_var_name).get_bytes() + + +def load_vars( + executor, + dirname, + main_program=None, + vars=None, + predicate=None, + filename=None, +): + """ + :api_attr: Static Graph + + This API loads variables from files by executor. + + There are two ways to specify the variables to be loaded: the first way, set + variables in a list and assign it to the `vars`; the second way, use the + `predicate` function to select variables that make `predicate(variable) == True`. + The first way has a higher priority. + + The `dirname` is used to specify the folder where to load variables. + If variables were saved in separate files in the folder `dirname`, + set `filename` None. If all variables were saved in a single file, + use `filename` to specify it. + + Args: + executor(Executor): The executor to run for loading variables. + dirname(str): The folder where to load the variables. + main_program(Program, optional): The program whose variables will be loaded. + If it is None, the default main program will + be used automatically. + Default: None + vars(list[Variable], optional): The list that contains all variables to be loaded. + Default: None + predicate(function, optional): The function selects variables that make + `predicate(variable) == True`. + Default: None + filename(str, optional): The file which saved all required variables. If variables + were saved in separate files, set it to be None. + Default: None + + Returns: + None + + Examples: + .. code-block:: python + + import paddle + import paddle.fluid as fluid + + paddle.enable_static() + main_prog = fluid.Program() + startup_prog = fluid.Program() + with fluid.program_guard(main_prog, startup_prog): + data = paddle.static.data(name="img", shape=[64, 784]) + w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w') + b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b') + hidden_w = paddle.matmul(x=data, y=w) + hidden_b = paddle.add(hidden_w, b) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(startup_prog) + + # The first usage: using `vars` to specify the variables. + path = "./my_paddle_vars" + var_list = [w, b] + fluid.io.save_vars(executor=exe, dirname=path, vars=var_list, + filename="vars_file") + fluid.io.load_vars(executor=exe, dirname=path, vars=var_list, + filename="vars_file") + # w and b will be loaded, and they are supposed to + # be saved in the same file named 'var_file' in the path "./my_paddle_vars". + + # The second usage: using the `predicate` function to select variables + param_path = "./my_paddle_model" + def name_has_fc(var): + res = "fc" in var.name + return res + fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog, + vars=None, predicate=name_has_fc) + fluid.io.load_vars(executor=exe, dirname=param_path, main_program=main_prog, + vars=None, predicate=name_has_fc) + # Load All variables in the `main_program` whose name includes "fc". + # And all the variables are supposed to be saved in separate files. + + """ + vars_from_memory = False + if dirname is not None: + dirname = os.path.normpath(dirname) + else: + vars_from_memory = True + + if vars is None: + if main_program is None: + main_program = default_main_program() + if not isinstance(main_program, Program): + raise TypeError( + "The type of input main_program is invalid, expected type is fluid.Program, but received %s" + % type(main_program) + ) + + load_vars( + executor, + dirname=dirname, + main_program=main_program, + vars=list(filter(predicate, main_program.list_vars())), + filename=filename, + ) + else: + load_prog = Program() + load_block = load_prog.global_block() + + if main_program is None: + main_program = default_main_program() + + if not isinstance(main_program, Program): + raise TypeError( + "The type of input main_program is invalid, expected type is fluid.Program, but received %s" + % type(main_program) + ) + + # save origin param shape + orig_para_shape = {} + load_var_map = {} + + check_vars = [] + sparse_vars = [] + + for each_var in vars: + assert isinstance(each_var, Variable) + + if each_var.type == core.VarDesc.VarType.RAW: + continue + + if isinstance(each_var, Parameter): + orig_para_shape[each_var.name] = tuple( + each_var.desc.get_shape() + ) + + if each_var.type == core.VarDesc.VarType.SELECTED_ROWS: + sparse_vars.append(each_var) + continue + + new_var = _clone_var_in_block_(load_block, each_var) + check_vars.append(each_var) + + if filename is None: + if dirname is None: + raise ValueError( + "The directory path and params cannot be None at the same time." + ) + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [new_var]}, + attrs={'file_path': os.path.join(dirname, new_var.name)}, + ) + else: + load_var_map[new_var.name] = new_var + + for each_var in sparse_vars: + assert isinstance(each_var, Variable) + + if filename is not None: + raise ValueError( + "SelectedRows can not be load with load_combine" + ) + + new_var = _clone_var_in_block_(load_block, each_var) + + var_path = os.path.join(dirname, new_var.name) + if not os.path.exists(var_path): + raise ValueError( + "SelectedRows var {} can not find at {}".format( + new_var.name, var_path + ) + ) + + if os.path.isfile(var_path): + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [new_var]}, + attrs={'file_path': os.path.join(dirname, new_var.name)}, + ) + else: + blocks = [] + block_paths = os.listdir(var_path) + + for block in block_paths: + if block.startswith(new_var.name): + blocks.append(block) + + slices = [] + for block in blocks: + slice = load_block.create_var( + name=block, + type=new_var.type, + shape=new_var.shape, + dtype=new_var.dtype, + persistable=False, + ) + slices.append(slice) + + file_path = os.path.join(var_path, block, "Param") + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [slice]}, + attrs={'file_path': file_path}, + ) + + load_block.append_op( + type='lookup_sparse_table_merge', + inputs={'X': slices}, + outputs={'Out': new_var}, + attrs={}, + ) + + if filename is not None: + load_var_list = [] + for name in sorted(load_var_map.keys()): + load_var_list.append(load_var_map[name]) + + if vars_from_memory is False: + filename = os.path.join(dirname, filename) + + load_block.append_op( + type='load_combine', + inputs={}, + outputs={"Out": load_var_list}, + attrs={ + 'file_path': filename, + 'model_from_memory': vars_from_memory, + }, + ) + executor.run(load_prog) + + # check var shape + for each_var in check_vars: + if not isinstance(each_var, Parameter): + continue + var_temp = paddle.fluid.global_scope().find_var(each_var.name) + assert var_temp is not None, "can't not find var: " + each_var.name + new_shape = (np.array(var_temp.get_tensor())).shape + assert each_var.name in orig_para_shape, ( + each_var.name + "MUST in var list" + ) + orig_shape = orig_para_shape.get(each_var.name) + if new_shape != orig_shape: + raise RuntimeError( + "Variable's shape does not match, the Program requires a parameter with the shape of ({}), " + "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( + orig_shape, each_var.name, new_shape + ) + ) + + +@static_only +def save(program, model_path, protocol=4, **configs): + """ + + This function save parameters, optimizer information and network description to model_path. + + The parameters contains all the trainable Tensor, will save to a file with suffix ".pdparams". + The optimizer information contains all the Tensor used by optimizer. For Adam optimizer, contains beta1, beta2, momentum etc. All the information will save to a file with suffix ".pdopt". (If the optimizer have no Tensor need to save (like SGD), the fill will not generated). + The network description is the description of the program. It's only used for deployment. The description will save to a file with a suffix ".pdmodel". + + Args: + program(Program) : The program to saved. + model_path(str): the file prefix to save the program. The format is "dirname/file_prefix". If file_prefix is empty str. A exception will be raised + protocol(int, optional): The protocol version of pickle module must be greater than 1 and less than 5. + Default: 4 + configs(dict, optional) : optional keyword arguments. + + Returns: + None + + Examples: + .. code-block:: python + + import paddle + import paddle.static as static + + paddle.enable_static() + + x = static.data(name="x", shape=[10, 10], dtype='float32') + y = static.nn.fc(x, 10) + z = static.nn.fc(y, 10) + + place = paddle.CPUPlace() + exe = static.Executor(place) + exe.run(static.default_startup_program()) + prog = static.default_main_program() + + static.save(prog, "./temp") + """ + + base_name = os.path.basename(model_path) + assert ( + base_name != "" + ), "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received model_path is empty string." + if 'pickle_protocol' in configs: + protocol = configs['pickle_protocol'] + warnings.warn( + "'pickle_protocol' is a deprecated argument. Please use 'protocol' instead." + ) + + if not isinstance(protocol, int): + raise ValueError( + "The 'protocol' MUST be `int`, but received {}".format( + type(protocol) + ) + ) + + if protocol < 2 or protocol > 4: + raise ValueError( + "Expected 1<'protocol'<5, but received protocol={}".format(protocol) + ) + + dir_name = os.path.dirname(model_path) + if dir_name and not os.path.exists(dir_name): + os.makedirs(dir_name) + + def get_tensor(var): + t = global_scope().find_var(var.name).get_tensor() + return np.array(t) + + parameter_list = list(filter(is_parameter, program.list_vars())) + param_dict = {p.name: get_tensor(p) for p in parameter_list} + + param_dict = _unpack_saved_dict(param_dict, protocol) + + # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' + if sys.platform == 'darwin' and sys.version_info.major == 3: + pickle_bytes = pickle.dumps(param_dict, protocol=protocol) + with open(model_path + ".pdparams", 'wb') as f: + max_bytes = 2**30 + for i in range(0, len(pickle_bytes), max_bytes): + f.write(pickle_bytes[i : i + max_bytes]) + else: + with open(model_path + ".pdparams", 'wb') as f: + pickle.dump(param_dict, f, protocol=protocol) + + optimizer_var_list = list( + filter(is_belong_to_optimizer, program.list_vars()) + ) + + opt_dict = {p.name: get_tensor(p) for p in optimizer_var_list} + with open(model_path + ".pdopt", 'wb') as f: + pickle.dump(opt_dict, f, protocol=protocol) + + main_program = program.clone() + program.desc.flush() + main_program.desc._set_version() + paddle.fluid.core.save_op_version_info(program.desc) + + with open(model_path + ".pdmodel", "wb") as f: + f.write(program.desc.serialize_to_string()) + + +@static_only +def load(program, model_path, executor=None, var_list=None): + """ + :api_attr: Static Graph + + This function get parameters and optimizer information from program, and then get corresponding value from file. + An exception will throw if shape or dtype of the parameters is not match. + + This function can also load model file saved with [ save_params, save_persistables, save_vars ]. + var_list can not be None when load single model file + ( filename is not None When save_params, save_persistables or save_vars is called ). + + Args: + program(Program): The program will be loaded + model_path(str): The file prefix store the program + executor(Executor, optional): The executor used for initialize the parameter + When startup program is not run. + var_list(list|tuple, optional): The Tensor list/tuple to load single model file saved with + [ save_params, save_persistables, save_vars ]. + Default: None + + Returns: + None + + Examples: + .. code-block:: python + + import paddle + import paddle.static as static + + paddle.enable_static() + + x = static.data(name="x", shape=[10, 10], dtype='float32') + y = static.nn.fc(x, 10) + z = static.nn.fc(y, 10) + + place = paddle.CPUPlace() + exe = static.Executor(place) + exe.run(static.default_startup_program()) + prog = static.default_main_program() + + static.save(prog, "./temp") + static.load(prog, "./temp") + """ + + assert executor is None or isinstance(executor, Executor) + + model_prefix = model_path + if model_prefix.endswith(".pdparams"): + model_prefix = model_prefix[:-9] + elif model_prefix.endswith(".pdopt"): + model_prefix = model_prefix[:-6] + elif model_prefix.endswith(".pdmodel"): + model_prefix = model_prefix[:-8] + + parameter_file_name = model_prefix + ".pdparams" + + if not os.path.exists(parameter_file_name): + # model file save by fluid.save not found, try to load model file saved with + # [save_vars, save_params, save_persistables] + _logger.debug( + "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format( + parameter_file_name + ) + ) + if executor is None: + raise ValueError( + "executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]" + ) + + if var_list is not None: + var_list_names = [var.name for var in var_list] + else: + var_list_names = None + + if os.path.isdir(model_path): + binary_file_set = set() + for root, dirs, files in os.walk(model_path, topdown=False): + for f in files: + binary_file_set.add( + os.path.join(root, f).replace("\\", "/") + ) + program_var_list = list(program.list_vars()) + loaded_var_list = [] + for var in program_var_list: + var_path = os.path.join(model_path, var.name).replace("\\", "/") + load_condition = ( + var_list_names is None or var.name in var_list_names + ) + if var_path in binary_file_set and load_condition: + loaded_var_list.append(var) + binary_file_set.remove(var_path) + if len(binary_file_set) > 0: + unused_var_list = " ".join(list(binary_file_set)) + _logger.warning( + "variable file [ %s ] not used" + % (" ".join(list(binary_file_set))) + ) + try: + load_vars( + executor=executor, dirname=model_path, vars=loaded_var_list + ) + except RuntimeError as e: + _logger.error(e) + raise e + except: + raise RuntimeError( + "Failed to load model file, please make sure model file is saved with the " + "following APIs: save_params, save_persistables, save_vars" + ) + + return + elif os.path.isfile(model_path): + if var_list is None: + raise ValueError( + "var_list is required when loading model file saved with [ save_params, save_persistables, save_vars ]" + ) + program_var_list = program.list_vars() + program_var_name_set = set([var.name for var in program_var_list]) + + # check all the variable inlcuded in program + for var in var_list: + if var.name not in program_var_name_set: + raise LookupError( + "loaded var [{}] is not in program variable list" + ) + + dir_name, file_name = os.path.split(model_path) + try: + load_vars( + executor=executor, + dirname=dir_name, + vars=var_list, + filename=file_name, + ) + except RuntimeError as e: + _logger.error(e) + raise e + except: + raise RuntimeError( + "Failed to load model file , please make sure model file is saved with the " + "the following APIs: [ save_params, save_persistables, save_vars ]. " + "When these API called, filename CANNOT be None" + ) + + return + + def set_var(var, ndarray): + t = global_scope().find_var(var.name).get_tensor() + p = t._place() + if p.is_cpu_place(): + place = paddle.fluid.CPUPlace() + elif p.is_cuda_pinned_place(): + place = paddle.fluid.CUDAPinnedPlace() + elif p.is_xpu_place(): + p = paddle.fluid.core.Place() + p.set_place(t._place()) + place = paddle.fluid.XPUPlace(p.xpu_device_id()) + elif p.is_npu_place(): + p = paddle.fluid.core.Place() + p.set_place(t._place()) + place = paddle.fluid.NPUPlace(p.npu_device_id()) + elif p.is_mlu_place(): + p = paddle.fluid.core.Place() + p.set_place(t._place()) + place = paddle.fluid.MLUPlace(p.mlu_device_id()) + else: + p = paddle.fluid.core.Place() + p.set_place(t._place()) + place = paddle.fluid.CUDAPlace(p.gpu_device_id()) + + t.set(ndarray, place) + + parameter_list = list(filter(is_parameter, program.list_vars())) + + if executor: + paddle.fluid.core._create_loaded_parameter( + parameter_list, global_scope(), executor._default_executor + ) + with open(parameter_file_name, 'rb') as f: + + # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' + if sys.platform == 'darwin' and sys.version_info.major == 3: + load_dict = _pickle_loads_mac(parameter_file_name, f) + else: + load_dict = pickle.load(f, encoding='latin1') + load_dict = _pack_loaded_dict(load_dict) + for v in parameter_list: + assert ( + v.name in load_dict + ), "Can not find [{}] in model file [{}]".format( + v.name, parameter_file_name + ) + set_var(v, load_dict[v.name]) + + optimizer_var_list = list( + filter(is_belong_to_optimizer, program.list_vars()) + ) + + if len(optimizer_var_list) > 0: + opt_file_name = model_prefix + ".pdopt" + assert os.path.exists( + opt_file_name + ), "Optimizer file [{}] not exits".format(opt_file_name) + + if executor: + paddle.fluid.core._create_loaded_parameter( + optimizer_var_list, global_scope(), executor._default_executor + ) + + with open(opt_file_name, 'rb') as f: + load_dict = pickle.load(f, encoding='latin1') + for v in optimizer_var_list: + assert ( + v.name in load_dict + ), "Can not find [{}] in model file [{}]".format( + v.name, opt_file_name + ) + set_var(v, load_dict[v.name]) + + +@static_only +def set_program_state(program, state_dict): + """ + Set program parameter from state_dict + + An exception will throw if shape or dtype of the parameters is not match. + + NOTICE: This function MUST called after run start_up_program + + Args: + program(Program): The program to be set + state_dict(dict): the dict store Parameter and optimizer information + Returns: + None + + Examples: + .. code-block:: python + + import paddle + import paddle.static as static + + paddle.enable_static() + + x = static.data(name="x", shape=[10, 10], dtype='float32') + y = static.nn.fc(x, 10) + z = static.nn.fc(y, 10) + + place = paddle.CPUPlace() + exe = static.Executor(place) + exe.run(static.default_startup_program()) + prog = static.default_main_program() + + static.save(prog, "./temp") + program_state = static.load_program_state("./temp") + + static.set_program_state(prog, program_state) + """ + state_dict = _pack_loaded_dict(state_dict) + parameter_list = list(filter(is_persistable, program.list_vars())) + + used_para_list = {} + for para in parameter_list: + var_temp = paddle.fluid.global_scope().find_var(para.name) + assert ( + var_temp is not None + ), "Variable [ {} ] Not found, Please make sure run startup program".format( + para.name + ) + if para.name in state_dict: + # set value from state dict + orig_para_np = np.array(var_temp.get_tensor()) + new_para_np = state_dict[para.name] + assert orig_para_np.shape == new_para_np.shape, ( + "Parameter's shape does not match, the Program requires a parameter with the shape of ({}), " + "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( + orig_para_np.shape, para.name, new_para_np.shape + ) + ) + assert orig_para_np.dtype == new_para_np.dtype, ( + "Parameter's data type does not match, the Program requires a parameter with a dtype of ({}), " + "while the loaded parameter (namely [ {} ]) has a dtype of ({}).".format( + orig_para_np.dtype, para.name, new_para_np.dtype + ) + ) + + ten = var_temp.get_tensor() + ten_place = ten._place() + + # assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \ + # "Place not support, only support CPUPlace and GPUPlace, now is {}".format(str(ten_place)) + py_place = paddle.fluid.CPUPlace() + if ten_place.is_cuda_pinned_place(): + place = paddle.fluid.CUDAPinnedPlace() + elif ten_place.is_gpu_place(): + p = paddle.fluid.core.Place() + p.set_place(ten_place) + py_place = paddle.fluid.CUDAPlace(p.gpu_device_id()) + elif ten_place.is_xpu_place(): + p = paddle.fluid.core.Place() + p.set_place(ten_place) + py_place = paddle.fluid.XPUPlace(p.xpu_device_id()) + elif ten_place.is_npu_place(): + p = paddle.fluid.core.Place() + p.set_place(ten_place) + py_place = paddle.fluid.NPUPlace(p.npu_device_id()) + elif ten_place.is_mlu_place(): + p = paddle.fluid.core.Place() + p.set_place(ten_place) + py_place = paddle.fluid.MLUPlace(p.mlu_device_id()) + + ten.set(new_para_np, py_place) + + used_para_list[para.name] = 1 + + unused_para_list = [] + for k, v in state_dict.items(): + if k not in used_para_list: + unused_para_list.append(k) + if len(unused_para_list) > 0: + warnings.warn( + "This list is not set, Because of Paramerter not found in program. There are: {}".format( + " ".join(unused_para_list) + ) + ) + + +@dygraph_not_support +def get_program_persistable_vars(program): + """ + Get all the persistable vars from Program. + Args: + var(Program): The Program to get persistable vars + Returns: + list: The list contains all persistable vars in the program + Examples: + .. code-block:: python + import paddle + import paddle.static.io as io + import paddle.fluid as fluid + paddle.enable_static() + data = fluid.data(name="img", shape=[64, 784]) + w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w') + b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b') + list_para = io.get_program_persistable_vars( fluid.default_main_program() ) + """ + return list(filter(is_persistable, program.list_vars())) + + +def load_program_state(model_path, var_list=None): + """ + + Load program state from local file + + Args: + model_path(str): The file prefix store the program + var_list(list|tuple, optional): The Tensor list/tuple to load saved with + [ save_params, save_persistables, save_vars ]. + Default: None. + The var_list is only used to get name, + will not be modified. + Returns: + state_dict(dict): the dict store Parameter and optimizer information + + Examples: + + .. code-block:: python + + import paddle + import paddle.static as static + + paddle.enable_static() + + x = static.data(name="x", shape=[10, 10], dtype='float32') + y = static.nn.fc(x, 10) + z = static.nn.fc(y, 10) + + place = paddle.CPUPlace() + exe = static.Executor(place) + exe.run(static.default_startup_program()) + prog = static.default_main_program() + + static.save(prog, "./temp") + program_state = static.load_program_state("./temp") + """ + model_prefix = model_path + if model_prefix.endswith(".pdparams"): + model_prefix = model_prefix[:-9] + elif model_prefix.endswith(".pdopt"): + model_prefix = model_prefix[:-6] + elif model_prefix.endswith(".pdmodel"): + model_prefix = model_prefix[:-8] + + parameter_file_name = model_prefix + ".pdparams" + if not os.path.exists(parameter_file_name): + # model file saved with fluid.save is not found, try to load model file saved with + # [save_vars, save_params, save_persistables] + _logger.debug( + "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format( + parameter_file_name + ) + ) + + var_name_list = [] + if var_list is None and os.path.isfile(model_path): + raise ValueError( + "var_list can not be None when model_path is a file type" + ) + + for root, dirs, files in os.walk(model_path, topdown=False): + for f in files: + file_path = os.path.join(root, f) + var_temp_name = os.path.relpath(file_path, model_path) + var_temp_name = var_temp_name.replace("\\", "/") + var_name_list.append(var_temp_name) + + with _load_program_scope(): + load_prog = Program() + load_block = load_prog.global_block() + + def clone_var_to_block(block, var): + if not isinstance(var, Variable): + raise TypeError("value in var_list must be variable") + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level + if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR + else None, + persistable=True, + ) + + def _load_vars_with_try_catch( + exe, dirname, vars, filename, raise_error=True + ): + try: + load_vars( + executor=exe, + dirname=dirname, + vars=vars, + filename=filename, + ) + return True + except: + error_str = ( + "Failed to load model/variables `%s`, please make sure " + "model/variables file is saved with the following APIs: " + "save_params, save_persistables, save_vars." + ) + filenames = ( + [var.name for var in vars] + if filename is None + else filename + ) + if raise_error: + raise RuntimeError(error_str % filenames) + else: + warnings.warn(error_str % filenames, RuntimeWarning) + return False + + place = paddle.fluid.CPUPlace() + exe = paddle.fluid.Executor(place) + + loaded_var_list = [] + + if os.path.isfile(model_path): + # when model_path is file, var_list cannot be None + dir_name, file_name = os.path.split(model_path) + for var in var_list: + loaded_var_list.append(clone_var_to_block(load_block, var)) + _load_vars_with_try_catch( + exe, dir_name, loaded_var_list, file_name + ) + else: + # var_list can be None or not None + if var_list is not None: + for var in var_list: + loaded_var_list.append( + clone_var_to_block(load_block, var) + ) + _load_vars_with_try_catch( + exe, model_path, loaded_var_list, None + ) + else: + for var_name in var_name_list: + # NOTE(chenweihang): If identify which files the user wants + # to load from the disk, we load these variables one by one. + # If a file does not exist, we only warn the user that the + # file may be an irrelevant file, but does not throw an error + # to ensure that other legal variables can be loaded. + temp_var = load_block.create_var( + name=var_name, persistable=True + ) + if _load_vars_with_try_catch( + exe, model_path, [temp_var], None, False + ): + loaded_var_list.append(temp_var) + + res_dict = {} + for var in loaded_var_list: + res_dict[var.name] = np.asarray( + paddle.fluid.global_scope().find_var(var.name).get_tensor() + ) + + return res_dict + + assert os.path.exists( + parameter_file_name + ), "Parameter file [{}] not exits".format(parameter_file_name) + + with open(parameter_file_name, 'rb') as f: + # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' + if sys.platform == 'darwin' and sys.version_info.major == 3: + para_dict = _pickle_loads_mac(parameter_file_name, f) + else: + para_dict = pickle.load(f, encoding='latin1') + para_dict = _pack_loaded_dict(para_dict) + + opt_file_name = model_prefix + ".pdopt" + if os.path.exists(opt_file_name): + with open(opt_file_name, 'rb') as f: + opti_dict = pickle.load(f, encoding='latin1') + + para_dict.update(opti_dict) + + return para_dict diff --git a/python/paddle/static/quantization/tests/test_graph.py b/python/paddle/static/quantization/tests/test_graph.py index 0704990df6273764dd8f0dab6afc77eee139349b..189ca7cfcae02258a099baf4b3f18ea78adc50e9 100644 --- a/python/paddle/static/quantization/tests/test_graph.py +++ b/python/paddle/static/quantization/tests/test_graph.py @@ -15,8 +15,6 @@ import os import unittest -import numpy as np - import paddle from paddle.fluid.framework import IrGraph from paddle.framework import core @@ -103,40 +101,6 @@ class TestGraph(unittest.TestCase): _train(origin_binary) _train(backup_binary) - checkponit_dir = "checkpoint_gpu" if use_cuda else "checkpoint_cpu" - - def _set_zero(var_name, scope, place): - var = scope.find_var(var_name).get_tensor() - var_array = np.zeros(var._get_dims()).astype("float32") - var.set(var_array, place) - - sum_before = np.sum( - np.array( - paddle.static.global_scope() - .find_var('conv2d_1.w_0') - .get_tensor() - ) - ) - paddle.fluid.io._save_persistable_nodes(exe, checkponit_dir, graph) - _set_zero('conv2d_1.w_0', paddle.static.global_scope(), place) - set_after = np.sum( - np.array( - paddle.static.global_scope() - .find_var('conv2d_1.w_0') - .get_tensor() - ) - ) - self.assertEqual(set_after, 0) - paddle.fluid.io._load_persistable_nodes(exe, checkponit_dir, graph) - sum_after = np.sum( - np.array( - paddle.static.global_scope() - .find_var('conv2d_1.w_0') - .get_tensor() - ) - ) - self.assertEqual(sum_before, sum_after) - marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('conv2d') > -1: