Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 2 年 前同步成功

通知 2325
Star 20933
Fork 5424
P
Paddle
“2e9d9dc9a7efcf23357fb71eb2cfe07b4e28895b”上不存在“...librispeech/git@gitcode.net:paddlepaddle/DeepSpeech.git”
未验证 提交 f8d0a358 编写于 作者: Y YuanRisheng 提交者： GitHub
浏览文件
操作

[Fluid Clean]Move and Remove some io API in Fluid (#50842) 

* move io

* fix ci bugs

* fix ci bugs

* fix py3 bugs

* fix example code

* fix example code

* fix text

* fix text

* deal with ci bugs

* perfect code according comment

* delete import batch
上级 25a77450
隐藏空白更改
内联 并排
Showing with 1669 addition and 2319 deletion
python/paddle/distributed/auto_parallel/utils.py
浏览文件 @ f8d0a358
......@@ -22,8 +22,8 @@ from functools import reduce
import numpy as np
import paddle
from paddle.fluid.io import is_belong_to_optimizer, is_parameter
from paddle.framework import core
from paddle.framework.io_utils import is_belong_to_optimizer, is_parameter
from paddle.static import Variable
from .dist_attribute import OperatorDistAttr, TensorDistAttr
......
python/paddle/distributed/fleet/runtime/parameter_server_runtime.py
浏览文件 @ f8d0a358
......@@ -109,7 +109,6 @@ class ParameterServerRuntime(RuntimeBase):
assert isinstance(each_var, Variable)
origin_varname, _, _ = _get_varname_parts(each_var.name)
new_var = paddle.static.io._clone_var_in_block(load_block, each_var)
var_path = os.path.join(dirname, origin_varname)
if not os.path.exists(var_path):
......
python/paddle/distributed/io.py
浏览文件 @ f8d0a358
......@@ -16,10 +16,175 @@ import os
import paddle
from paddle.fluid.framework import Program, static_only
from paddle.fluid.io import load_persistables
from paddle.framework import core, dygraph_not_support
def _load_distributed_persistables(executor, dirname, main_program=None):
"""
customized load_persistables for distributed training.
it should be used on parameter server,
Args:
executor(Executor): The executor to run for saving parameters.
dirname(str): The load directory path.
main_program(Program): The program whose parameters will be
loaded. the main_program must be the pserver_program
get after transpiler.
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
exe = fluid.Executor(fluid.CPUPlace())
param_path = "./my_paddle_model"
t = paddle.distributed.transpiler.DistributeTranspiler()
t.transpile(...)
pserver_prog = t.get_pserver_program(...)
_load_distributed_persistables(executor=exe, dirname=param_path, main_program=pserver_prog)
"""
def __is_distributed_part_var(varname):
trainer_idx = varname.find(".trainer_")
block_idx = varname.find(".block")
return trainer_idx or block_idx
def __load_persistable_vars(executor, dirname, need_load_vars):
load_prog = Program()
load_block = load_prog.global_block()
need_delete_vars = []
for param in need_load_vars:
origin_var = param.origin
slice_var = param.slice
is_slice = param.is_slice
offset = param.offset
if is_slice:
slice = load_block.create_var(
name=slice_var.name,
type=slice_var.type,
shape=slice_var.shape,
dtype=slice_var.dtype,
persistable=True,
)
load_block.append_op(
type='load',
inputs={},
outputs={'Out': [slice]},
attrs={
'file_path': os.path.join(dirname, origin_var.name),
'seek': offset,
'shape': slice.shape,
},
)
else:
origin = load_block.create_var(
name="{}".format(origin_var.name),
type=origin_var.type,
shape=origin_var.shape,
dtype=origin_var.dtype,
persistable=True,
)
load_block.append_op(
type='load',
inputs={},
outputs={'Out': [origin]},
attrs={'file_path': os.path.join(dirname, origin_var.name)},
)
load_block.append_op(
type='delete_var',
inputs={'X': need_delete_vars},
)
executor.run(load_prog)
if not isinstance(main_program, Program):
raise TypeError("'main_program' should be an instance of Program.")
if not main_program._is_distributed:
raise ValueError(
"'_load_distributed_persistables' just be designed for distributed training."
)
if not main_program._ps_endpoint:
raise ValueError(
"'_load_distributed_persistables' need current_endpoint set in DistributeTranspiler.transpile"
)
need_load_vars = (
main_program._parameters_on_pservers.get_distributed_vars_by_ep(
main_program._ps_endpoint
)
)
__load_persistable_vars(executor, dirname, need_load_vars)
@dygraph_not_support
def load_persistables(executor, dirname, main_program=None, filename=None):
"""
:api_attr: Static Graph
This API filters out all variables with ``persistable==True`` from the
given ``main_program`` and then tries to load these variables from the
directory ``dirname`` or the file ``filename``.
Use the ``dirname`` to specify the directory where persistable variables
(refer to :ref:`api_guide_model_save_reader_en`) were saved. If variables
were saved in separate files, set ``filename`` as None; if all variables
were saved in a single file, use ``filename`` to specify the file name.
Args:
executor(Executor): The executor used for loading persistable variables.
See :ref:`api_guide_executor_en` for more details about it.
dirname(str): The directory path.
main_program(Program, optional): The program whose persistable variables will
be loaded. If it is None, the ``default_main_program``
will be used automatically. See :ref:`api_guide_Program_en`
for more about ``Program``.
Default: None.
filename(str, optional): The file which saved all persistable variables. If variables
were saved in separated files, set it to None.
Default: None.
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
exe = fluid.Executor(fluid.CPUPlace())
param_path = "./my_paddle_model"
prog = fluid.default_main_program()
paddle.distributed.io.load_persistables(executor=exe, dirname=param_path,
main_program=None)
"""
if main_program and main_program._is_distributed:
_load_distributed_persistables(
executor, dirname=dirname, main_program=main_program
)
else:
paddle.static.io.load_vars(
executor,
dirname=dirname,
main_program=main_program,
predicate=is_persistable,
filename=filename,
)
def _save_distributed_persistables(executor, dirname, main_program):
"""
save_persistables for distributed training.
......
python/paddle/fluid/__init__.py
浏览文件 @ f8d0a358
......@@ -89,7 +89,6 @@ from .compiler import *
from paddle.fluid.layers.math_op_patch import monkey_patch_variable
from .dygraph.layers import *
from .dygraph.base import enable_dygraph, disable_dygraph
from .io import save, load, load_program_state, set_program_state
from .dygraph.varbase_patch_methods import monkey_patch_varbase
from .core import _cuda_synchronize
from .trainer_desc import (
......@@ -146,8 +145,6 @@ __all__ = (
'profiler',
'unique_name',
'Scope',
'save',
'load',
'_cuda_synchronize',
]
)
......
python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py
浏览文件 @ f8d0a358
......@@ -34,7 +34,7 @@ class PaddleModel(SerializableBase):
self._file_name = "_paddle_fleet_param__"
def serialize(self, path):
from ...io import save_persistables
from paddle.distributed.io import save_persistables
save_persistables(
executor=self._exe,
......@@ -44,7 +44,7 @@ class PaddleModel(SerializableBase):
)
def deserialize(self, path):
from ...io import load_persistables
from paddle.distributed.io import load_persistables
load_persistables(
executor=self._exe,
......
python/paddle/fluid/io.py
浏览文件 @ f8d0a358
......@@ -62,1205 +62,15 @@ from . import core
from paddle.utils import deprecated
from paddle.fluid.framework import static_only
batch = paddle.batch
__all__ = [
'save_vars',
'save_params',
'save_persistables',
'load_vars',
'load_params',
'load_persistables',
'save_inference_model',
'load_inference_model',
'batch',
'save',
'load',
'load_program_state',
'set_program_state',
'get_program_parameter',
'get_program_persistable_vars',
] + reader.__all__
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
class _open_buffer:
def __init__(self, buffer):
self.buffer = buffer
def __enter__(self):
return self.buffer
class _buffer_reader(_open_buffer):
def __init__(self, buffer):
super().__init__(buffer)
self.initial_tell = self.buffer.tell()
def __exit__(self, *args):
# `args[0]` is type of exception. When the `read` is abnormal, the file pointer returns to the initial position.
if args[0] is not None:
self.buffer.seek(self.initial_tell)
class _buffer_writer(_open_buffer):
def __exit__(self, *args):
self.buffer.flush()
def _is_file_path(path):
return isinstance(path, str)
def _open_file_buffer(path_or_buffer, mode):
if _is_file_path(path_or_buffer):
return open(path_or_buffer, mode)
else:
if 'w' in mode:
return _buffer_writer(path_or_buffer)
elif 'r' in mode:
return _buffer_reader(path_or_buffer)
else:
raise ValueError(
"Expected 'r' or 'w' in mode but got {}".format(mode)
)
def _is_memory_buffer(buffer):
return isinstance(buffer, BytesIO)
def is_parameter(var):
"""
Check whether the given variable is an instance of Parameter.
Args:
var(Variable): The variable to be checked.
Returns:
bool: True if the given `var` is an instance of Parameter,
False if not.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
param = fluid.default_main_program().global_block().var('fc.w')
res = fluid.io.is_parameter(param)
"""
return isinstance(var, Parameter)
def is_persistable(var):
"""
Check whether the given variable is persistable.
Args:
var(Variable): The variable to be checked.
Returns:
bool: True if the given `var` is persistable
False if not.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
param = fluid.default_main_program().global_block().var('fc.b')
res = fluid.io.is_persistable(param)
"""
if (
var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH
or var.desc.type() == core.VarDesc.VarType.FETCH_LIST
or var.desc.type() == core.VarDesc.VarType.READER
):
return False
return var.persistable
def is_belong_to_optimizer(var):
if not (isinstance(var, Parameter) or var.desc.need_check_feed()):
return is_persistable(var)
return False
@dygraph_not_support
def get_program_parameter(program):
"""
Get all the parameters from Program.
Args:
var(Program): The Program to get parameters
Returns:
list: The list contains all parameters in the program
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
data = fluid.data(name="img", shape=[64, 784])
w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
list_para = fluid.io.get_program_parameter( fluid.default_main_program() )
"""
return list(filter(is_parameter, program.list_vars()))
@dygraph_not_support
def get_program_persistable_vars(program):
"""
Get all the persistable vars from Program.
Args:
var(Program): The Program to get persistable vars
Returns:
list: The list contains all persistable vars in the program
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
data = fluid.data(name="img", shape=[64, 784])
w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
list_para = fluid.io.get_program_persistable_vars( fluid.default_main_program() )
"""
return list(filter(is_persistable, program.list_vars()))
def _clone_var_in_block_(block, var):
assert isinstance(var, Variable)
if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR:
return block.create_var(
name=var.name,
shape=var.shape,
dtype=var.dtype,
type=var.type,
lod_level=var.lod_level,
persistable=True,
)
else:
return block.create_var(
name=var.name,
shape=var.shape,
dtype=var.dtype,
type=var.type,
persistable=True,
)
@signature_safe_contextmanager
def _load_program_scope(main=None, startup=None, scope=None):
prog = main if main else paddle.fluid.Program()
startup_prog = startup if startup else paddle.fluid.Program()
scope = scope if scope else paddle.fluid.core.Scope()
with paddle.fluid.scope_guard(scope):
with paddle.fluid.program_guard(prog, startup_prog):
with paddle.fluid.unique_name.guard():
with paddle.fluid.framework._dygraph_guard(None):
yield
def _get_valid_program(main_program=None):
if main_program is None:
main_program = default_main_program()
elif isinstance(main_program, CompiledProgram):
main_program = main_program._program
if main_program is None:
raise TypeError(
"The type of input main_program is invalid, expected tyep is Program, but received None"
)
warnings.warn(
"The input is a CompiledProgram, this is not recommended."
)
if not isinstance(main_program, Program):
raise TypeError(
"The type of input main_program is invalid, expected type is fluid.Program, but received %s"
% type(main_program)
)
return main_program
@dygraph_not_support
def save_vars(
executor,
dirname,
main_program=None,
vars=None,
predicate=None,
filename=None,
):
"""
Save specific variables in the `Program` to files.
There are two ways to specify the variables to be saved: set variables in
a list and assign it to the `vars`, or use the `predicate` function to select
variables that make `predicate(variable) == True`. The first way has a higher priority.
The `dirname` is used to specify the folder where to save variables.
If you prefer to save variables in separate files in the `dirname` folder,
do not set `filename`. If you prefer to save all variables in a single file,
use `filename` to specify it.
Args:
executor(Executor): The executor to run for saving variables.
dirname(str, optional): The folder where to save variables.
When you need to save the parameter to the memory, set it to None.
main_program(Program, optional): The program whose variables will be saved.
If it is None, the default main program will
be used automatically.
Default: None
vars(list[Variable], optional): The list contains all variables to be saved.
Default: None
predicate(function, optional): The function selects the variables that make
`predicate(variable) == True`.
Default: None
filename(str, optional): If you prefer to save all variables in a single file,
use `filename` to specify it. Otherwise, let `filename` be None.
Default: None
Returns:
str: When saving parameters to a file, returns None.
When saving parameters to memory, returns a binary string containing parameters.
Raises:
TypeError: If `main_program` is not an instance of Program nor None.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
main_prog = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(main_prog, startup_prog):
data = paddle.static.data(name="img", shape=[64, 784])
w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
hidden_w = paddle.matmul(x=data, y=w)
hidden_b = paddle.add(hidden_w, b)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup_prog)
# The first usage: use `vars` to set the saved variables.
var_list = [w, b]
path = "./my_paddle_vars"
fluid.io.save_vars(executor=exe, dirname=path, vars=var_list,
filename="vars_file")
# w and b will be save in a file named "var_file".
# The second usage: use `predicate` to select the saved variable.
def name_has_fc(var):
res = "fc" in var.name
return res
param_path = "./my_paddle_model"
fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog, vars=None, predicate = name_has_fc)
# all variables whose names contain "fc " are saved.
"""
save_to_memory = False
if dirname is None and filename is None:
save_to_memory = True
main_program = _get_valid_program(main_program)
if vars is None:
return save_vars(
executor,
main_program=main_program,
dirname=dirname,
vars=list(filter(predicate, main_program.list_vars())),
filename=filename,
)
else:
params_var_name = "saved_params"
# give warning when there is no var in model
if len(list(vars)) == 0:
warnings.warn(
"no variable in your model, please ensure there are any variables in your model to save"
)
return None
save_program = Program()
save_block = save_program.global_block()
save_var_map = {}
for each_var in vars:
# NOTE: don't save the variable which type is RAW
if each_var.type == core.VarDesc.VarType.RAW:
continue
new_var = _clone_var_in_block_(save_block, each_var)
if filename is None and save_to_memory is False:
save_file_path = os.path.join(
os.path.normpath(dirname), new_var.name
)
save_block.append_op(
type='save',
inputs={'X': [new_var]},
outputs={},
attrs={'file_path': os.path.normpath(save_file_path)},
)
else:
save_var_map[new_var.name] = new_var
if filename is not None or save_to_memory:
save_var_list = []
for name in sorted(save_var_map.keys()):
save_var_list.append(save_var_map[name])
save_path = str()
if save_to_memory is False:
save_path = os.path.join(os.path.normpath(dirname), filename)
saved_params = save_block.create_var(
type=core.VarDesc.VarType.RAW, name=params_var_name
)
saved_params.desc.set_persistable(True)
save_block.append_op(
type='save_combine',
inputs={'X': save_var_list},
outputs={'Y': saved_params},
attrs={
'file_path': save_path,
'save_to_memory': save_to_memory,
},
)
# NOTE(zhiqiu): save op will add variable kLookupTablePath in save_program.desc,
# which leads to diff on save_program and its desc. Call _sync_with_cpp
# to keep consistency.
save_program._sync_with_cpp()
executor.run(save_program)
if save_to_memory:
return global_scope().find_var(params_var_name).get_bytes()
@dygraph_not_support
def save_params(executor, dirname, main_program=None, filename=None):
"""
Save all parameters from the :code:`main_program` to
the folder :code:`dirname` or file :code:`filename`. You can refer to
:ref:`api_guide_model_save_reader_en` for more details.
Use the :code:`dirname` to specify the saving folder. If you would like to
save parameters in separate files, set :code:`filename` None; if you would
like to save all parameters in a single file, use :code:`filename` to specify
the file name.
Note:
Some variables are not Parameter while they are necessary for
training, such as learning rate, global step, etc. So you can NOT save
and continue your training just by :ref:`api_fluid_io_save_params`
and :ref:`api_fluid_io_load_params`. Please use :ref:`api_fluid_io_save_persistables`
and :ref:`api_fluid_io_load_persistables` instead.
If you want to save your model for the inference, please use the
:ref:`api_fluid_io_save_inference_model`. You can refer to
:ref:`api_guide_model_save_reader_en` for more details.
Args:
executor(Executor): The executor to run for saving parameters, You can
refer to :ref:`api_guide_executor_en`.
dirname(str, optional): The saving directory path.
When you need to save the parameter to the memory, set it to None.
main_program(Program, optional): The program whose parameters will be
saved. You can refer to
:ref:`api_guide_Program_en` for more
details. If it is None, the default main
program will be used.
Default: None
filename(str, optional): The file to save all parameters. If you prefer
to save parameters in different files, set it
to None.
Default: None
Returns:
str: When saving parameters to a file, returns None.
When saving parameters to memory, returns a binary string containing parameters.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
params_path = "./my_paddle_model"
image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
feeder = fluid.DataFeeder(feed_list=[image, label], place=fluid.CPUPlace())
predict = paddle.static.nn.fc(x=image, size=10, activation='softmax')
loss = paddle.nn.functional.cross_entropy(
input=predict, label=label,
reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
fluid.io.save_params(executor=exe, dirname=params_path)
# The parameters weights and bias of the fc layer in the network are going to
# be saved in different files in the path "./my_paddle_model"
"""
return save_vars(
executor,
dirname=dirname,
main_program=main_program,
vars=None,
predicate=is_parameter,
filename=filename,
)
def _save_distributed_persistables(executor, dirname, main_program):
"""
save_persistables for distributed training.
the method will do things listed below:
1.save part of persistable variables on trainer.
2.receive "remote prefetch variables" from parameter servers and merge them.
3.save "distributed lookup table" on parameter servers.
4.receive "optimizer variables" from parameter servers and merge them.
Args:
executor(Executor): The executor to run for saving parameters.
dirname(str): The saving directory path.
main_program(Program): The program whose parameters will be
saved. the main_program must be the trainer_program
get after transpiler.
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
exe = fluid.Executor(fluid.CPUPlace())
param_path = "./my_paddle_model"
t = paddle.distributed.transpiler.DistributeTranspiler()
t.transpile(...)
train_program = t.get_trainer_program()
_save_distributed_persistables(executor=exe, dirname=param_path, main_program=train_program)
"""
def __save_remote_params(executor, dirname, remote_params_map):
"""
receive params on pserver through rpc.
if the params are be sliced, will concat them to one, then save it.
"""
if not remote_params_map:
return
prog = Program()
block = prog.global_block()
# recv optimize vars from pserver
for name, remote_params in remote_params_map.items():
origin = remote_params[0].origin
is_slice = remote_params[0].is_slice
slices = [None] * len(remote_params)
slice_varnames = [None] * len(remote_params)
remote_varnames = [None] * len(remote_params)
endpoints = [None] * len(remote_params)
for idx, optimizer in enumerate(remote_params):
block_id = optimizer.block_id
slice = optimizer.slice
endpoint = optimizer.endpoint
index = block_id if is_slice else idx
slices[index] = slice
slice_varnames[index] = "{}.slice.{}".format(slice.name, idx)
remote_varnames[index] = slice.name
endpoints[index] = endpoint
slice_shapes = []
for slice in slices:
tmp = [str(dim) for dim in slice.shape]
slice_shapes.append(",".join(tmp))
block.append_op(
type='recv_save',
attrs={
"trainer_id": 0,
"shape": origin.shape,
"slice_shapes": slice_shapes,
"slice_varnames": slice_varnames,
"remote_varnames": remote_varnames,
"endpoints": endpoints,
"file_path": os.path.join(dirname, origin.name),
},
)
executor.run(prog)
def __save_distributed_lookup_tables(
executor, dirname, distributed_lookup_table, endpoints
):
"""
because the distributed lookup table may too huge to merge and save at one place,
it will be saved at parameter server independent respectively.
the save directory is dirname/"__lookup_table__".
"""
prog = Program()
block = prog.global_block()
# if there is lookup table, the trainer 0 will notify all pserver to save.
lookup_table_filename = os.path.join(dirname, "__lookup_table__")
attrs = {}
attrs['epmap'] = endpoints
attrs['dir'] = lookup_table_filename
attrs['lookup_table'] = distributed_lookup_table
block.append_op(
type='checkpoint_notify', inputs={}, outputs={}, attrs=attrs
)
executor.run(prog)
def __exclude_vars(exclude_var_names=[]):
def is_valid(var):
if var.name in exclude_var_names:
return False
if (
var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH
or var.desc.type() == core.VarDesc.VarType.FETCH_LIST
or var.desc.type() == core.VarDesc.VarType.READER
):
return False
return var.persistable
return is_valid
if not isinstance(main_program, Program):
raise TypeError("'main_program' should be an instance of Program.")
if not main_program._is_distributed:
raise ValueError(
"'_save_distributed_persistables' just be designed for distributed training."
)
remote_params_map = (
main_program._parameters_on_pservers.get_distributed_vars_by_vtypes(
["Optimizer", "RemotePrefetch"], groupby=True
)
)
exclude_var_names = []
if remote_params_map:
exclude_var_names.extend(remote_params_map.keys())
if main_program._distributed_lookup_table:
if isinstance(main_program._distributed_lookup_table, list):
exclude_var_names.extend(main_program._distributed_lookup_table)
else:
exclude_var_names.append(main_program._distributed_lookup_table)
local_vars = list(
filter(__exclude_vars(exclude_var_names), main_program.list_vars())
)
save_vars(
executor, main_program=main_program, dirname=dirname, vars=local_vars
)
if main_program._is_chief:
if remote_params_map:
__save_remote_params(executor, dirname, remote_params_map)
if main_program._distributed_lookup_table:
__save_distributed_lookup_tables(
executor,
dirname,
main_program._distributed_lookup_table,
main_program._endpoints,
)
@dygraph_not_support
def save_persistables(executor, dirname, main_program=None, filename=None):
"""
Save all persistable variables from :code:`main_program` to
the folder :code:`dirname` or file :code:`filename`. You can refer to
:ref:`api_guide_model_save_reader_en` for more details. And then
saves these persistables variables to the folder :code:`dirname` or file
:code:`filename`.
The :code:`dirname` is used to specify the folder where persistable variables
are going to be saved. If you would like to save variables in separate
files, set :code:`filename` None; if you would like to save all variables in a
single file, use :code:`filename` to specify the file name.
Args:
executor(Executor): The executor to run for saving persistable variables.
You can refer to :ref:`api_guide_executor_en` for
more details.
dirname(str, optional): The saving directory path.
When you need to save the parameter to the memory, set it to None.
main_program(Program, optional): The program whose persistbale variables will
be saved. You can refer to
:ref:`api_guide_Program_en` for more details.
If it is None, the default main program will
be used.
Default: None.
filename(str, optional): The file to save all variables. If you prefer to
save variables in different files, set it to None.
Default: None.
Returns:
str: When saving parameters to a file, returns None.
When saving parameters to memory, returns a binary string containing parameters.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
dir_path = "./my_paddle_model"
file_name = "persistables"
image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
feeder = fluid.DataFeeder(feed_list=[image, label], place=fluid.CPUPlace())
predict = paddle.static.nn.fc(x=image, size=10, activation='softmax')
loss = paddle.nn.functional.cross_entropy(
input=predict, label=label,
reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
fluid.io.save_persistables(executor=exe, dirname=dir_path, filename=file_name)
# The persistables variables weights and bias in the fc layer of the network
# are going to be saved in the same file named "persistables" in the path
# "./my_paddle_model"
"""
if main_program and main_program._is_distributed:
return _save_distributed_persistables(
executor, dirname=dirname, main_program=main_program
)
else:
return save_vars(
executor,
dirname=dirname,
main_program=main_program,
vars=None,
predicate=is_persistable,
filename=filename,
)
def load_vars(
executor,
dirname,
main_program=None,
vars=None,
predicate=None,
filename=None,
):
"""
:api_attr: Static Graph
This API loads variables from files by executor.
There are two ways to specify the variables to be loaded: the first way, set
variables in a list and assign it to the `vars`; the second way, use the
`predicate` function to select variables that make `predicate(variable) == True`.
The first way has a higher priority.
The `dirname` is used to specify the folder where to load variables.
If variables were saved in separate files in the folder `dirname`,
set `filename` None. If all variables were saved in a single file,
use `filename` to specify it.
Args:
executor(Executor): The executor to run for loading variables.
dirname(str): The folder where to load the variables.
main_program(Program, optional): The program whose variables will be loaded.
If it is None, the default main program will
be used automatically.
Default: None
vars(list[Variable], optional): The list that contains all variables to be loaded.
Default: None
predicate(function, optional): The function selects variables that make
`predicate(variable) == True`.
Default: None
filename(str, optional): The file which saved all required variables. If variables
were saved in separate files, set it to be None.
Default: None
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
main_prog = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(main_prog, startup_prog):
data = paddle.static.data(name="img", shape=[64, 784])
w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
hidden_w = paddle.matmul(x=data, y=w)
hidden_b = paddle.add(hidden_w, b)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup_prog)
# The first usage: using `vars` to specify the variables.
path = "./my_paddle_vars"
var_list = [w, b]
fluid.io.save_vars(executor=exe, dirname=path, vars=var_list,
filename="vars_file")
fluid.io.load_vars(executor=exe, dirname=path, vars=var_list,
filename="vars_file")
# w and b will be loaded, and they are supposed to
# be saved in the same file named 'var_file' in the path "./my_paddle_vars".
# The second usage: using the `predicate` function to select variables
param_path = "./my_paddle_model"
def name_has_fc(var):
res = "fc" in var.name
return res
fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog,
vars=None, predicate=name_has_fc)
fluid.io.load_vars(executor=exe, dirname=param_path, main_program=main_prog,
vars=None, predicate=name_has_fc)
# Load All variables in the `main_program` whose name includes "fc".
# And all the variables are supposed to be saved in separate files.
"""
vars_from_memory = False
if dirname is not None:
dirname = os.path.normpath(dirname)
else:
vars_from_memory = True
if vars is None:
if main_program is None:
main_program = default_main_program()
if not isinstance(main_program, Program):
raise TypeError(
"The type of input main_program is invalid, expected type is fluid.Program, but received %s"
% type(main_program)
)
load_vars(
executor,
dirname=dirname,
main_program=main_program,
vars=list(filter(predicate, main_program.list_vars())),
filename=filename,
)
else:
load_prog = Program()
load_block = load_prog.global_block()
if main_program is None:
main_program = default_main_program()
if not isinstance(main_program, Program):
raise TypeError(
"The type of input main_program is invalid, expected type is fluid.Program, but received %s"
% type(main_program)
)
# save origin param shape
orig_para_shape = {}
load_var_map = {}
check_vars = []
sparse_vars = []
for each_var in vars:
assert isinstance(each_var, Variable)
if each_var.type == core.VarDesc.VarType.RAW:
continue
if isinstance(each_var, Parameter):
orig_para_shape[each_var.name] = tuple(
each_var.desc.get_shape()
)
if each_var.type == core.VarDesc.VarType.SELECTED_ROWS:
sparse_vars.append(each_var)
continue
new_var = _clone_var_in_block_(load_block, each_var)
check_vars.append(each_var)
if filename is None:
if dirname is None:
raise ValueError(
"The directory path and params cannot be None at the same time."
)
load_block.append_op(
type='load',
inputs={},
outputs={'Out': [new_var]},
attrs={'file_path': os.path.join(dirname, new_var.name)},
)
else:
load_var_map[new_var.name] = new_var
for each_var in sparse_vars:
assert isinstance(each_var, Variable)
if filename is not None:
raise ValueError(
"SelectedRows can not be load with load_combine"
)
new_var = _clone_var_in_block_(load_block, each_var)
var_path = os.path.join(dirname, new_var.name)
if not os.path.exists(var_path):
raise ValueError(
"SelectedRows var {} can not find at {}".format(
new_var.name, var_path
)
)
if os.path.isfile(var_path):
load_block.append_op(
type='load',
inputs={},
outputs={'Out': [new_var]},
attrs={'file_path': os.path.join(dirname, new_var.name)},
)
else:
blocks = []
block_paths = os.listdir(var_path)
for block in block_paths:
if block.startswith(new_var.name):
blocks.append(block)
slices = []
for block in blocks:
slice = load_block.create_var(
name=block,
type=new_var.type,
shape=new_var.shape,
dtype=new_var.dtype,
persistable=False,
)
slices.append(slice)
file_path = os.path.join(var_path, block, "Param")
load_block.append_op(
type='load',
inputs={},
outputs={'Out': [slice]},
attrs={'file_path': file_path},
)
load_block.append_op(
type='lookup_sparse_table_merge',
inputs={'X': slices},
outputs={'Out': new_var},
attrs={},
)
if filename is not None:
load_var_list = []
for name in sorted(load_var_map.keys()):
load_var_list.append(load_var_map[name])
if vars_from_memory is False:
filename = os.path.join(dirname, filename)
load_block.append_op(
type='load_combine',
inputs={},
outputs={"Out": load_var_list},
attrs={
'file_path': filename,
'model_from_memory': vars_from_memory,
},
)
executor.run(load_prog)
# check var shape
for each_var in check_vars:
if not isinstance(each_var, Parameter):
continue
var_temp = paddle.fluid.global_scope().find_var(each_var.name)
assert var_temp is not None, "can't not find var: " + each_var.name
new_shape = (np.array(var_temp.get_tensor())).shape
assert each_var.name in orig_para_shape, (
each_var.name + "MUST in var list"
)
orig_shape = orig_para_shape.get(each_var.name)
if new_shape != orig_shape:
raise RuntimeError(
"Variable's shape does not match, the Program requires a parameter with the shape of ({}), "
"while the loaded parameter (namely [ {} ]) has a shape of ({}).".format(
orig_shape, each_var.name, new_shape
)
)
@dygraph_not_support
def load_params(executor, dirname, main_program=None, filename=None):
"""
:api_attr: Static Graph
This API filters out all parameters from the give ``main_program``
and then tries to load these parameters from the directory ``dirname`` or
the file ``filename``.
Use the ``dirname`` to specify the directory where parameters were saved. If
parameters were saved in separate files under the directory `dirname`, set
``filename`` as None; if all parameters were saved in a single file, use
``filename`` to specify the file name.
**Note**:
Some variables are not Parameter while they are necessary for
training, such as learning rate, global step, etc. So you cannot save and
continue your training just by using :ref:`api_fluid_io_save_params` and
:ref:`api_fluid_io_load_params`. Please use :ref:`api_fluid_io_save_persistables`
and :ref:`api_fluid_io_load_persistables` instead.
If you want to load the pre-trained model structure and parameters
for the inference, please use the :ref:`api_fluid_io_load_inference_model` API. You can
refer to :ref:`api_guide_model_save_reader_en` for more details.
Args:
executor(Executor): The executor used for loading parameters.
See :ref:`api_guide_executor_en` for more details about it.
dirname(str): The directory path.
main_program(Program, optional): The program whose parameters will be
loaded. If it is None, the ``default_main_program``
will be used automatically. See :ref:`api_guide_Program_en`
for more about ``Program``.
Default: None.
filename(str, optional): The file which saved all parameters. If parameters
were saved in separated files, set it to None.
Default: None.
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
exe = fluid.Executor(fluid.CPUPlace())
param_path = "./my_paddle_model"
prog = fluid.default_main_program()
fluid.io.load_params(executor=exe, dirname=param_path,
main_program=None)
"""
load_vars(
executor,
dirname=dirname,
main_program=main_program,
predicate=is_parameter,
filename=filename,
)
@dygraph_not_support
def load_persistables(executor, dirname, main_program=None, filename=None):
"""
:api_attr: Static Graph
This API filters out all variables with ``persistable==True`` from the
given ``main_program`` and then tries to load these variables from the
directory ``dirname`` or the file ``filename``.
Use the ``dirname`` to specify the directory where persistable variables
(refer to :ref:`api_guide_model_save_reader_en`) were saved. If variables
were saved in separate files, set ``filename`` as None; if all variables
were saved in a single file, use ``filename`` to specify the file name.
Args:
executor(Executor): The executor used for loading persistable variables.
See :ref:`api_guide_executor_en` for more details about it.
dirname(str): The directory path.
main_program(Program, optional): The program whose persistable variables will
be loaded. If it is None, the ``default_main_program``
will be used automatically. See :ref:`api_guide_Program_en`
for more about ``Program``.
Default: None.
filename(str, optional): The file which saved all persistable variables. If variables
were saved in separated files, set it to None.
Default: None.
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
exe = fluid.Executor(fluid.CPUPlace())
param_path = "./my_paddle_model"
prog = fluid.default_main_program()
fluid.io.load_persistables(executor=exe, dirname=param_path,
main_program=None)
"""
if main_program and main_program._is_distributed:
_load_distributed_persistables(
executor, dirname=dirname, main_program=main_program
)
else:
load_vars(
executor,
dirname=dirname,
main_program=main_program,
predicate=is_persistable,
filename=filename,
)
def _load_distributed_persistables(executor, dirname, main_program=None):
"""
customized load_persistables for distributed training.
it should be used on parameter server,
Args:
executor(Executor): The executor to run for saving parameters.
dirname(str): The load directory path.
main_program(Program): The program whose parameters will be
loaded. the main_program must be the pserver_program
get after transpiler.
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
exe = fluid.Executor(fluid.CPUPlace())
param_path = "./my_paddle_model"
t = paddle.distributed.transpiler.DistributeTranspiler()
t.transpile(...)
pserver_prog = t.get_pserver_program(...)
_load_distributed_persistables(executor=exe, dirname=param_path, main_program=pserver_prog)
"""
def __is_distributed_part_var(varname):
trainer_idx = varname.find(".trainer_")
block_idx = varname.find(".block")
return trainer_idx or block_idx
def __load_persistable_vars(executor, dirname, need_load_vars):
load_prog = Program()
load_block = load_prog.global_block()
need_delete_vars = []
for param in need_load_vars:
origin_var = param.origin
slice_var = param.slice
is_slice = param.is_slice
offset = param.offset
if is_slice:
slice = load_block.create_var(
name=slice_var.name,
type=slice_var.type,
shape=slice_var.shape,
dtype=slice_var.dtype,
persistable=True,
)
load_block.append_op(
type='load',
inputs={},
outputs={'Out': [slice]},
attrs={
'file_path': os.path.join(dirname, origin_var.name),
'seek': offset,
'shape': slice.shape,
},
)
else:
origin = load_block.create_var(
name="{}".format(origin_var.name),
type=origin_var.type,
shape=origin_var.shape,
dtype=origin_var.dtype,
persistable=True,
)
load_block.append_op(
type='load',
inputs={},
outputs={'Out': [origin]},
attrs={'file_path': os.path.join(dirname, origin_var.name)},
)
load_block.append_op(
type='delete_var',
inputs={'X': need_delete_vars},
)
executor.run(load_prog)
if not isinstance(main_program, Program):
raise TypeError("'main_program' should be an instance of Program.")
if not main_program._is_distributed:
raise ValueError(
"'_load_distributed_persistables' just be designed for distributed training."
)
__all__ = [
'save_inference_model',
'load_inference_model',
] + reader.__all__
if not main_program._ps_endpoint:
raise ValueError(
"'_load_distributed_persistables' need current_endpoint set in DistributeTranspiler.transpile"
)
need_load_vars = (
main_program._parameters_on_pservers.get_distributed_vars_by_ep(
main_program._ps_endpoint
)
)
__load_persistable_vars(executor, dirname, need_load_vars)
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
def prepend_feed_ops(
......@@ -1430,7 +240,7 @@ def save_inference_model(
):
raise ValueError("'target_vars' should be a list of Variable.")
main_program = _get_valid_program(main_program)
main_program = paddle.static.io._get_valid_program(main_program)
# remind user to set auc_states to zeros if the program contains auc op
all_ops = main_program.global_block().ops
......@@ -1534,7 +344,9 @@ def save_inference_model(
if params_filename is not None:
params_filename = os.path.basename(params_filename)
save_persistables(executor, save_dirname, main_program, params_filename)
paddle.distributed.io.save_persistables(
executor, save_dirname, main_program, params_filename
)
return target_var_name_list
......@@ -1671,7 +483,9 @@ def load_inference_model(
"Unsupported program version: %d\n" % program._version()
)
# Binary data also need versioning.
load_persistables(executor, load_dirname, program, params_filename)
paddle.distributed.io.load_persistables(
executor, load_dirname, program, params_filename
)
if pserver_endpoints:
program = _endpoints_replacement(program, pserver_endpoints)
......@@ -1692,859 +506,3 @@ def _endpoints_replacement(program, endpoints):
op.set_attr(ENDPOINT_MAP, endpoints)
program._sync_with_cpp()
return program
def get_parameter_value(para, executor):
"""
Get the LoDTensor value of the given parameter.
Args:
para(Parameter): The parameter to get value from.
executor(Executor): The executor to run for retrieving the value.
Returns:
numpy.array: The given parameter's values.
Raises:
AssertionError: If the `para` is not an instance of Parameter.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
exe = fluid.Executor(fluid.CPUPlace())
param = fluid.default_main_program().global_block().var('fc.w')
p = fluid.io.get_parameter_value(param, exe)
"""
assert is_parameter(para), "The input variable is not parameter."
get_program = Program()
block = get_program.global_block()
new_var = _clone_var_in_block_(block, para)
return executor.run(get_program, feed={}, fetch_list=[new_var])[0]
def get_parameter_value_by_name(name, executor, program=None):
"""
Get the LoDTensor value of a certain parameter by its name.
Args:
name(str): The parameter's name.
executor(Executor): The executor to run for retrieving the value.
program(Program | None): The program where to find the parameter.
If it's set to be None, the function will
try to find the parameter in the default
main program.
Returns:
numpy.array: The parameter's values.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
exe = fluid.Executor(fluid.CPUPlace())
p = fluid.io.get_parameter_value('fc.w', exe)
"""
if program is None:
program = default_main_program()
var = program.global_block().var(name)
return get_parameter_value(var, executor)
def _save_persistable_nodes(executor, dirname, graph):
"""
Save persistable nodes to the given directory by the executor.
Args:
executor(Executor): The executor to run for saving node values.
dirname(str): The directory path.
graph(IrGraph): All the required persistable nodes in the graph will be saved.
"""
persistable_node_names = set()
persistable_nodes = []
all_persistable_nodes = graph.all_persistable_nodes()
for node in all_persistable_nodes:
name = node.name()
if name not in persistable_node_names:
persistable_node_names.add(name)
persistable_nodes.append(node)
program = Program()
var_list = []
for node in persistable_nodes:
var_desc = node.var()
if (
var_desc.type() == core.VarDesc.VarType.RAW
or var_desc.type() == core.VarDesc.VarType.READER
):
continue
var = program.global_block().create_var(
name=var_desc.name(),
shape=var_desc.shape(),
dtype=var_desc.dtype(),
type=var_desc.type(),
lod_level=var_desc.lod_level(),
persistable=var_desc.persistable(),
)
var_list.append(var)
save_vars(executor=executor, dirname=dirname, vars=var_list)
def _load_persistable_nodes(executor, dirname, graph):
"""
Load persistable node values from the given directory by the executor.
Args:
executor(Executor): The executor to run for loading node values.
dirname(str): The directory path.
graph(IrGraph): All the required persistable nodes in the graph will be loaded.
"""
persistable_node_names = set()
persistable_nodes = []
all_persistable_nodes = graph.all_persistable_nodes()
for node in all_persistable_nodes:
name = node.name()
if name not in persistable_node_names:
persistable_node_names.add(name)
persistable_nodes.append(node)
program = Program()
var_list = []
def _exist(var):
return os.path.exists(os.path.join(dirname, var.name))
for node in persistable_nodes:
var_desc = node.var()
if (
var_desc.type() == core.VarDesc.VarType.RAW
or var_desc.type() == core.VarDesc.VarType.READER
):
continue
var = program.global_block().create_var(
name=var_desc.name(),
shape=var_desc.shape(),
dtype=var_desc.dtype(),
type=var_desc.type(),
lod_level=var_desc.lod_level(),
persistable=var_desc.persistable(),
)
if _exist(var):
var_list.append(var)
else:
_logger.warn("Cannot find the var %s!!!" % (node.name()))
load_vars(executor=executor, dirname=dirname, vars=var_list)
def _unpack_saved_dict(saved_obj, protocol):
temp_saved_obj = {}
unpack_infor = {}
# When pickle protocol=2 or protocol=3 the serialized object cannot be larger than 4G.
if 1 < protocol < 4:
if isinstance(saved_obj, dict):
for key, value in saved_obj.items():
if isinstance(value, np.ndarray):
MAX_NUMBER_OF_ELEMENT = int(
(2**30 - 1) / value.dtype.itemsize
)
num_element = np.prod(value.shape)
if num_element > MAX_NUMBER_OF_ELEMENT:
unpack_infor[key] = {}
unpack_infor[key]["OriginShape"] = value.shape
unpack_infor[key]["slices"] = []
value = value.flatten()
for i in range(
int(
math.ceil(
num_element * 1.0 / MAX_NUMBER_OF_ELEMENT
)
)
):
part_name = key + "@@." + str(i)
unpack_infor[key]["slices"].append(part_name)
temp_saved_obj[part_name] = value[
i
* MAX_NUMBER_OF_ELEMENT : MAX_NUMBER_OF_ELEMENT
* (i + 1)
]
if unpack_infor:
for key, value in unpack_infor.items():
if key in saved_obj:
saved_obj.pop(key)
for part in value['slices']:
saved_obj[part] = temp_saved_obj[part]
saved_obj['UnpackBigParamInfor@@'] = unpack_infor
return saved_obj
def _pack_loaded_dict(load_obj):
if isinstance(load_obj, dict):
unpack_info = 'UnpackBigParamInfor@@'
if unpack_info in load_obj:
removes = []
for key, value in load_obj[unpack_info].items():
slices = [load_obj[part] for part in value["slices"]]
load_obj[key] = np.concatenate(slices).reshape(
value["OriginShape"]
)
removes += value["slices"]
for key in removes:
load_obj.pop(key)
load_obj.pop(unpack_info)
return load_obj
@static_only
def _legacy_save(param_dict, model_path, protocol=2):
def get_tensor(var):
if isinstance(var, (core.VarBase, core.eager.Tensor)):
return var.numpy()
elif isinstance(var, core.LoDTensor):
return np.array(var)
return var
param_dict = {name: get_tensor(param_dict[name]) for name in param_dict}
# When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
if (
_is_file_path(model_path)
and sys.platform == 'darwin'
and sys.version_info.major == 3
):
pickle_bytes = pickle.dumps(param_dict, protocol=protocol)
with open(model_path, 'wb') as f:
max_bytes = 2**30
for i in range(0, len(pickle_bytes), max_bytes):
f.write(pickle_bytes[i : i + max_bytes])
else:
with _open_file_buffer(model_path, 'wb') as f:
pickle.dump(param_dict, f, protocol=protocol)
@static_only
def save(program, model_path, protocol=4, **configs):
"""
This function save parameters, optimizer information and network description to model_path.
The parameters contains all the trainable Tensor, will save to a file with suffix ".pdparams".
The optimizer information contains all the Tensor used by optimizer. For Adam optimizer, contains beta1, beta2, momentum etc. All the information will save to a file with suffix ".pdopt". (If the optimizer have no Tensor need to save (like SGD), the fill will not generated).
The network description is the description of the program. It's only used for deployment. The description will save to a file with a suffix ".pdmodel".
Args:
program(Program) : The program to saved.
model_path(str): the file prefix to save the program. The format is "dirname/file_prefix". If file_prefix is empty str. A exception will be raised
protocol(int, optional): The protocol version of pickle module must be greater than 1 and less than 5.
Default: 4
configs(dict, optional) : optional keyword arguments.
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.static as static
paddle.enable_static()
x = static.data(name="x", shape=[10, 10], dtype='float32')
y = static.nn.fc(x, 10)
z = static.nn.fc(y, 10)
place = paddle.CPUPlace()
exe = static.Executor(place)
exe.run(static.default_startup_program())
prog = static.default_main_program()
static.save(prog, "./temp")
"""
base_name = os.path.basename(model_path)
assert (
base_name != ""
), "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received model_path is empty string."
if 'pickle_protocol' in configs:
protocol = configs['pickle_protocol']
warnings.warn(
"'pickle_protocol' is a deprecated argument. Please use 'protocol' instead."
)
if not isinstance(protocol, int):
raise ValueError(
"The 'protocol' MUST be `int`, but received {}".format(
type(protocol)
)
)
if protocol < 2 or protocol > 4:
raise ValueError(
"Expected 1<'protocol'<5, but received protocol={}".format(protocol)
)
dir_name = os.path.dirname(model_path)
if dir_name and not os.path.exists(dir_name):
os.makedirs(dir_name)
def get_tensor(var):
t = global_scope().find_var(var.name).get_tensor()
return np.array(t)
parameter_list = list(filter(is_parameter, program.list_vars()))
param_dict = {p.name: get_tensor(p) for p in parameter_list}
param_dict = _unpack_saved_dict(param_dict, protocol)
# When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
if sys.platform == 'darwin' and sys.version_info.major == 3:
pickle_bytes = pickle.dumps(param_dict, protocol=protocol)
with open(model_path + ".pdparams", 'wb') as f:
max_bytes = 2**30
for i in range(0, len(pickle_bytes), max_bytes):
f.write(pickle_bytes[i : i + max_bytes])
else:
with open(model_path + ".pdparams", 'wb') as f:
pickle.dump(param_dict, f, protocol=protocol)
optimizer_var_list = list(
filter(is_belong_to_optimizer, program.list_vars())
)
opt_dict = {p.name: get_tensor(p) for p in optimizer_var_list}
with open(model_path + ".pdopt", 'wb') as f:
pickle.dump(opt_dict, f, protocol=protocol)
main_program = program.clone()
program.desc.flush()
main_program.desc._set_version()
paddle.fluid.core.save_op_version_info(program.desc)
with open(model_path + ".pdmodel", "wb") as f:
f.write(program.desc.serialize_to_string())
def _pickle_loads_mac(path, f):
pickle_bytes = bytearray(0)
file_size = os.path.getsize(path)
max_bytes = 2**30
for _ in range(0, file_size, max_bytes):
pickle_bytes += f.read(max_bytes)
load_result = pickle.loads(pickle_bytes, encoding='latin1')
return load_result
@static_only
def load(program, model_path, executor=None, var_list=None):
"""
:api_attr: Static Graph
This function get parameters and optimizer information from program, and then get corresponding value from file.
An exception will throw if shape or dtype of the parameters is not match.
This function can also load model file saved with [ save_params, save_persistables, save_vars ].
var_list can not be None when load single model file
( filename is not None When save_params, save_persistables or save_vars is called ).
Args:
program(Program): The program will be loaded
model_path(str): The file prefix store the program
executor(Executor, optional): The executor used for initialize the parameter
When startup program is not run.
var_list(list|tuple, optional): The Tensor list/tuple to load single model file saved with
[ save_params, save_persistables, save_vars ].
Default: None
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.static as static
paddle.enable_static()
x = static.data(name="x", shape=[10, 10], dtype='float32')
y = static.nn.fc(x, 10)
z = static.nn.fc(y, 10)
place = paddle.CPUPlace()
exe = static.Executor(place)
exe.run(static.default_startup_program())
prog = static.default_main_program()
static.save(prog, "./temp")
static.load(prog, "./temp")
"""
assert executor is None or isinstance(executor, Executor)
model_prefix = model_path
if model_prefix.endswith(".pdparams"):
model_prefix = model_prefix[:-9]
elif model_prefix.endswith(".pdopt"):
model_prefix = model_prefix[:-6]
elif model_prefix.endswith(".pdmodel"):
model_prefix = model_prefix[:-8]
parameter_file_name = model_prefix + ".pdparams"
if not os.path.exists(parameter_file_name):
# model file save by fluid.save not found, try to load model file saved with
# [save_vars, save_params, save_persistables]
_logger.debug(
"{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format(
parameter_file_name
)
)
if executor is None:
raise ValueError(
"executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
)
if var_list is not None:
var_list_names = [var.name for var in var_list]
else:
var_list_names = None
if os.path.isdir(model_path):
binary_file_set = set()
for root, dirs, files in os.walk(model_path, topdown=False):
for f in files:
binary_file_set.add(
os.path.join(root, f).replace("\\", "/")
)
program_var_list = list(program.list_vars())
loaded_var_list = []
for var in program_var_list:
var_path = os.path.join(model_path, var.name).replace("\\", "/")
load_condition = (
var_list_names is None or var.name in var_list_names
)
if var_path in binary_file_set and load_condition:
loaded_var_list.append(var)
binary_file_set.remove(var_path)
if len(binary_file_set) > 0:
unused_var_list = " ".join(list(binary_file_set))
_logger.warning(
"variable file [ %s ] not used"
% (" ".join(list(binary_file_set)))
)
try:
load_vars(
executor=executor, dirname=model_path, vars=loaded_var_list
)
except RuntimeError as e:
_logger.error(e)
raise e
except:
raise RuntimeError(
"Failed to load model file, please make sure model file is saved with the "
"following APIs: save_params, save_persistables, save_vars"
)
return
elif os.path.isfile(model_path):
if var_list is None:
raise ValueError(
"var_list is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
)
program_var_list = program.list_vars()
program_var_name_set = set([var.name for var in program_var_list])
# check all the variable inlcuded in program
for var in var_list:
if var.name not in program_var_name_set:
raise LookupError(
"loaded var [{}] is not in program variable list"
)
dir_name, file_name = os.path.split(model_path)
try:
load_vars(
executor=executor,
dirname=dir_name,
vars=var_list,
filename=file_name,
)
except RuntimeError as e:
_logger.error(e)
raise e
except:
raise RuntimeError(
"Failed to load model file , please make sure model file is saved with the "
"the following APIs: [ save_params, save_persistables, save_vars ]. "
"When these API called, filename CANNOT be None"
)
return
def set_var(var, ndarray):
t = global_scope().find_var(var.name).get_tensor()
p = t._place()
if p.is_cpu_place():
place = paddle.fluid.CPUPlace()
elif p.is_cuda_pinned_place():
place = paddle.fluid.CUDAPinnedPlace()
elif p.is_xpu_place():
p = paddle.fluid.core.Place()
p.set_place(t._place())
place = paddle.fluid.XPUPlace(p.xpu_device_id())
elif p.is_npu_place():
p = paddle.fluid.core.Place()
p.set_place(t._place())
place = paddle.fluid.NPUPlace(p.npu_device_id())
elif p.is_mlu_place():
p = paddle.fluid.core.Place()
p.set_place(t._place())
place = paddle.fluid.MLUPlace(p.mlu_device_id())
else:
p = paddle.fluid.core.Place()
p.set_place(t._place())
place = paddle.fluid.CUDAPlace(p.gpu_device_id())
t.set(ndarray, place)
parameter_list = list(filter(is_parameter, program.list_vars()))
if executor:
paddle.fluid.core._create_loaded_parameter(
parameter_list, global_scope(), executor._default_executor
)
with open(parameter_file_name, 'rb') as f:
# When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
if sys.platform == 'darwin' and sys.version_info.major == 3:
load_dict = _pickle_loads_mac(parameter_file_name, f)
else:
load_dict = pickle.load(f, encoding='latin1')
load_dict = _pack_loaded_dict(load_dict)
for v in parameter_list:
assert (
v.name in load_dict
), "Can not find [{}] in model file [{}]".format(
v.name, parameter_file_name
)
set_var(v, load_dict[v.name])
optimizer_var_list = list(
filter(is_belong_to_optimizer, program.list_vars())
)
if len(optimizer_var_list) > 0:
opt_file_name = model_prefix + ".pdopt"
assert os.path.exists(
opt_file_name
), "Optimizer file [{}] not exits".format(opt_file_name)
if executor:
paddle.fluid.core._create_loaded_parameter(
optimizer_var_list, global_scope(), executor._default_executor
)
with open(opt_file_name, 'rb') as f:
load_dict = pickle.load(f, encoding='latin1')
for v in optimizer_var_list:
assert (
v.name in load_dict
), "Can not find [{}] in model file [{}]".format(
v.name, opt_file_name
)
set_var(v, load_dict[v.name])
def load_program_state(model_path, var_list=None):
"""
Load program state from local file
Args:
model_path(str): The file prefix store the program
var_list(list|tuple, optional): The Tensor list/tuple to load saved with
[ save_params, save_persistables, save_vars ].
Default: None.
The var_list is only used to get name,
will not be modified.
Returns:
state_dict(dict): the dict store Parameter and optimizer information
Examples:
.. code-block:: python
import paddle
import paddle.static as static
paddle.enable_static()
x = static.data(name="x", shape=[10, 10], dtype='float32')
y = static.nn.fc(x, 10)
z = static.nn.fc(y, 10)
place = paddle.CPUPlace()
exe = static.Executor(place)
exe.run(static.default_startup_program())
prog = static.default_main_program()
static.save(prog, "./temp")
program_state = static.load_program_state("./temp")
"""
model_prefix = model_path
if model_prefix.endswith(".pdparams"):
model_prefix = model_prefix[:-9]
elif model_prefix.endswith(".pdopt"):
model_prefix = model_prefix[:-6]
elif model_prefix.endswith(".pdmodel"):
model_prefix = model_prefix[:-8]
parameter_file_name = model_prefix + ".pdparams"
if not os.path.exists(parameter_file_name):
# model file saved with fluid.save is not found, try to load model file saved with
# [save_vars, save_params, save_persistables]
_logger.debug(
"{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format(
parameter_file_name
)
)
var_name_list = []
if var_list is None and os.path.isfile(model_path):
raise ValueError(
"var_list can not be None when model_path is a file type"
)
for root, dirs, files in os.walk(model_path, topdown=False):
for f in files:
file_path = os.path.join(root, f)
var_temp_name = os.path.relpath(file_path, model_path)
var_temp_name = var_temp_name.replace("\\", "/")
var_name_list.append(var_temp_name)
with _load_program_scope():
load_prog = Program()
load_block = load_prog.global_block()
def clone_var_to_block(block, var):
if not isinstance(var, Variable):
raise TypeError("value in var_list must be variable")
return block.create_var(
name=var.name,
shape=var.shape,
dtype=var.dtype,
type=var.type,
lod_level=var.lod_level
if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR
else None,
persistable=True,
)
def _load_vars_with_try_catch(
exe, dirname, vars, filename, raise_error=True
):
try:
load_vars(
executor=exe,
dirname=dirname,
vars=vars,
filename=filename,
)
return True
except:
error_str = (
"Failed to load model/variables `%s`, please make sure "
"model/variables file is saved with the following APIs: "
"save_params, save_persistables, save_vars."
)
filenames = (
[var.name for var in vars]
if filename is None
else filename
)
if raise_error:
raise RuntimeError(error_str % filenames)
else:
warnings.warn(error_str % filenames, RuntimeWarning)
return False
place = paddle.fluid.CPUPlace()
exe = paddle.fluid.Executor(place)
loaded_var_list = []
if os.path.isfile(model_path):
# when model_path is file, var_list cannot be None
dir_name, file_name = os.path.split(model_path)
for var in var_list:
loaded_var_list.append(clone_var_to_block(load_block, var))
_load_vars_with_try_catch(
exe, dir_name, loaded_var_list, file_name
)
else:
# var_list can be None or not None
if var_list is not None:
for var in var_list:
loaded_var_list.append(
clone_var_to_block(load_block, var)
)
_load_vars_with_try_catch(
exe, model_path, loaded_var_list, None
)
else:
for var_name in var_name_list:
# NOTE(chenweihang): If identify which files the user wants
# to load from the disk, we load these variables one by one.
# If a file does not exist, we only warn the user that the
# file may be an irrelevant file, but does not throw an error
# to ensure that other legal variables can be loaded.
temp_var = load_block.create_var(
name=var_name, persistable=True
)
if _load_vars_with_try_catch(
exe, model_path, [temp_var], None, False
):
loaded_var_list.append(temp_var)
res_dict = {}
for var in loaded_var_list:
res_dict[var.name] = np.asarray(
paddle.fluid.global_scope().find_var(var.name).get_tensor()
)
return res_dict
assert os.path.exists(
parameter_file_name
), "Parameter file [{}] not exits".format(parameter_file_name)
with open(parameter_file_name, 'rb') as f:
# When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
if sys.platform == 'darwin' and sys.version_info.major == 3:
para_dict = _pickle_loads_mac(parameter_file_name, f)
else:
para_dict = pickle.load(f, encoding='latin1')
para_dict = _pack_loaded_dict(para_dict)
opt_file_name = model_prefix + ".pdopt"
if os.path.exists(opt_file_name):
with open(opt_file_name, 'rb') as f:
opti_dict = pickle.load(f, encoding='latin1')
para_dict.update(opti_dict)
return para_dict
@static_only
def set_program_state(program, state_dict):
"""
Set program parameter from state_dict
An exception will throw if shape or dtype of the parameters is not match.
NOTICE: This function MUST called after run start_up_program
Args:
program(Program): The program to be set
state_dict(dict): the dict store Parameter and optimizer information
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.static as static
paddle.enable_static()
x = static.data(name="x", shape=[10, 10], dtype='float32')
y = static.nn.fc(x, 10)
z = static.nn.fc(y, 10)
place = paddle.CPUPlace()
exe = static.Executor(place)
exe.run(static.default_startup_program())
prog = static.default_main_program()
static.save(prog, "./temp")
program_state = static.load_program_state("./temp")
static.set_program_state(prog, program_state)
"""
state_dict = _pack_loaded_dict(state_dict)
parameter_list = list(filter(is_persistable, program.list_vars()))
used_para_list = {}
for para in parameter_list:
var_temp = paddle.fluid.global_scope().find_var(para.name)
assert (
var_temp is not None
), "Variable [ {} ] Not found, Please make sure run startup program".format(
para.name
)
if para.name in state_dict:
# set value from state dict
orig_para_np = np.array(var_temp.get_tensor())
new_para_np = state_dict[para.name]
assert orig_para_np.shape == new_para_np.shape, (
"Parameter's shape does not match, the Program requires a parameter with the shape of ({}), "
"while the loaded parameter (namely [ {} ]) has a shape of ({}).".format(
orig_para_np.shape, para.name, new_para_np.shape
)
)
assert orig_para_np.dtype == new_para_np.dtype, (
"Parameter's data type does not match, the Program requires a parameter with a dtype of ({}), "
"while the loaded parameter (namely [ {} ]) has a dtype of ({}).".format(
orig_para_np.dtype, para.name, new_para_np.dtype
)
)
ten = var_temp.get_tensor()
ten_place = ten._place()
# assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \
# "Place not support, only support CPUPlace and GPUPlace, now is {}".format(str(ten_place))
py_place = paddle.fluid.CPUPlace()
if ten_place.is_cuda_pinned_place():
place = paddle.fluid.CUDAPinnedPlace()
elif ten_place.is_gpu_place():
p = paddle.fluid.core.Place()
p.set_place(ten_place)
py_place = paddle.fluid.CUDAPlace(p.gpu_device_id())
elif ten_place.is_xpu_place():
p = paddle.fluid.core.Place()
p.set_place(ten_place)
py_place = paddle.fluid.XPUPlace(p.xpu_device_id())
elif ten_place.is_npu_place():
p = paddle.fluid.core.Place()
p.set_place(ten_place)
py_place = paddle.fluid.NPUPlace(p.npu_device_id())
elif ten_place.is_mlu_place():
p = paddle.fluid.core.Place()
p.set_place(ten_place)
py_place = paddle.fluid.MLUPlace(p.mlu_device_id())
ten.set(new_para_np, py_place)
used_para_list[para.name] = 1
unused_para_list = []
for k, v in state_dict.items():
if k not in used_para_list:
unused_para_list.append(k)
if len(unused_para_list) > 0:
warnings.warn(
"This list is not set, Because of Paramerter not found in program. There are: {}".format(
" ".join(unused_para_list)
)
)
python/paddle/fluid/tests/unittests/dist_save_load.py
浏览文件 @ f8d0a358
......@@ -22,7 +22,7 @@ from test_dist_base import RUN_STEP, runtime_main
import paddle
import paddle.fluid as fluid
from paddle.fluid import core, io
from paddle.fluid import core
class TestDistSaveLoad2x2(TestDistSimnetBow2x2):
......@@ -56,7 +56,7 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2):
return var.persistable
io.load_vars(
paddle.static.io.load_vars(
executor,
dirname=dirname,
main_program=program,
......@@ -89,7 +89,9 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2):
exe.run(startup_prog)
if need_load and model_dir:
fluid.io.load_persistables(exe, model_dir, pserver_prog)
paddle.distributed.io.load_persistables(
exe, model_dir, pserver_prog
)
exe.run(pserver_prog)
......
python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py
浏览文件 @ f8d0a358
......@@ -172,7 +172,7 @@ def train_static(args, batch_generator):
model_path = os.path.join(
args.save_static_model_path, "transformer"
)
fluid.save(train_prog, model_path)
paddle.static.save(train_prog, model_path)
break
return np.array(avg_loss)
......
python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py
浏览文件 @ f8d0a358
......@@ -20,7 +20,6 @@ import numpy as np
import paddle
import paddle.dataset.wmt16 as wmt16
import paddle.fluid as fluid
def get_input_descs(args, mode="train"):
......@@ -310,7 +309,7 @@ def load(program, model_path, executor=None, var_list=None):
To load python2 saved models in python3.
"""
try:
fluid.load(program, model_path, executor, var_list)
paddle.static.load(program, model_path, executor, var_list)
except UnicodeDecodeError:
warnings.warn(
"An UnicodeDecodeError is catched, which might be caused by loading "
......@@ -319,7 +318,7 @@ def load(program, model_path, executor=None, var_list=None):
)
load_bak = pickle.load
pickle.load = partial(load_bak, encoding="latin1")
fluid.load(program, model_path, executor, var_list)
paddle.static.load(program, model_path, executor, var_list)
pickle.load = load_bak
......
python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py
浏览文件 @ f8d0a358
......@@ -42,7 +42,7 @@ def main_test_func(place, dtype):
adam_optimizer.minimize(avg_cost)
fetch_list = [avg_cost]
train_reader = fluid.io.batch(
train_reader = paddle.batch(
paddle.dataset.uci_housing.train(), batch_size=1
)
feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
......
python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py
浏览文件 @ f8d0a358
......@@ -40,7 +40,7 @@ class TestClass(unittest.TestCase):
yield img, label
reader = paddle.reader.cache(fake_reader)
batch_reader = fluid.io.batch(reader, batch_size=batch_size)
batch_reader = paddle.batch(reader, batch_size=batch_size)
places = [fluid.CPUPlace()]
if fluid.core.is_compiled_with_cuda():
......
python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py
浏览文件 @ f8d0a358
......@@ -264,7 +264,7 @@ class TestDistMnistAsync2x2WithGauss(TestFleetBase):
feeder = fluid.DataFeeder(place=fluid.CPUPlace(), feed_list=datas)
exe.run(fluid.default_startup_program())
fluid.io.load_persistables(exe, model_file)
paddle.distributed.io.load_persistables(exe, model_file)
for batch_id, data in enumerate(reader()):
score = exe.run(
......
python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py
浏览文件 @ f8d0a358
......@@ -150,12 +150,12 @@ class TestDygraphLoadStatic(unittest.TestCase):
)
out = exe.run(framework.default_startup_program())
fluid.save(
paddle.static.save(
framework.default_main_program(),
os.path.join(temp_dir.name, "test_1"),
)
para_dict = fluid.load_program_state(
para_dict = paddle.static.load_program_state(
os.path.join(temp_dir.name, "test_1")
)
......
python/paddle/fluid/tests/unittests/test_inference_model_io.py
浏览文件 @ f8d0a358
......@@ -25,14 +25,13 @@ import paddle.fluid as fluid
import paddle.fluid.core as core
import paddle.fluid.executor as executor
import paddle.fluid.optimizer as optimizer
from paddle.distributed.io import load_inference_model_distributed
from paddle.fluid.compiler import CompiledProgram
from paddle.fluid.framework import Program, program_guard
from paddle.fluid.io import (
load_inference_model,
save_inference_model,
from paddle.distributed.io import (
load_inference_model_distributed,
save_persistables,
)
from paddle.fluid.compiler import CompiledProgram
from paddle.fluid.framework import Program, program_guard
from paddle.fluid.io import load_inference_model, save_inference_model
paddle.enable_static()
......
python/paddle/fluid/tests/unittests/test_io_save_load.py
浏览文件 @ f8d0a358
......@@ -18,6 +18,7 @@ import unittest
import paddle
import paddle.fluid as fluid
import paddle.static as static
from paddle.fluid import core
......@@ -34,24 +35,24 @@ class TestSaveLoadAPIError(unittest.TestCase):
graph = core.Graph(core.ProgramDesc())
compiled_program = fluid.CompiledProgram(graph)
with self.assertRaises(TypeError):
fluid.io._get_valid_program(compiled_program)
paddle.static.io._get_valid_program(compiled_program)
# case 2: main_program type error
with self.assertRaises(TypeError):
fluid.io._get_valid_program("program")
paddle.static.io._get_valid_program("program")
def test_load_vars_error(self):
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# case 1: main_program type error when vars None
with self.assertRaises(TypeError):
fluid.io.load_vars(
static.io.load_vars(
executor=exe, dirname=self.save_dir, main_program="program"
)
# case 2: main_program type error when vars not None
with self.assertRaises(TypeError):
fluid.io.load_vars(
static.io.load_vars(
executor=exe,
dirname=self.save_dir,
main_program="program",
......
python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py
浏览文件 @ f8d0a358
......@@ -75,7 +75,7 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_model(self, only_params=False):
def train_and_save_model(self):
with new_program_scope():
startup_program = fluid.default_startup_program()
main_program = fluid.default_main_program()
......@@ -122,19 +122,14 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
param.name
)
if only_params:
fluid.io.save_params(
exe, self.save_dirname, filename=self.params_filename
)
else:
fluid.io.save_inference_model(
self.save_dirname,
["img"],
[prediction],
exe,
model_filename=self.model_filename,
params_filename=self.params_filename,
)
fluid.io.save_inference_model(
self.save_dirname,
["img"],
[prediction],
exe,
model_filename=self.model_filename,
params_filename=self.params_filename,
)
return static_param_dict
......@@ -195,16 +190,6 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
)
self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_state_dict_from_save_params(self):
self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.save_params"
)
self.params_filename = None
orig_param_dict = self.train_and_save_model(True)
new_load_param_dict = paddle.load(self.save_dirname)
self.check_load_state_dict(orig_param_dict, new_load_param_dict)
if __name__ == '__main__':
unittest.main()
python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py 已删除 100644 → 0
浏览文件 @ 25a77450
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import shutil
import unittest
import paddle
import paddle.fluid as fluid
from paddle.fluid.executor import Executor
class TestLoadVarsShapeCheck(unittest.TestCase):
def setUp(self):
self.model_path = "./model_temp/"
def test_shape_check_save_load(self):
program_1 = fluid.Program()
startup_program_1 = fluid.Program()
with fluid.program_guard(program_1, startup_program_1):
input = paddle.static.data(
name="x", shape=[-1, 10], dtype='float32'
)
out = paddle.static.nn.fc(input, 20)
place = fluid.CPUPlace()
exe = Executor(place)
exe.run(startup_program_1)
fluid.io.save_params(exe, self.model_path, main_program=program_1)
fluid.io.load_params(exe, self.model_path, main_program=program_1)
def tearDown(self):
if os.path.exists(self.model_path):
shutil.rmtree(self.model_path)
if __name__ == "__main__":
unittest.main()
python/paddle/fluid/tests/unittests/test_paddle_save_load.py
浏览文件 @ f8d0a358
......@@ -235,7 +235,7 @@ class TestSaveLoadAny(unittest.TestCase):
# paddle.save, legacy paddle.fluid.load
self.replace_static_save(prog, path)
self.set_zero(prog, place)
paddle.fluid.io.load(prog, path)
paddle.static.load(prog, path)
for var in prog.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
new_t = np.array(
......@@ -244,7 +244,7 @@ class TestSaveLoadAny(unittest.TestCase):
base_t = base_map[var.name]
np.testing.assert_array_equal(new_t, np.array(base_t))
# legacy paddle.fluid.save, paddle.load
paddle.fluid.io.save(prog, path)
paddle.static.save(prog, path)
self.set_zero(prog, place)
self.replace_static_load(prog, path)
for var in prog.list_vars():
......@@ -904,7 +904,7 @@ class TestSaveLoadToMemory(unittest.TestCase):
with self.assertRaises(ValueError):
paddle.save(state_dict, '')
with self.assertRaises(ValueError):
paddle.fluid.io._open_file_buffer('temp', 'b')
paddle.framework.io_utils._open_file_buffer('temp', 'b')
def test_static_save_to_memory(self):
paddle.enable_static()
......
python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py
浏览文件 @ f8d0a358
......@@ -107,7 +107,7 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
var_list = list(
filter(lambda var: var.persistable, prog.list_vars())
)
fluid.io.load_vars(
paddle.static.io.load_vars(
exe, path_vars1, main_program=prog, vars=var_list
)
......@@ -123,7 +123,7 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
path_vars2 = os.path.join(
self.temp_dir.name, 'test_replace_save_load_vars_binary2/model/'
)
fluid.io.save_vars(
paddle.static.io.save_vars(
exe, path_vars2, main_program=prog, vars=var_list
)
self.set_zero(prog, place)
......
python/paddle/fluid/tests/unittests/test_parameter.py
浏览文件 @ f8d0a358
......@@ -19,7 +19,6 @@ import numpy as np
import paddle
import paddle.fluid.core as core
import paddle.fluid.io as io
from paddle.fluid.dygraph import guard
from paddle.fluid.executor import Executor
from paddle.fluid.framework import ParamBase, Variable, default_main_program
......@@ -47,8 +46,6 @@ class ParameterChecks(unittest.TestCase):
exe = Executor(paddle.CPUPlace())
p = exe.run(main_program, fetch_list=[param])[0]
np.testing.assert_array_equal(p, np.ones(shape) * val)
p = io.get_parameter_value_by_name('fc.w', exe, main_program)
np.testing.assert_array_equal(p, np.ones(shape) * val)
zero_dim_param = b.create_parameter(name='x', shape=[], dtype='float32')
self.assertEqual(zero_dim_param.shape, ())
......
python/paddle/fluid/tests/unittests/test_static_save_load.py
浏览文件 @ f8d0a358
......@@ -346,7 +346,9 @@ class TestSaveLoadBase(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
paddle.static.save(
main_program, os.path.join(temp_dir.name, "test_1")
)
# set var to zero
for var in main_program.list_vars():
......@@ -360,7 +362,7 @@ class TestSaveLoadBase(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(
paddle.static.load(
main_program,
os.path.join(temp_dir.name, "test_1.pdparams"),
exe,
......@@ -485,7 +487,9 @@ class TestSaveLoadPartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
paddle.static.save(
main_program, os.path.join(temp_dir.name, "test_1")
)
# set var to zero
for var in main_program.list_vars():
......@@ -499,7 +503,7 @@ class TestSaveLoadPartial(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(
paddle.static.load(
test_program, os.path.join(temp_dir.name, "test_1.pdopt"), None
)
......@@ -510,7 +514,7 @@ class TestSaveLoadPartial(unittest.TestCase):
)
base_t = base_map[var.name]
np.testing.assert_array_equal(new_t, base_t)
fluid.load(
paddle.static.load(
test_program,
os.path.join(temp_dir.name, "test_1.pdmodel"),
None,
......@@ -617,7 +621,9 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
paddle.static.save(
main_program, os.path.join(temp_dir.name, "test_1")
)
# set var to zero
for var in main_program.list_vars():
......@@ -631,7 +637,9 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, os.path.join(temp_dir.name, "test_1"), exe)
paddle.static.load(
main_program, os.path.join(temp_dir.name, "test_1"), exe
)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -752,7 +760,9 @@ class TestProgramStatePartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, os.path.join(temp_dir.name, 'test_1'))
paddle.static.save(
main_program, os.path.join(temp_dir.name, 'test_1')
)
# set var to zero
for var in main_program.list_vars():
......@@ -767,23 +777,23 @@ class TestProgramStatePartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(new_t)) == 0)
# fluid.load(test_program, "./test_1", None )
program_state = fluid.load_program_state(
program_state = paddle.static.load_program_state(
os.path.join(temp_dir.name, 'test_1')
)
program_state_1 = fluid.load_program_state(
program_state_1 = paddle.static.load_program_state(
os.path.join(temp_dir.name, 'test_1.pdparams')
)
program_state_2 = fluid.load_program_state(
program_state_2 = paddle.static.load_program_state(
os.path.join(temp_dir.name, 'test_1.pdopt')
)
program_state_3 = fluid.load_program_state(
program_state_3 = paddle.static.load_program_state(
os.path.join(temp_dir.name, 'test_1.pdmodel')
)
fluid.set_program_state(test_program, program_state)
paddle.static.set_program_state(test_program, program_state)
for var in test_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -805,7 +815,7 @@ class TestProgramStatePartial(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.set_program_state(test_program, program_state_1)
paddle.static.set_program_state(test_program, program_state_1)
for var in test_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -827,7 +837,7 @@ class TestProgramStatePartial(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.set_program_state(test_program, program_state_2)
paddle.static.set_program_state(test_program, program_state_2)
for var in test_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -849,7 +859,7 @@ class TestProgramStatePartial(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.set_program_state(test_program, program_state_3)
paddle.static.set_program_state(test_program, program_state_3)
for var in test_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -880,7 +890,7 @@ class TestVariableInit(unittest.TestCase):
exe.run(fluid.default_startup_program())
temp_dir = tempfile.TemporaryDirectory()
fluid.save(
paddle.static.save(
fluid.default_main_program(),
os.path.join(temp_dir.name, "test_path"),
)
......@@ -905,7 +915,7 @@ class TestVariableInit(unittest.TestCase):
place = self.set_place()
exe = fluid.Executor(place)
parameter_list = list(
filter(fluid.io.is_parameter, program.list_vars())
filter(paddle.framework.is_parameter, program.list_vars())
)
fluid.core._create_loaded_parameter(
......@@ -925,7 +935,10 @@ class TestVariableInit(unittest.TestCase):
set_var(new_v, load_dict[v.name])
opt_list = list(
filter(fluid.io.is_belong_to_optimizer, program.list_vars())
filter(
paddle.framework.io_utils.is_belong_to_optimizer,
program.list_vars(),
)
)
fluid.core._create_loaded_parameter(
......@@ -1092,7 +1105,7 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(
paddle.static.load(
main_program, os.path.join(self.temp_dir.name, "test_path"), exe
)
......@@ -1112,7 +1125,7 @@ class TestLoadFromOldInterface(unittest.TestCase):
var.desc.set_shape(new_shape)
with self.assertRaises(RuntimeError):
fluid.load(
paddle.static.load(
main_program,
os.path.join(self.temp_dir.name, "test_path"),
exe,
......@@ -1120,7 +1133,7 @@ class TestLoadFromOldInterface(unittest.TestCase):
# check unused parameter
fluid.load(
paddle.static.load(
test_clone_program,
os.path.join(self.temp_dir.name, "test_path"),
exe,
......@@ -1239,7 +1252,7 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(
paddle.static.load(
main_program,
os.path.join(self.temp_dir.name, "test_static_load_var_list"),
exe,
......@@ -1377,11 +1390,11 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
self.assertTrue(np.sum(np.abs(new_t)) == 0)
file_model_path = os.path.join(save_dir, "model_single")
fluid.load(
paddle.static.load(
main_program,
file_model_path,
exe,
fluid.io.get_program_persistable_vars(main_program),
paddle.static.io.get_program_persistable_vars(main_program),
)
for var in main_program.list_vars():
......@@ -1403,36 +1416,33 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
var.desc.set_shape(new_shape)
with self.assertRaises(RuntimeError):
fluid.load(
paddle.static.load(
main_program,
file_model_path,
exe,
fluid.io.get_program_persistable_vars(main_program),
paddle.static.io.get_program_persistable_vars(main_program),
)
fluid.io.save_params(
exe, "test_path", main_program, filename="model_single"
)
with self.assertRaises(RuntimeError):
fluid.load(
paddle.static.load(
main_program,
file_model_path,
exe,
fluid.io.get_program_persistable_vars(main_program),
paddle.static.io.get_program_persistable_vars(main_program),
)
# check when executor is None
with self.assertRaises(ValueError):
fluid.load(
paddle.static.load(
main_program,
file_model_path,
None,
fluid.io.get_program_persistable_vars(main_program),
paddle.static.io.get_program_persistable_vars(main_program),
)
# check when var list is None
with self.assertRaises(ValueError):
fluid.load(main_program, file_model_path, exe, None)
paddle.static.load(main_program, file_model_path, exe, None)
# check save params, load var_list = get_program_persistable_vars
with self.assertRaises(RuntimeError):
......@@ -1440,7 +1450,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
main_program.global_block(), shape=[1], name="test_temp_var"
)
all_var_list = list(main_program.list_vars())
fluid.load(
paddle.static.load(
main_program,
file_model_path,
exe,
......@@ -1579,8 +1589,8 @@ class TestProgramStateOldSave(unittest.TestCase):
self.assertTrue(np.sum(np.abs(new_t)) == 0)
# case 1: load basic
program_state = fluid.load_program_state(save_dir)
fluid.set_program_state(main_program, program_state)
program_state = paddle.static.load_program_state(save_dir)
paddle.static.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map)
# case 2: load with no need file
......@@ -1594,21 +1604,21 @@ class TestProgramStateOldSave(unittest.TestCase):
else:
raise e
program_state = fluid.load_program_state(save_dir)
fluid.set_program_state(main_program, program_state)
program_state = paddle.static.load_program_state(save_dir)
paddle.static.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map)
# case 3: load with var_list
program_state = fluid.load_program_state(
program_state = paddle.static.load_program_state(
save_dir, main_program.all_parameters()
)
fluid.set_program_state(main_program, program_state)
paddle.static.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map)
if self.test_dygraph:
# make sure `load_program_state` can be used in dynamic graph mode
with fluid.dygraph.guard(place):
load_state = fluid.load_program_state(save_dir)
load_state = paddle.static.load_program_state(save_dir)
for k, v in load_state.items():
np.testing.assert_array_equal(base_map[k], v)
......@@ -1758,11 +1768,13 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self.assertTrue(np.sum(np.abs(new_t)) == 0)
# fluid.load(test_program, "./test_1", None )
program_state = fluid.load_program_state(
program_state = paddle.static.load_program_state(
os.path.join(save_dir, "model_1"),
var_list=fluid.io.get_program_persistable_vars(main_program),
var_list=paddle.static.io.get_program_persistable_vars(
main_program
),
)
fluid.set_program_state(main_program, program_state)
paddle.static.set_program_state(main_program, program_state)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -1773,15 +1785,17 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
np.testing.assert_array_equal(new_t, base_t)
with self.assertRaises(ValueError):
fluid.load_program_state(os.path.join(save_dir, "model_1"))
paddle.static.load_program_state(
os.path.join(save_dir, "model_1")
)
with self.assertRaises(TypeError):
fluid.load_program_state(
paddle.static.load_program_state(
os.path.join(save_dir, "model_1"), var_list=["str"]
)
with self.assertRaises(RuntimeError):
fluid.load_program_state(
paddle.static.load_program_state(
os.path.join(save_dir, "model_1"),
var_list=[
main_program.global_block().create_var(
......@@ -1827,17 +1841,17 @@ class TestStaticSaveLoadPickle(unittest.TestCase):
)
with self.assertRaises(ValueError):
paddle.fluid.save(prog, path, 2.0)
paddle.static.save(prog, path, 2.0)
with self.assertRaises(ValueError):
paddle.fluid.save(prog, path, 1)
paddle.static.save(prog, path, 1)
with self.assertRaises(ValueError):
paddle.fluid.save(prog, path, 5)
paddle.static.save(prog, path, 5)
protocols = [2, 3, 4]
for protocol in protocols:
paddle.fluid.save(prog, path, protocol)
paddle.static.save(prog, path, protocol)
# set var to zero
for var in prog.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -1851,7 +1865,7 @@ class TestStaticSaveLoadPickle(unittest.TestCase):
)
self.assertTrue(np.sum(np.abs(new_t)) == 0)
paddle.fluid.load(prog, path)
paddle.static.load(prog, path)
for var in prog.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......
python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py
浏览文件 @ f8d0a358
......@@ -134,7 +134,7 @@ class TestSaveLoadBF16(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
save_dir = os.path.join(self.temp_dir.name, "test_1")
fluid.save(main_program, save_dir)
paddle.static.save(main_program, save_dir)
# set var to zero
for var in main_program.list_vars():
......@@ -148,7 +148,7 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(
paddle.static.load(
main_program,
os.path.join(self.temp_dir.name, "test_1.pdparams"),
exe,
......
python/paddle/fluid/tests/unittests/test_static_save_load_large.py
浏览文件 @ f8d0a358
......@@ -58,7 +58,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
)
path = os.path.join(path, "static_save")
protocol = 4
paddle.fluid.save(prog, path, pickle_protocol=protocol)
paddle.static.save(prog, path, pickle_protocol=protocol)
# set var to zero
for var in prog.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -70,7 +70,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
)
self.assertTrue(np.sum(np.abs(new_t)) == 0)
paddle.fluid.load(prog, path)
paddle.static.load(prog, path)
for var in prog.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -91,8 +91,8 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
)
self.assertTrue(np.sum(np.abs(new_t)) == 0)
program_state = fluid.load_program_state(path)
fluid.set_program_state(prog, program_state)
program_state = paddle.static.load_program_state(path)
paddle.static.set_program_state(prog, program_state)
for var in prog.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
new_t = np.array(
......
python/paddle/framework/__init__.py
浏览文件 @ f8d0a358
......@@ -36,6 +36,16 @@ from ..fluid.dygraph.base import grad # noqa: F401
from .io import save # noqa: F401
from .io import load # noqa: F401
from .io_utils import _open_file_buffer # noqa: F401
from .io_utils import is_parameter # noqa: F401
from .io_utils import is_persistable # noqa: F401
from .io_utils import is_belong_to_optimizer # noqa: F401
from .io_utils import _clone_var_in_block_ # noqa: F401
from .io_utils import _pickle_loads_mac
from .io_utils import _pack_loaded_dict
from .io_utils import _unpack_saved_dict
from .io_utils import _load_program_scope
from ..fluid import monkey_patch_variable
from ..fluid.dygraph import monkey_patch_math_varbase
from ..fluid.framework import disable_signal_handler # noqa: F401
......
python/paddle/framework/io.py
浏览文件 @ f8d0a358
......@@ -37,14 +37,6 @@ from paddle.fluid.framework import (
_non_static_mode,
_varbase_creator,
)
from paddle.fluid.io import _is_file_path, _is_memory_buffer
from paddle.fluid.io import _legacy_save as _legacy_static_save
from paddle.fluid.io import (
_open_file_buffer,
_pack_loaded_dict,
_pickle_loads_mac,
_unpack_saved_dict,
)
from paddle.jit.api import _SaveLoadConfig
from paddle.jit.translated_layer import (
INFER_MODEL_SUFFIX,
......@@ -53,6 +45,16 @@ from paddle.jit.translated_layer import (
_construct_program_holders,
)
from .io_utils import (
_is_file_path,
_is_memory_buffer,
_legacy_static_save,
_open_file_buffer,
_pack_loaded_dict,
_pickle_loads_mac,
_unpack_saved_dict,
)
__all__ = []
......
python/paddle/framework/io_utils.py 0 → 100644
浏览文件 @ f8d0a358
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import math
import os
import pickle
import sys
from io import BytesIO
import numpy as np
import paddle
from paddle.fluid import core
from paddle.fluid.framework import Parameter, Variable, static_only
from paddle.fluid.log_helper import get_logger
from paddle.fluid.wrapped_decorator import signature_safe_contextmanager
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
# This file contains various utility functions that are used in static.io(io related api that used in static graph)
# and framework.io(io related api that used in dygraph)
class _open_buffer:
def __init__(self, buffer):
self.buffer = buffer
def __enter__(self):
return self.buffer
class _buffer_reader(_open_buffer):
def __init__(self, buffer):
super().__init__(buffer)
self.initial_tell = self.buffer.tell()
def __exit__(self, *args):
# `args[0]` is type of exception. When the `read` is abnormal, the file pointer returns to the initial position.
if args[0] is not None:
self.buffer.seek(self.initial_tell)
class _buffer_writer(_open_buffer):
def __exit__(self, *args):
self.buffer.flush()
def _is_file_path(path):
return isinstance(path, str)
def _open_file_buffer(path_or_buffer, mode):
if _is_file_path(path_or_buffer):
return open(path_or_buffer, mode)
else:
if 'w' in mode:
return _buffer_writer(path_or_buffer)
elif 'r' in mode:
return _buffer_reader(path_or_buffer)
else:
raise ValueError(
"Expected 'r' or 'w' in mode but got {}".format(mode)
)
def _is_memory_buffer(buffer):
return isinstance(buffer, BytesIO)
def is_persistable(var):
"""
Check whether the given variable is persistable.
Args:
var(Variable): The variable to be checked.
Returns:
bool: True if the given `var` is persistable
False if not.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
param = fluid.default_main_program().global_block().var('fc.b')
res = fluid.io.is_persistable(param)
"""
if (
var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH
or var.desc.type() == core.VarDesc.VarType.FETCH_LIST
or var.desc.type() == core.VarDesc.VarType.READER
):
return False
return var.persistable
def is_parameter(var):
"""
Check whether the given variable is an instance of Parameter.
Args:
var(Variable): The variable to be checked.
Returns:
bool: True if the given `var` is an instance of Parameter,
False if not.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
param = fluid.default_main_program().global_block().var('fc.w')
res = fluid.io.is_parameter(param)
"""
return isinstance(var, Parameter)
def is_belong_to_optimizer(var):
if not (isinstance(var, Parameter) or var.desc.need_check_feed()):
return is_persistable(var)
return False
def _clone_var_in_block_(block, var):
assert isinstance(var, Variable)
if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR:
return block.create_var(
name=var.name,
shape=var.shape,
dtype=var.dtype,
type=var.type,
lod_level=var.lod_level,
persistable=True,
)
else:
return block.create_var(
name=var.name,
shape=var.shape,
dtype=var.dtype,
type=var.type,
persistable=True,
)
@signature_safe_contextmanager
def _load_program_scope(main=None, startup=None, scope=None):
prog = main if main else paddle.fluid.Program()
startup_prog = startup if startup else paddle.fluid.Program()
scope = scope if scope else paddle.fluid.core.Scope()
with paddle.fluid.scope_guard(scope):
with paddle.fluid.program_guard(prog, startup_prog):
with paddle.fluid.unique_name.guard():
with paddle.fluid.framework._dygraph_guard(None):
yield
@static_only
def _legacy_static_save(param_dict, model_path, protocol=2):
def get_tensor(var):
if isinstance(var, (core.VarBase, core.eager.Tensor)):
return var.numpy()
elif isinstance(var, core.LoDTensor):
return np.array(var)
return var
param_dict = {name: get_tensor(param_dict[name]) for name in param_dict}
# When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
if (
_is_file_path(model_path)
and sys.platform == 'darwin'
and sys.version_info.major == 3
):
pickle_bytes = pickle.dumps(param_dict, protocol=protocol)
with open(model_path, 'wb') as f:
max_bytes = 2**30
for i in range(0, len(pickle_bytes), max_bytes):
f.write(pickle_bytes[i : i + max_bytes])
else:
with _open_file_buffer(model_path, 'wb') as f:
pickle.dump(param_dict, f, protocol=protocol)
def _pickle_loads_mac(path, f):
pickle_bytes = bytearray(0)
file_size = os.path.getsize(path)
max_bytes = 2**30
for _ in range(0, file_size, max_bytes):
pickle_bytes += f.read(max_bytes)
load_result = pickle.loads(pickle_bytes, encoding='latin1')
return load_result
def _pack_loaded_dict(load_obj):
if isinstance(load_obj, dict):
unpack_info = 'UnpackBigParamInfor@@'
if unpack_info in load_obj:
removes = []
for key, value in load_obj[unpack_info].items():
slices = [load_obj[part] for part in value["slices"]]
load_obj[key] = np.concatenate(slices).reshape(
value["OriginShape"]
)
removes += value["slices"]
for key in removes:
load_obj.pop(key)
load_obj.pop(unpack_info)
return load_obj
def _unpack_saved_dict(saved_obj, protocol):
temp_saved_obj = {}
unpack_infor = {}
# When pickle protocol=2 or protocol=3 the serialized object cannot be larger than 4G.
if 1 < protocol < 4:
if isinstance(saved_obj, dict):
for key, value in saved_obj.items():
if isinstance(value, np.ndarray):
MAX_NUMBER_OF_ELEMENT = int(
(2**30 - 1) / value.dtype.itemsize
)
num_element = np.prod(value.shape)
if num_element > MAX_NUMBER_OF_ELEMENT:
unpack_infor[key] = {}
unpack_infor[key]["OriginShape"] = value.shape
unpack_infor[key]["slices"] = []
value = value.flatten()
for i in range(
int(
math.ceil(
num_element * 1.0 / MAX_NUMBER_OF_ELEMENT
)
)
):
part_name = key + "@@." + str(i)
unpack_infor[key]["slices"].append(part_name)
temp_saved_obj[part_name] = value[
i
* MAX_NUMBER_OF_ELEMENT : MAX_NUMBER_OF_ELEMENT
* (i + 1)
]
if unpack_infor:
for key, value in unpack_infor.items():
if key in saved_obj:
saved_obj.pop(key)
for part in value['slices']:
saved_obj[part] = temp_saved_obj[part]
saved_obj['UnpackBigParamInfor@@'] = unpack_infor
return saved_obj
python/paddle/hapi/model.py
浏览文件 @ f8d0a358
......@@ -34,9 +34,9 @@ from paddle.fluid.executor import global_scope
from paddle.fluid.framework import Variable
from paddle.fluid.framework import _current_expected_place as _get_device
from paddle.fluid.framework import _get_paddle_place, _non_static_mode
from paddle.fluid.io import is_belong_to_optimizer
from paddle.fluid.layers import collective
from paddle.fluid.layers.utils import flatten
from paddle.framework.io_utils import is_belong_to_optimizer
from paddle.io import DataLoader, Dataset, DistributedBatchSampler
from paddle.jit.translated_layer import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX
from paddle.metric import Metric
......
python/paddle/incubate/distributed/fleet/fleet_util.py
浏览文件 @ f8d0a358
......@@ -23,6 +23,7 @@ import time
import numpy as np
import paddle
import paddle.fluid as fluid
from paddle.distributed.fleet.utils.fs import HDFSClient
from paddle.fluid.log_helper import get_logger
......@@ -1087,11 +1088,13 @@ class FleetUtil:
vars = [program.global_block().var(i) for i in var_names]
with fluid.scope_guard(scope):
if save_combine:
fluid.io.save_vars(
paddle.static.io.save_vars(
executor, "./", program, vars=vars, filename=model_name
)
else:
fluid.io.save_vars(executor, model_name, program, vars=vars)
paddle.static.io.save_vars(
executor, model_name, program, vars=vars
)
configs = {
"fs.default.name": hadoop_fs_name,
......
python/paddle/incubate/distributed/fleet/utils.py
浏览文件 @ f8d0a358
......@@ -22,6 +22,7 @@ from google.protobuf import text_format
import paddle
import paddle.fluid as fluid
import paddle.framework.io_utils as io_utils
from paddle.fluid import core, debugger
from paddle.fluid.framework import Program
from paddle.fluid.proto import framework_pb2
......@@ -92,7 +93,7 @@ def check_pruned_program_vars(train_prog, pruned_prog):
pruned_vars = [
(v.name, v)
for v in pruned_prog.list_vars()
if fluid.io.is_persistable(v)
if io_utils.is_persistable(v)
]
pruned_vars = OrderedDict(pruned_vars)
pruned_vars_name = [name for name in pruned_vars]
......@@ -451,7 +452,7 @@ def check_saved_vars_try_dump(
os.path.join(dump_dir, dump_prog_fn), is_text_dump_program
)
saved_params = [
v for v in dump_prog.list_vars() if fluid.io.is_persistable(v)
v for v in dump_prog.list_vars() if io_utils.is_persistable(v)
]
logger.info(
"persistable vars in dump program: {}".format(
......@@ -477,7 +478,7 @@ def parse_program(program, output_dir):
# persistable vars
output = {}
persistable_vars = [
v for v in program.list_vars() if fluid.io.is_persistable(v)
v for v in program.list_vars() if io_utils.is_persistable(v)
]
output["persistable_vars"] = [
{
......
python/paddle/jit/api.py
浏览文件 @ f8d0a358
......@@ -1209,7 +1209,11 @@ def save(layer, path, input_spec=None, **configs):
paddle.static.save_vars(
Executor(_current_expected_place()),
dirname=model_path,
vars=list(filter(paddle.fluid.io.is_persistable, ordered_vars)),
vars=list(
filter(
paddle.framework.io_utils.is_persistable, ordered_vars
)
),
filename=params_filename,
)
# save property
......
python/paddle/static/__init__.py
浏览文件 @ f8d0a358
......@@ -27,6 +27,13 @@ from .io import serialize_program # noqa: F401
from .io import load_from_file # noqa: F401
from .io import save_to_file # noqa: F401
from .io import normalize_program # noqa: F401
from .io import is_persistable # noqa: F401
from .io import save_vars # noqa: F401
from .io import load_vars # noqa: F401
from .io import save # noqa: F401
from .io import load # noqa: F401
from .io import load_program_state # noqa: F401
from .io import set_program_state # noqa: F401
from ..fluid import Scope # noqa: F401
from .input import data # noqa: F401
from .input import InputSpec # noqa: F401
......@@ -66,13 +73,6 @@ from ..fluid.param_attr import WeightNormParamAttr # noqa: F401
from ..fluid.optimizer import Optimizer # noqa: F401
from ..fluid.optimizer import Adam # noqa: F401
from ..fluid.optimizer import ExponentialMovingAverage # noqa: F401
from ..fluid.io import save # noqa: F401
from ..fluid.io import load # noqa: F401
from ..fluid.io import load_program_state # noqa: F401
from ..fluid.io import set_program_state # noqa: F401
from ..fluid.io import load_vars # noqa: F401
from ..fluid.io import save_vars # noqa: F401
from ..fluid.io import batch # noqa: F401
from ..fluid.contrib.layers import ctr_metric_bundle # noqa: F401
from ..fluid.layers import exponential_decay # noqa: F401
......
python/paddle/static/io.py
浏览文件 @ f8d0a358
......@@ -16,6 +16,8 @@ import errno
import inspect
import logging
import os
import pickle
import sys
import warnings
import numpy as np
......@@ -30,10 +32,20 @@ from paddle.fluid import (
program_guard,
unique_name,
)
from paddle.fluid.executor import global_scope
from paddle.fluid.framework import Parameter, static_only
from paddle.fluid.executor import Executor, global_scope
from paddle.fluid.framework import Parameter, dygraph_not_support, static_only
from paddle.fluid.io import append_fetch_ops, prepend_feed_ops
from paddle.fluid.log_helper import get_logger
from paddle.framework.io_utils import (
_clone_var_in_block_,
_load_program_scope,
_pack_loaded_dict,
_pickle_loads_mac,
_unpack_saved_dict,
is_belong_to_optimizer,
is_parameter,
is_persistable,
)
__all__ = []
......@@ -230,37 +242,6 @@ def normalize_program(program, feed_vars, fetch_vars):
return copy_program
def is_persistable(var):
"""
Check whether the given variable is persistable.
Args:
var(Variable): The variable to be checked.
Returns:
bool: True if the given `var` is persistable
False if not.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
param = fluid.default_main_program().global_block().var('fc.b')
res = fluid.io.is_persistable(param)
"""
if (
var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH
or var.desc.type() == core.VarDesc.VarType.FETCH_LIST
or var.desc.type() == core.VarDesc.VarType.READER
):
return False
return var.persistable
@static_only
def serialize_program(feed_vars, fetch_vars, **kwargs):
"""
......@@ -882,3 +863,1048 @@ def load_inference_model(path_prefix, executor, **kwargs):
]
return [program, feed_target_names, fetch_targets]
@dygraph_not_support
def save_vars(
executor,
dirname,
main_program=None,
vars=None,
predicate=None,
filename=None,
):
"""
Save specific variables in the `Program` to files.
There are two ways to specify the variables to be saved: set variables in
a list and assign it to the `vars`, or use the `predicate` function to select
variables that make `predicate(variable) == True`. The first way has a higher priority.
The `dirname` is used to specify the folder where to save variables.
If you prefer to save variables in separate files in the `dirname` folder,
do not set `filename`. If you prefer to save all variables in a single file,
use `filename` to specify it.
Args:
executor(Executor): The executor to run for saving variables.
dirname(str, optional): The folder where to save variables.
When you need to save the parameter to the memory, set it to None.
main_program(Program, optional): The program whose variables will be saved.
If it is None, the default main program will
be used automatically.
Default: None
vars(list[Variable], optional): The list contains all variables to be saved.
Default: None
predicate(function, optional): The function selects the variables that make
`predicate(variable) == True`.
Default: None
filename(str, optional): If you prefer to save all variables in a single file,
use `filename` to specify it. Otherwise, let `filename` be None.
Default: None
Returns:
str: When saving parameters to a file, returns None.
When saving parameters to memory, returns a binary string containing parameters.
Raises:
TypeError: If `main_program` is not an instance of Program nor None.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
main_prog = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(main_prog, startup_prog):
data = paddle.static.data(name="img", shape=[64, 784])
w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
hidden_w = paddle.matmul(x=data, y=w)
hidden_b = paddle.add(hidden_w, b)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup_prog)
# The first usage: use `vars` to set the saved variables.
var_list = [w, b]
path = "./my_paddle_vars"
fluid.io.save_vars(executor=exe, dirname=path, vars=var_list,
filename="vars_file")
# w and b will be save in a file named "var_file".
# The second usage: use `predicate` to select the saved variable.
def name_has_fc(var):
res = "fc" in var.name
return res
param_path = "./my_paddle_model"
fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog, vars=None, predicate = name_has_fc)
# all variables whose names contain "fc " are saved.
"""
save_to_memory = False
if dirname is None and filename is None:
save_to_memory = True
main_program = paddle.static.io._get_valid_program(main_program)
if vars is None:
return save_vars(
executor,
main_program=main_program,
dirname=dirname,
vars=list(filter(predicate, main_program.list_vars())),
filename=filename,
)
else:
params_var_name = "saved_params"
# give warning when there is no var in model
if len(list(vars)) == 0:
warnings.warn(
"no variable in your model, please ensure there are any variables in your model to save"
)
return None
save_program = Program()
save_block = save_program.global_block()
save_var_map = {}
for each_var in vars:
# NOTE: don't save the variable which type is RAW
if each_var.type == core.VarDesc.VarType.RAW:
continue
new_var = _clone_var_in_block_(save_block, each_var)
if filename is None and save_to_memory is False:
save_file_path = os.path.join(
os.path.normpath(dirname), new_var.name
)
save_block.append_op(
type='save',
inputs={'X': [new_var]},
outputs={},
attrs={'file_path': os.path.normpath(save_file_path)},
)
else:
save_var_map[new_var.name] = new_var
if filename is not None or save_to_memory:
save_var_list = []
for name in sorted(save_var_map.keys()):
save_var_list.append(save_var_map[name])
save_path = str()
if save_to_memory is False:
save_path = os.path.join(os.path.normpath(dirname), filename)
saved_params = save_block.create_var(
type=core.VarDesc.VarType.RAW, name=params_var_name
)
saved_params.desc.set_persistable(True)
save_block.append_op(
type='save_combine',
inputs={'X': save_var_list},
outputs={'Y': saved_params},
attrs={
'file_path': save_path,
'save_to_memory': save_to_memory,
},
)
# NOTE(zhiqiu): save op will add variable kLookupTablePath in save_program.desc,
# which leads to diff on save_program and its desc. Call _sync_with_cpp
# to keep consistency.
save_program._sync_with_cpp()
executor.run(save_program)
if save_to_memory:
return global_scope().find_var(params_var_name).get_bytes()
def load_vars(
executor,
dirname,
main_program=None,
vars=None,
predicate=None,
filename=None,
):
"""
:api_attr: Static Graph
This API loads variables from files by executor.
There are two ways to specify the variables to be loaded: the first way, set
variables in a list and assign it to the `vars`; the second way, use the
`predicate` function to select variables that make `predicate(variable) == True`.
The first way has a higher priority.
The `dirname` is used to specify the folder where to load variables.
If variables were saved in separate files in the folder `dirname`,
set `filename` None. If all variables were saved in a single file,
use `filename` to specify it.
Args:
executor(Executor): The executor to run for loading variables.
dirname(str): The folder where to load the variables.
main_program(Program, optional): The program whose variables will be loaded.
If it is None, the default main program will
be used automatically.
Default: None
vars(list[Variable], optional): The list that contains all variables to be loaded.
Default: None
predicate(function, optional): The function selects variables that make
`predicate(variable) == True`.
Default: None
filename(str, optional): The file which saved all required variables. If variables
were saved in separate files, set it to be None.
Default: None
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
paddle.enable_static()
main_prog = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(main_prog, startup_prog):
data = paddle.static.data(name="img", shape=[64, 784])
w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
hidden_w = paddle.matmul(x=data, y=w)
hidden_b = paddle.add(hidden_w, b)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup_prog)
# The first usage: using `vars` to specify the variables.
path = "./my_paddle_vars"
var_list = [w, b]
fluid.io.save_vars(executor=exe, dirname=path, vars=var_list,
filename="vars_file")
fluid.io.load_vars(executor=exe, dirname=path, vars=var_list,
filename="vars_file")
# w and b will be loaded, and they are supposed to
# be saved in the same file named 'var_file' in the path "./my_paddle_vars".
# The second usage: using the `predicate` function to select variables
param_path = "./my_paddle_model"
def name_has_fc(var):
res = "fc" in var.name
return res
fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog,
vars=None, predicate=name_has_fc)
fluid.io.load_vars(executor=exe, dirname=param_path, main_program=main_prog,
vars=None, predicate=name_has_fc)
# Load All variables in the `main_program` whose name includes "fc".
# And all the variables are supposed to be saved in separate files.
"""
vars_from_memory = False
if dirname is not None:
dirname = os.path.normpath(dirname)
else:
vars_from_memory = True
if vars is None:
if main_program is None:
main_program = default_main_program()
if not isinstance(main_program, Program):
raise TypeError(
"The type of input main_program is invalid, expected type is fluid.Program, but received %s"
% type(main_program)
)
load_vars(
executor,
dirname=dirname,
main_program=main_program,
vars=list(filter(predicate, main_program.list_vars())),
filename=filename,
)
else:
load_prog = Program()
load_block = load_prog.global_block()
if main_program is None:
main_program = default_main_program()
if not isinstance(main_program, Program):
raise TypeError(
"The type of input main_program is invalid, expected type is fluid.Program, but received %s"
% type(main_program)
)
# save origin param shape
orig_para_shape = {}
load_var_map = {}
check_vars = []
sparse_vars = []
for each_var in vars:
assert isinstance(each_var, Variable)
if each_var.type == core.VarDesc.VarType.RAW:
continue
if isinstance(each_var, Parameter):
orig_para_shape[each_var.name] = tuple(
each_var.desc.get_shape()
)
if each_var.type == core.VarDesc.VarType.SELECTED_ROWS:
sparse_vars.append(each_var)
continue
new_var = _clone_var_in_block_(load_block, each_var)
check_vars.append(each_var)
if filename is None:
if dirname is None:
raise ValueError(
"The directory path and params cannot be None at the same time."
)
load_block.append_op(
type='load',
inputs={},
outputs={'Out': [new_var]},
attrs={'file_path': os.path.join(dirname, new_var.name)},
)
else:
load_var_map[new_var.name] = new_var
for each_var in sparse_vars:
assert isinstance(each_var, Variable)
if filename is not None:
raise ValueError(
"SelectedRows can not be load with load_combine"
)
new_var = _clone_var_in_block_(load_block, each_var)
var_path = os.path.join(dirname, new_var.name)
if not os.path.exists(var_path):
raise ValueError(
"SelectedRows var {} can not find at {}".format(
new_var.name, var_path
)
)
if os.path.isfile(var_path):
load_block.append_op(
type='load',
inputs={},
outputs={'Out': [new_var]},
attrs={'file_path': os.path.join(dirname, new_var.name)},
)
else:
blocks = []
block_paths = os.listdir(var_path)
for block in block_paths:
if block.startswith(new_var.name):
blocks.append(block)
slices = []
for block in blocks:
slice = load_block.create_var(
name=block,
type=new_var.type,
shape=new_var.shape,
dtype=new_var.dtype,
persistable=False,
)
slices.append(slice)
file_path = os.path.join(var_path, block, "Param")
load_block.append_op(
type='load',
inputs={},
outputs={'Out': [slice]},
attrs={'file_path': file_path},
)
load_block.append_op(
type='lookup_sparse_table_merge',
inputs={'X': slices},
outputs={'Out': new_var},
attrs={},
)
if filename is not None:
load_var_list = []
for name in sorted(load_var_map.keys()):
load_var_list.append(load_var_map[name])
if vars_from_memory is False:
filename = os.path.join(dirname, filename)
load_block.append_op(
type='load_combine',
inputs={},
outputs={"Out": load_var_list},
attrs={
'file_path': filename,
'model_from_memory': vars_from_memory,
},
)
executor.run(load_prog)
# check var shape
for each_var in check_vars:
if not isinstance(each_var, Parameter):
continue
var_temp = paddle.fluid.global_scope().find_var(each_var.name)
assert var_temp is not None, "can't not find var: " + each_var.name
new_shape = (np.array(var_temp.get_tensor())).shape
assert each_var.name in orig_para_shape, (
each_var.name + "MUST in var list"
)
orig_shape = orig_para_shape.get(each_var.name)
if new_shape != orig_shape:
raise RuntimeError(
"Variable's shape does not match, the Program requires a parameter with the shape of ({}), "
"while the loaded parameter (namely [ {} ]) has a shape of ({}).".format(
orig_shape, each_var.name, new_shape
)
)
@static_only
def save(program, model_path, protocol=4, **configs):
"""
This function save parameters, optimizer information and network description to model_path.
The parameters contains all the trainable Tensor, will save to a file with suffix ".pdparams".
The optimizer information contains all the Tensor used by optimizer. For Adam optimizer, contains beta1, beta2, momentum etc. All the information will save to a file with suffix ".pdopt". (If the optimizer have no Tensor need to save (like SGD), the fill will not generated).
The network description is the description of the program. It's only used for deployment. The description will save to a file with a suffix ".pdmodel".
Args:
program(Program) : The program to saved.
model_path(str): the file prefix to save the program. The format is "dirname/file_prefix". If file_prefix is empty str. A exception will be raised
protocol(int, optional): The protocol version of pickle module must be greater than 1 and less than 5.
Default: 4
configs(dict, optional) : optional keyword arguments.
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.static as static
paddle.enable_static()
x = static.data(name="x", shape=[10, 10], dtype='float32')
y = static.nn.fc(x, 10)
z = static.nn.fc(y, 10)
place = paddle.CPUPlace()
exe = static.Executor(place)
exe.run(static.default_startup_program())
prog = static.default_main_program()
static.save(prog, "./temp")
"""
base_name = os.path.basename(model_path)
assert (
base_name != ""
), "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received model_path is empty string."
if 'pickle_protocol' in configs:
protocol = configs['pickle_protocol']
warnings.warn(
"'pickle_protocol' is a deprecated argument. Please use 'protocol' instead."
)
if not isinstance(protocol, int):
raise ValueError(
"The 'protocol' MUST be `int`, but received {}".format(
type(protocol)
)
)
if protocol < 2 or protocol > 4:
raise ValueError(
"Expected 1<'protocol'<5, but received protocol={}".format(protocol)
)
dir_name = os.path.dirname(model_path)
if dir_name and not os.path.exists(dir_name):
os.makedirs(dir_name)
def get_tensor(var):
t = global_scope().find_var(var.name).get_tensor()
return np.array(t)
parameter_list = list(filter(is_parameter, program.list_vars()))
param_dict = {p.name: get_tensor(p) for p in parameter_list}
param_dict = _unpack_saved_dict(param_dict, protocol)
# When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
if sys.platform == 'darwin' and sys.version_info.major == 3:
pickle_bytes = pickle.dumps(param_dict, protocol=protocol)
with open(model_path + ".pdparams", 'wb') as f:
max_bytes = 2**30
for i in range(0, len(pickle_bytes), max_bytes):
f.write(pickle_bytes[i : i + max_bytes])
else:
with open(model_path + ".pdparams", 'wb') as f:
pickle.dump(param_dict, f, protocol=protocol)
optimizer_var_list = list(
filter(is_belong_to_optimizer, program.list_vars())
)
opt_dict = {p.name: get_tensor(p) for p in optimizer_var_list}
with open(model_path + ".pdopt", 'wb') as f:
pickle.dump(opt_dict, f, protocol=protocol)
main_program = program.clone()
program.desc.flush()
main_program.desc._set_version()
paddle.fluid.core.save_op_version_info(program.desc)
with open(model_path + ".pdmodel", "wb") as f:
f.write(program.desc.serialize_to_string())
@static_only
def load(program, model_path, executor=None, var_list=None):
"""
:api_attr: Static Graph
This function get parameters and optimizer information from program, and then get corresponding value from file.
An exception will throw if shape or dtype of the parameters is not match.
This function can also load model file saved with [ save_params, save_persistables, save_vars ].
var_list can not be None when load single model file
( filename is not None When save_params, save_persistables or save_vars is called ).
Args:
program(Program): The program will be loaded
model_path(str): The file prefix store the program
executor(Executor, optional): The executor used for initialize the parameter
When startup program is not run.
var_list(list|tuple, optional): The Tensor list/tuple to load single model file saved with
[ save_params, save_persistables, save_vars ].
Default: None
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.static as static
paddle.enable_static()
x = static.data(name="x", shape=[10, 10], dtype='float32')
y = static.nn.fc(x, 10)
z = static.nn.fc(y, 10)
place = paddle.CPUPlace()
exe = static.Executor(place)
exe.run(static.default_startup_program())
prog = static.default_main_program()
static.save(prog, "./temp")
static.load(prog, "./temp")
"""
assert executor is None or isinstance(executor, Executor)
model_prefix = model_path
if model_prefix.endswith(".pdparams"):
model_prefix = model_prefix[:-9]
elif model_prefix.endswith(".pdopt"):
model_prefix = model_prefix[:-6]
elif model_prefix.endswith(".pdmodel"):
model_prefix = model_prefix[:-8]
parameter_file_name = model_prefix + ".pdparams"
if not os.path.exists(parameter_file_name):
# model file save by fluid.save not found, try to load model file saved with
# [save_vars, save_params, save_persistables]
_logger.debug(
"{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format(
parameter_file_name
)
)
if executor is None:
raise ValueError(
"executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
)
if var_list is not None:
var_list_names = [var.name for var in var_list]
else:
var_list_names = None
if os.path.isdir(model_path):
binary_file_set = set()
for root, dirs, files in os.walk(model_path, topdown=False):
for f in files:
binary_file_set.add(
os.path.join(root, f).replace("\\", "/")
)
program_var_list = list(program.list_vars())
loaded_var_list = []
for var in program_var_list:
var_path = os.path.join(model_path, var.name).replace("\\", "/")
load_condition = (
var_list_names is None or var.name in var_list_names
)
if var_path in binary_file_set and load_condition:
loaded_var_list.append(var)
binary_file_set.remove(var_path)
if len(binary_file_set) > 0:
unused_var_list = " ".join(list(binary_file_set))
_logger.warning(
"variable file [ %s ] not used"
% (" ".join(list(binary_file_set)))
)
try:
load_vars(
executor=executor, dirname=model_path, vars=loaded_var_list
)
except RuntimeError as e:
_logger.error(e)
raise e
except:
raise RuntimeError(
"Failed to load model file, please make sure model file is saved with the "
"following APIs: save_params, save_persistables, save_vars"
)
return
elif os.path.isfile(model_path):
if var_list is None:
raise ValueError(
"var_list is required when loading model file saved with [ save_params, save_persistables, save_vars ]"
)
program_var_list = program.list_vars()
program_var_name_set = set([var.name for var in program_var_list])
# check all the variable inlcuded in program
for var in var_list:
if var.name not in program_var_name_set:
raise LookupError(
"loaded var [{}] is not in program variable list"
)
dir_name, file_name = os.path.split(model_path)
try:
load_vars(
executor=executor,
dirname=dir_name,
vars=var_list,
filename=file_name,
)
except RuntimeError as e:
_logger.error(e)
raise e
except:
raise RuntimeError(
"Failed to load model file , please make sure model file is saved with the "
"the following APIs: [ save_params, save_persistables, save_vars ]. "
"When these API called, filename CANNOT be None"
)
return
def set_var(var, ndarray):
t = global_scope().find_var(var.name).get_tensor()
p = t._place()
if p.is_cpu_place():
place = paddle.fluid.CPUPlace()
elif p.is_cuda_pinned_place():
place = paddle.fluid.CUDAPinnedPlace()
elif p.is_xpu_place():
p = paddle.fluid.core.Place()
p.set_place(t._place())
place = paddle.fluid.XPUPlace(p.xpu_device_id())
elif p.is_npu_place():
p = paddle.fluid.core.Place()
p.set_place(t._place())
place = paddle.fluid.NPUPlace(p.npu_device_id())
elif p.is_mlu_place():
p = paddle.fluid.core.Place()
p.set_place(t._place())
place = paddle.fluid.MLUPlace(p.mlu_device_id())
else:
p = paddle.fluid.core.Place()
p.set_place(t._place())
place = paddle.fluid.CUDAPlace(p.gpu_device_id())
t.set(ndarray, place)
parameter_list = list(filter(is_parameter, program.list_vars()))
if executor:
paddle.fluid.core._create_loaded_parameter(
parameter_list, global_scope(), executor._default_executor
)
with open(parameter_file_name, 'rb') as f:
# When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
if sys.platform == 'darwin' and sys.version_info.major == 3:
load_dict = _pickle_loads_mac(parameter_file_name, f)
else:
load_dict = pickle.load(f, encoding='latin1')
load_dict = _pack_loaded_dict(load_dict)
for v in parameter_list:
assert (
v.name in load_dict
), "Can not find [{}] in model file [{}]".format(
v.name, parameter_file_name
)
set_var(v, load_dict[v.name])
optimizer_var_list = list(
filter(is_belong_to_optimizer, program.list_vars())
)
if len(optimizer_var_list) > 0:
opt_file_name = model_prefix + ".pdopt"
assert os.path.exists(
opt_file_name
), "Optimizer file [{}] not exits".format(opt_file_name)
if executor:
paddle.fluid.core._create_loaded_parameter(
optimizer_var_list, global_scope(), executor._default_executor
)
with open(opt_file_name, 'rb') as f:
load_dict = pickle.load(f, encoding='latin1')
for v in optimizer_var_list:
assert (
v.name in load_dict
), "Can not find [{}] in model file [{}]".format(
v.name, opt_file_name
)
set_var(v, load_dict[v.name])
@static_only
def set_program_state(program, state_dict):
"""
Set program parameter from state_dict
An exception will throw if shape or dtype of the parameters is not match.
NOTICE: This function MUST called after run start_up_program
Args:
program(Program): The program to be set
state_dict(dict): the dict store Parameter and optimizer information
Returns:
None
Examples:
.. code-block:: python
import paddle
import paddle.static as static
paddle.enable_static()
x = static.data(name="x", shape=[10, 10], dtype='float32')
y = static.nn.fc(x, 10)
z = static.nn.fc(y, 10)
place = paddle.CPUPlace()
exe = static.Executor(place)
exe.run(static.default_startup_program())
prog = static.default_main_program()
static.save(prog, "./temp")
program_state = static.load_program_state("./temp")
static.set_program_state(prog, program_state)
"""
state_dict = _pack_loaded_dict(state_dict)
parameter_list = list(filter(is_persistable, program.list_vars()))
used_para_list = {}
for para in parameter_list:
var_temp = paddle.fluid.global_scope().find_var(para.name)
assert (
var_temp is not None
), "Variable [ {} ] Not found, Please make sure run startup program".format(
para.name
)
if para.name in state_dict:
# set value from state dict
orig_para_np = np.array(var_temp.get_tensor())
new_para_np = state_dict[para.name]
assert orig_para_np.shape == new_para_np.shape, (
"Parameter's shape does not match, the Program requires a parameter with the shape of ({}), "
"while the loaded parameter (namely [ {} ]) has a shape of ({}).".format(
orig_para_np.shape, para.name, new_para_np.shape
)
)
assert orig_para_np.dtype == new_para_np.dtype, (
"Parameter's data type does not match, the Program requires a parameter with a dtype of ({}), "
"while the loaded parameter (namely [ {} ]) has a dtype of ({}).".format(
orig_para_np.dtype, para.name, new_para_np.dtype
)
)
ten = var_temp.get_tensor()
ten_place = ten._place()
# assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \
# "Place not support, only support CPUPlace and GPUPlace, now is {}".format(str(ten_place))
py_place = paddle.fluid.CPUPlace()
if ten_place.is_cuda_pinned_place():
place = paddle.fluid.CUDAPinnedPlace()
elif ten_place.is_gpu_place():
p = paddle.fluid.core.Place()
p.set_place(ten_place)
py_place = paddle.fluid.CUDAPlace(p.gpu_device_id())
elif ten_place.is_xpu_place():
p = paddle.fluid.core.Place()
p.set_place(ten_place)
py_place = paddle.fluid.XPUPlace(p.xpu_device_id())
elif ten_place.is_npu_place():
p = paddle.fluid.core.Place()
p.set_place(ten_place)
py_place = paddle.fluid.NPUPlace(p.npu_device_id())
elif ten_place.is_mlu_place():
p = paddle.fluid.core.Place()
p.set_place(ten_place)
py_place = paddle.fluid.MLUPlace(p.mlu_device_id())
ten.set(new_para_np, py_place)
used_para_list[para.name] = 1
unused_para_list = []
for k, v in state_dict.items():
if k not in used_para_list:
unused_para_list.append(k)
if len(unused_para_list) > 0:
warnings.warn(
"This list is not set, Because of Paramerter not found in program. There are: {}".format(
" ".join(unused_para_list)
)
)
@dygraph_not_support
def get_program_persistable_vars(program):
"""
Get all the persistable vars from Program.
Args:
var(Program): The Program to get persistable vars
Returns:
list: The list contains all persistable vars in the program
Examples:
.. code-block:: python
import paddle
import paddle.static.io as io
import paddle.fluid as fluid
paddle.enable_static()
data = fluid.data(name="img", shape=[64, 784])
w = paddle.create_parameter(shape=[784, 200], dtype='float32', name='fc_w')
b = paddle.create_parameter(shape=[200], dtype='float32', name='fc_b')
list_para = io.get_program_persistable_vars( fluid.default_main_program() )
"""
return list(filter(is_persistable, program.list_vars()))
def load_program_state(model_path, var_list=None):
"""
Load program state from local file
Args:
model_path(str): The file prefix store the program
var_list(list|tuple, optional): The Tensor list/tuple to load saved with
[ save_params, save_persistables, save_vars ].
Default: None.
The var_list is only used to get name,
will not be modified.
Returns:
state_dict(dict): the dict store Parameter and optimizer information
Examples:
.. code-block:: python
import paddle
import paddle.static as static
paddle.enable_static()
x = static.data(name="x", shape=[10, 10], dtype='float32')
y = static.nn.fc(x, 10)
z = static.nn.fc(y, 10)
place = paddle.CPUPlace()
exe = static.Executor(place)
exe.run(static.default_startup_program())
prog = static.default_main_program()
static.save(prog, "./temp")
program_state = static.load_program_state("./temp")
"""
model_prefix = model_path
if model_prefix.endswith(".pdparams"):
model_prefix = model_prefix[:-9]
elif model_prefix.endswith(".pdopt"):
model_prefix = model_prefix[:-6]
elif model_prefix.endswith(".pdmodel"):
model_prefix = model_prefix[:-8]
parameter_file_name = model_prefix + ".pdparams"
if not os.path.exists(parameter_file_name):
# model file saved with fluid.save is not found, try to load model file saved with
# [save_vars, save_params, save_persistables]
_logger.debug(
"{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format(
parameter_file_name
)
)
var_name_list = []
if var_list is None and os.path.isfile(model_path):
raise ValueError(
"var_list can not be None when model_path is a file type"
)
for root, dirs, files in os.walk(model_path, topdown=False):
for f in files:
file_path = os.path.join(root, f)
var_temp_name = os.path.relpath(file_path, model_path)
var_temp_name = var_temp_name.replace("\\", "/")
var_name_list.append(var_temp_name)
with _load_program_scope():
load_prog = Program()
load_block = load_prog.global_block()
def clone_var_to_block(block, var):
if not isinstance(var, Variable):
raise TypeError("value in var_list must be variable")
return block.create_var(
name=var.name,
shape=var.shape,
dtype=var.dtype,
type=var.type,
lod_level=var.lod_level
if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR
else None,
persistable=True,
)
def _load_vars_with_try_catch(
exe, dirname, vars, filename, raise_error=True
):
try:
load_vars(
executor=exe,
dirname=dirname,
vars=vars,
filename=filename,
)
return True
except:
error_str = (
"Failed to load model/variables `%s`, please make sure "
"model/variables file is saved with the following APIs: "
"save_params, save_persistables, save_vars."
)
filenames = (
[var.name for var in vars]
if filename is None
else filename
)
if raise_error:
raise RuntimeError(error_str % filenames)
else:
warnings.warn(error_str % filenames, RuntimeWarning)
return False
place = paddle.fluid.CPUPlace()
exe = paddle.fluid.Executor(place)
loaded_var_list = []
if os.path.isfile(model_path):
# when model_path is file, var_list cannot be None
dir_name, file_name = os.path.split(model_path)
for var in var_list:
loaded_var_list.append(clone_var_to_block(load_block, var))
_load_vars_with_try_catch(
exe, dir_name, loaded_var_list, file_name
)
else:
# var_list can be None or not None
if var_list is not None:
for var in var_list:
loaded_var_list.append(
clone_var_to_block(load_block, var)
)
_load_vars_with_try_catch(
exe, model_path, loaded_var_list, None
)
else:
for var_name in var_name_list:
# NOTE(chenweihang): If identify which files the user wants
# to load from the disk, we load these variables one by one.
# If a file does not exist, we only warn the user that the
# file may be an irrelevant file, but does not throw an error
# to ensure that other legal variables can be loaded.
temp_var = load_block.create_var(
name=var_name, persistable=True
)
if _load_vars_with_try_catch(
exe, model_path, [temp_var], None, False
):
loaded_var_list.append(temp_var)
res_dict = {}
for var in loaded_var_list:
res_dict[var.name] = np.asarray(
paddle.fluid.global_scope().find_var(var.name).get_tensor()
)
return res_dict
assert os.path.exists(
parameter_file_name
), "Parameter file [{}] not exits".format(parameter_file_name)
with open(parameter_file_name, 'rb') as f:
# When value of dict is lager than 4GB ,there is a Bug on 'MAC python3'
if sys.platform == 'darwin' and sys.version_info.major == 3:
para_dict = _pickle_loads_mac(parameter_file_name, f)
else:
para_dict = pickle.load(f, encoding='latin1')
para_dict = _pack_loaded_dict(para_dict)
opt_file_name = model_prefix + ".pdopt"
if os.path.exists(opt_file_name):
with open(opt_file_name, 'rb') as f:
opti_dict = pickle.load(f, encoding='latin1')
para_dict.update(opti_dict)
return para_dict
python/paddle/static/quantization/tests/test_graph.py
浏览文件 @ f8d0a358
......@@ -15,8 +15,6 @@
import os
import unittest
import numpy as np
import paddle
from paddle.fluid.framework import IrGraph
from paddle.framework import core
......@@ -103,40 +101,6 @@ class TestGraph(unittest.TestCase):
_train(origin_binary)
_train(backup_binary)
checkponit_dir = "checkpoint_gpu" if use_cuda else "checkpoint_cpu"
def _set_zero(var_name, scope, place):
var = scope.find_var(var_name).get_tensor()
var_array = np.zeros(var._get_dims()).astype("float32")
var.set(var_array, place)
sum_before = np.sum(
np.array(
paddle.static.global_scope()
.find_var('conv2d_1.w_0')
.get_tensor()
)
)
paddle.fluid.io._save_persistable_nodes(exe, checkponit_dir, graph)
_set_zero('conv2d_1.w_0', paddle.static.global_scope(), place)
set_after = np.sum(
np.array(
paddle.static.global_scope()
.find_var('conv2d_1.w_0')
.get_tensor()
)
)
self.assertEqual(set_after, 0)
paddle.fluid.io._load_persistable_nodes(exe, checkponit_dir, graph)
sum_after = np.sum(
np.array(
paddle.static.global_scope()
.find_var('conv2d_1.w_0')
.get_tensor()
)
)
self.assertEqual(sum_before, sum_after)
marked_nodes = set()
for op in graph.all_op_nodes():
if op.name().find('conv2d') > -1:
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或