Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
机器未来
Paddle
提交
24a33bed
P
Paddle
项目概览
机器未来
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
24a33bed
编写于
9月 27, 2020
作者:
C
Chen Weihang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
replace config by kwargs
上级
6b727e08
变更
13
展开全部
显示空白变更内容
内联
并排
Showing
13 changed file
with
237 addition
and
496 deletion
+237
-496
python/paddle/__init__.py
python/paddle/__init__.py
+0
-1
python/paddle/fluid/dygraph/checkpoint.py
python/paddle/fluid/dygraph/checkpoint.py
+36
-37
python/paddle/fluid/dygraph/jit.py
python/paddle/fluid/dygraph/jit.py
+92
-355
python/paddle/fluid/dygraph/static_runner.py
python/paddle/fluid/dygraph/static_runner.py
+2
-2
python/paddle/fluid/tests/unittests/dygraph_to_static/test_lac.py
...addle/fluid/tests/unittests/dygraph_to_static/test_lac.py
+1
-3
python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py
...dle/fluid/tests/unittests/dygraph_to_static/test_mnist.py
+1
-3
python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py
.../unittests/dygraph_to_static/test_save_inference_model.py
+3
-8
python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py
...fluid/tests/unittests/dygraph_to_static/test_se_resnet.py
+4
-4
python/paddle/fluid/tests/unittests/test_directory_migration.py
.../paddle/fluid/tests/unittests/test_directory_migration.py
+9
-12
python/paddle/fluid/tests/unittests/test_jit_save_load.py
python/paddle/fluid/tests/unittests/test_jit_save_load.py
+29
-47
python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py
...d/tests/unittests/test_load_state_dict_from_old_format.py
+18
-14
python/paddle/framework/__init__.py
python/paddle/framework/__init__.py
+2
-3
python/paddle/framework/io.py
python/paddle/framework/io.py
+40
-7
未找到文件。
python/paddle/__init__.py
浏览文件 @
24a33bed
...
...
@@ -234,7 +234,6 @@ from .framework import grad #DEFINE_ALIAS
from
.framework
import
no_grad
#DEFINE_ALIAS
from
.framework
import
save
#DEFINE_ALIAS
from
.framework
import
load
#DEFINE_ALIAS
from
.framework
import
SaveLoadConfig
#DEFINE_ALIAS
from
.framework
import
DataParallel
#DEFINE_ALIAS
from
.framework
import
NoamDecay
#DEFINE_ALIAS
...
...
python/paddle/fluid/dygraph/checkpoint.py
浏览文件 @
24a33bed
...
...
@@ -24,7 +24,7 @@ from . import learning_rate_scheduler
import
warnings
from
..
import
core
from
.base
import
guard
from
paddle.fluid.dygraph.jit
import
SaveLoadConfig
,
deprecate_save_load_configs
from
paddle.fluid.dygraph.jit
import
_SaveLoadConfig
from
paddle.fluid.dygraph.io
import
_construct_program_holders
,
_construct_params_and_buffers
,
EXTRA_VAR_INFO_FILENAME
__all__
=
[
...
...
@@ -33,35 +33,27 @@ __all__ = [
]
# NOTE(chenweihang): deprecate load_dygraph's argument keep_name_table,
# ensure compatibility when user still use keep_name_table argument
def
deprecate_keep_name_table
(
func
):
@
functools
.
wraps
(
func
)
def
wrapper
(
*
args
,
**
kwargs
):
def
__warn_and_build_configs__
(
keep_name_table
):
warnings
.
warn
(
"The argument `keep_name_table` has deprecated, please use `SaveLoadConfig.keep_name_table`."
,
DeprecationWarning
)
config
=
SaveLoadConfig
()
config
.
keep_name_table
=
keep_name_table
return
config
# deal with arg `keep_name_table`
if
len
(
args
)
>
1
and
isinstance
(
args
[
1
],
bool
):
args
=
list
(
args
)
args
[
1
]
=
__warn_and_build_configs__
(
args
[
1
])
# deal with kwargs
elif
'keep_name_table'
in
kwargs
:
kwargs
[
'config'
]
=
__warn_and_build_configs__
(
kwargs
[
'keep_name_table'
])
kwargs
.
pop
(
'keep_name_table'
)
else
:
# do nothing
pass
def
_parse_load_config
(
configs
):
supported_configs
=
[
'model_filename'
,
'params_filename'
,
'separate_params'
,
'keep_name_table'
]
# input check
for
key
in
configs
:
if
key
not
in
supported_configs
:
raise
ValueError
(
"The additional config (%s) of `paddle.fluid.load_dygraph` is not supported."
%
(
key
))
return
func
(
*
args
,
**
kwargs
)
# construct inner config
inner_config
=
_SaveLoadConfig
()
inner_config
.
model_filename
=
configs
.
get
(
'model_filename'
,
None
)
inner_config
.
params_filename
=
configs
.
get
(
'params_filename'
,
None
)
inner_config
.
separate_params
=
configs
.
get
(
'separate_params'
,
None
)
inner_config
.
keep_name_table
=
configs
.
get
(
'keep_name_table'
,
None
)
return
wrapper
return
inner_config
@
dygraph_only
...
...
@@ -135,9 +127,7 @@ def save_dygraph(state_dict, model_path):
# TODO(qingqing01): remove dygraph_only to support loading static model.
# maybe need to unify the loading interface after 2.0 API is ready.
# @dygraph_only
@
deprecate_save_load_configs
@
deprecate_keep_name_table
def
load_dygraph
(
model_path
,
config
=
None
):
def
load_dygraph
(
model_path
,
**
configs
):
'''
:api_attr: imperative
...
...
@@ -152,10 +142,20 @@ def load_dygraph(model_path, config=None):
Args:
model_path(str) : The file prefix store the state_dict.
(The path should Not contain suffix '.pdparams')
config (SaveLoadConfig, optional): :ref:`api_imperative_jit_saveLoadConfig`
object that specifies additional configuration options, these options
are for compatibility with ``jit.save/io.save_inference_model`` formats.
Default None.
configs (dict, optional): other save configuration options for compatibility. We do not
recommend using these configurations, if not necessary, DO NOT use them. Default None.
The following options are currently supported:
(1) model_filename (string): The filename to load the translated program of target Layer.
Default filename is :code:`__model__` .
(2) params_filename (string): The filename to load all persistable variables in target Layer.
Default file name is :code:`__variables__` .
(3) separate_params (bool): Configure whether to load the Layer parameters from separete files.
If True, each parameter will be loaded from a file separately, the file name is the parameter name,
and the params_filename configuration will not take effect. Default False.
(4) keep_name_table (bool): Configures whether keep ``structured_name -> parameter_name`` dict in
loaded state dict. This dict is the debugging information saved when call ``paddle.fluid.save_dygraph`` .
It is generally only used for debugging and does not affect the actual training or inference.
By default, it will not be retained in ``paddle.fluid.load_dygraph`` result. Default: False.
Returns:
state_dict(dict) : the dict store the state_dict
...
...
@@ -196,8 +196,7 @@ def load_dygraph(model_path, config=None):
opti_file_path
=
model_prefix
+
".pdopt"
# deal with argument `config`
if
config
is
None
:
config
=
SaveLoadConfig
()
config
=
_parse_load_config
(
configs
)
if
os
.
path
.
exists
(
params_file_path
)
or
os
.
path
.
exists
(
opti_file_path
):
# Load state dict by `save_dygraph` save format
...
...
python/paddle/fluid/dygraph/jit.py
浏览文件 @
24a33bed
此差异已折叠。
点击以展开。
python/paddle/fluid/dygraph/static_runner.py
浏览文件 @
24a33bed
...
...
@@ -14,7 +14,7 @@
from
__future__
import
print_function
from
paddle.fluid.dygraph.jit
import
SaveLoadConfig
from
paddle.fluid.dygraph.jit
import
_
SaveLoadConfig
from
paddle.fluid.dygraph.io
import
TranslatedLayer
...
...
@@ -31,7 +31,7 @@ class StaticModelRunner(object):
"""
def
__new__
(
cls
,
model_dir
,
model_filename
=
None
,
params_filename
=
None
):
configs
=
SaveLoadConfig
()
configs
=
_
SaveLoadConfig
()
if
model_filename
is
not
None
:
configs
.
model_filename
=
model_filename
if
params_filename
is
not
None
:
...
...
python/paddle/fluid/tests/unittests/dygraph_to_static/test_lac.py
浏览文件 @
24a33bed
...
...
@@ -498,13 +498,11 @@ def do_train(args, to_static):
step
+=
1
# save inference model
if
to_static
:
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
configs
.
output_spec
=
[
crf_decode
]
fluid
.
dygraph
.
jit
.
save
(
layer
=
model
,
model_path
=
args
.
model_save_dir
,
input_spec
=
[
words
,
length
],
configs
=
configs
)
output_spec
=
[
crf_decode
]
)
else
:
fluid
.
dygraph
.
save_dygraph
(
model
.
state_dict
(),
args
.
dy_param_path
)
...
...
python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py
浏览文件 @
24a33bed
...
...
@@ -218,13 +218,11 @@ class TestMNISTWithToStatic(TestMNIST):
def
check_jit_save_load
(
self
,
model
,
inputs
,
input_spec
,
to_static
,
gt_out
):
if
to_static
:
infer_model_path
=
"./test_mnist_inference_model_by_jit_save"
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
configs
.
output_spec
=
[
gt_out
]
fluid
.
dygraph
.
jit
.
save
(
layer
=
model
,
model_path
=
infer_model_path
,
input_spec
=
input_spec
,
configs
=
configs
)
output_spec
=
[
gt_out
]
)
# load in static mode
static_infer_out
=
self
.
jit_load_and_run_inference_static
(
infer_model_path
,
inputs
)
...
...
python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py
浏览文件 @
24a33bed
...
...
@@ -67,13 +67,11 @@ class TestDyToStaticSaveInferenceModel(unittest.TestCase):
layer
.
clear_gradients
()
# test for saving model in dygraph.guard
infer_model_dir
=
"./test_dy2stat_save_inference_model_in_guard"
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
configs
.
output_spec
=
[
pred
]
fluid
.
dygraph
.
jit
.
save
(
layer
=
layer
,
model_path
=
infer_model_dir
,
input_spec
=
[
x
],
configs
=
configs
)
output_spec
=
[
pred
]
)
# Check the correctness of the inference
dygraph_out
,
_
=
layer
(
x
)
self
.
check_save_inference_model
(
layer
,
[
x_data
],
dygraph_out
.
numpy
())
...
...
@@ -92,15 +90,12 @@ class TestDyToStaticSaveInferenceModel(unittest.TestCase):
expected_persistable_vars
=
set
([
p
.
name
for
p
in
model
.
parameters
()])
infer_model_dir
=
"./test_dy2stat_save_inference_model"
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
if
fetch
is
not
None
:
configs
.
output_spec
=
fetch
configs
.
separate_params
=
True
fluid
.
dygraph
.
jit
.
save
(
layer
=
model
,
model_path
=
infer_model_dir
,
input_spec
=
feed
if
feed
else
None
,
configs
=
configs
)
separate_params
=
True
,
output_spec
=
fetch
if
fetch
else
None
)
saved_var_names
=
set
([
filename
for
filename
in
os
.
listdir
(
infer_model_dir
)
if
filename
!=
'__model__'
and
filename
!=
EXTRA_VAR_INFO_FILENAME
...
...
python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py
浏览文件 @
24a33bed
...
...
@@ -383,10 +383,10 @@ def train(train_reader, to_static):
step_idx
+=
1
if
step_idx
==
STEP_NUM
:
if
to_static
:
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
configs
.
output_spec
=
[
pred
]
fluid
.
dygraph
.
jit
.
save
(
se_resnext
,
MODEL_SAVE_PATH
,
[
img
],
configs
)
fluid
.
dygraph
.
jit
.
save
(
se_resnext
,
MODEL_SAVE_PATH
,
[
img
]
,
output_spec
=
[
pred
]
)
else
:
fluid
.
dygraph
.
save_dygraph
(
se_resnext
.
state_dict
(),
DY_STATE_DICT_SAVE_PATH
)
...
...
python/paddle/fluid/tests/unittests/test_directory_migration.py
浏览文件 @
24a33bed
...
...
@@ -43,15 +43,14 @@ class TestDirectory(unittest.TestCase):
'paddle.distributed.prepare_context'
,
'paddle.DataParallel'
,
'paddle.jit'
,
'paddle.jit.TracedLayer'
,
'paddle.jit.to_static'
,
'paddle.jit.ProgramTranslator'
,
'paddle.jit.TranslatedLayer'
,
'paddle.jit.save'
,
'paddle.jit.load'
,
'paddle.SaveLoadConfig'
,
'paddle.NoamDecay'
,
'paddle.PiecewiseDecay'
,
'paddle.NaturalExpDecay'
,
'paddle.ExponentialDecay'
,
'paddle.InverseTimeDecay'
,
'paddle.PolynomialDecay'
,
'paddle.CosineDecay'
,
'paddle.static.Executor'
,
'paddle.static.global_scope'
,
'paddle.static.scope_guard'
,
'paddle.static.append_backward'
,
'paddle.static.gradients'
,
'paddle.static.BuildStrategy'
,
'paddle.static.CompiledProgram'
,
'paddle.static.ExecutionStrategy'
,
'paddle.jit.save'
,
'paddle.jit.load'
,
'paddle.NoamDecay'
,
'paddle.PiecewiseDecay'
,
'paddle.NaturalExpDecay'
,
'paddle.ExponentialDecay'
,
'paddle.InverseTimeDecay'
,
'paddle.PolynomialDecay'
,
'paddle.CosineDecay'
,
'paddle.static.Executor'
,
'paddle.static.global_scope'
,
'paddle.static.scope_guard'
,
'paddle.static.append_backward'
,
'paddle.static.gradients'
,
'paddle.static.BuildStrategy'
,
'paddle.static.CompiledProgram'
,
'paddle.static.ExecutionStrategy'
,
'paddle.static.default_main_program'
,
'paddle.static.default_startup_program'
,
'paddle.static.Program'
,
'paddle.static.name_scope'
,
'paddle.static.program_guard'
,
...
...
@@ -104,9 +103,7 @@ class TestDirectory(unittest.TestCase):
'paddle.imperative.TracedLayer'
,
'paddle.imperative.declarative'
,
'paddle.imperative.ProgramTranslator'
,
'paddle.imperative.TranslatedLayer'
,
'paddle.imperative.jit.save'
,
'paddle.imperative.jit.load'
,
'paddle.imperative.jit.SaveLoadConfig'
,
'paddle.imperative.NoamDecay'
'paddle.imperative.jit.load'
,
'paddle.imperative.NoamDecay'
'paddle.imperative.PiecewiseDecay'
,
'paddle.imperative.NaturalExpDecay'
,
'paddle.imperative.ExponentialDecay'
,
...
...
python/paddle/fluid/tests/unittests/test_jit_save_load.py
浏览文件 @
24a33bed
...
...
@@ -225,16 +225,13 @@ class TestJitSaveLoad(unittest.TestCase):
paddle
.
manual_seed
(
SEED
)
paddle
.
framework
.
random
.
_manual_program_seed
(
SEED
)
def
train_and_save_model
(
self
,
model_path
=
None
,
configs
=
None
):
def
train_and_save_model
(
self
,
model_path
=
None
):
layer
=
LinearNet
(
784
,
1
)
example_inputs
,
layer
,
_
=
train
(
layer
)
final_model_path
=
model_path
if
model_path
else
self
.
model_path
orig_input_types
=
[
type
(
x
)
for
x
in
example_inputs
]
fluid
.
dygraph
.
jit
.
save
(
layer
=
layer
,
model_path
=
final_model_path
,
input_spec
=
example_inputs
,
configs
=
configs
)
layer
=
layer
,
model_path
=
final_model_path
,
input_spec
=
example_inputs
)
new_input_types
=
[
type
(
x
)
for
x
in
example_inputs
]
self
.
assertEqual
(
orig_input_types
,
new_input_types
)
return
layer
...
...
@@ -314,7 +311,6 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
[
None
,
8
],
name
=
'x'
)])
model_path
=
"model.input_spec.output_spec"
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
# check inputs and outputs
self
.
assertTrue
(
len
(
net
.
forward
.
inputs
)
==
1
)
input_x
=
net
.
forward
.
inputs
[
0
]
...
...
@@ -322,11 +318,11 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
self
.
assertTrue
(
input_x
.
name
==
'x'
)
# 1. prune loss
configs
.
output_spec
=
net
.
forward
.
outputs
[:
1
]
fluid
.
dygraph
.
jit
.
save
(
net
,
model_path
,
configs
=
configs
)
output_spec
=
net
.
forward
.
outputs
[:
1
]
fluid
.
dygraph
.
jit
.
save
(
net
,
model_path
,
output_spec
=
output_spec
)
# 2. load to infer
infer_layer
=
fluid
.
dygraph
.
jit
.
load
(
model_path
,
configs
=
configs
)
infer_layer
=
fluid
.
dygraph
.
jit
.
load
(
model_path
)
x
=
fluid
.
dygraph
.
to_variable
(
np
.
random
.
random
((
4
,
8
)).
astype
(
'float32'
))
pred
=
infer_layer
(
x
)
...
...
@@ -335,7 +331,6 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
net
=
LinearNetMultiInput
(
8
,
8
)
model_path
=
"model.multi_inout.output_spec1"
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
# 1. check inputs and outputs
self
.
assertTrue
(
len
(
net
.
forward
.
inputs
)
==
2
)
input_x
=
net
.
forward
.
inputs
[
0
]
...
...
@@ -344,11 +339,11 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
self
.
assertTrue
(
input_y
.
shape
==
(
-
1
,
8
))
# 2. prune loss
configs
.
output_spec
=
net
.
forward
.
outputs
[:
2
]
fluid
.
dygraph
.
jit
.
save
(
net
,
model_path
,
configs
=
configs
)
output_spec
=
net
.
forward
.
outputs
[:
2
]
fluid
.
dygraph
.
jit
.
save
(
net
,
model_path
,
output_spec
=
output_spec
)
# 3. load to infer
infer_layer
=
fluid
.
dygraph
.
jit
.
load
(
model_path
,
configs
=
configs
)
infer_layer
=
fluid
.
dygraph
.
jit
.
load
(
model_path
)
x
=
fluid
.
dygraph
.
to_variable
(
np
.
random
.
random
((
4
,
8
)).
astype
(
'float32'
))
y
=
fluid
.
dygraph
.
to_variable
(
...
...
@@ -358,10 +353,11 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
# 1. prune y and loss
model_path
=
"model.multi_inout.output_spec2"
configs
.
output_spec
=
net
.
forward
.
outputs
[:
1
]
fluid
.
dygraph
.
jit
.
save
(
net
,
model_path
,
[
input_x
],
configs
)
output_spec
=
net
.
forward
.
outputs
[:
1
]
fluid
.
dygraph
.
jit
.
save
(
net
,
model_path
,
[
input_x
],
output_spec
=
output_spec
)
# 2. load again
infer_layer2
=
fluid
.
dygraph
.
jit
.
load
(
model_path
,
configs
=
configs
)
infer_layer2
=
fluid
.
dygraph
.
jit
.
load
(
model_path
)
# 3. predict
pred_xx
=
infer_layer2
(
x
)
...
...
@@ -377,16 +373,16 @@ class TestJitSaveLoadConfig(unittest.TestCase):
paddle
.
manual_seed
(
SEED
)
paddle
.
framework
.
random
.
_manual_program_seed
(
SEED
)
def
basic_save_load
(
self
,
layer
,
model_path
,
configs
):
def
basic_save_load
(
self
,
layer
,
model_path
,
**
configs
):
# 1. train & save
example_inputs
,
train_layer
,
_
=
train
(
layer
)
fluid
.
dygraph
.
jit
.
save
(
layer
=
train_layer
,
model_path
=
model_path
,
input_spec
=
example_inputs
,
configs
=
configs
)
**
configs
)
# 2. load
infer_layer
=
fluid
.
dygraph
.
jit
.
load
(
model_path
,
configs
=
configs
)
infer_layer
=
fluid
.
dygraph
.
jit
.
load
(
model_path
,
**
configs
)
train_layer
.
eval
()
# 3. inference & compare
x
=
fluid
.
dygraph
.
to_variable
(
...
...
@@ -397,23 +393,18 @@ class TestJitSaveLoadConfig(unittest.TestCase):
def
test_model_filename
(
self
):
layer
=
LinearNet
(
784
,
1
)
model_path
=
"model.save_load_config.output_spec"
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
configs
.
model_filename
=
"__simplenet__"
self
.
basic_save_load
(
layer
,
model_path
,
configs
)
self
.
basic_save_load
(
layer
,
model_path
,
model_filename
=
"__simplenet__"
)
def
test_params_filename
(
self
):
layer
=
LinearNet
(
784
,
1
)
model_path
=
"model.save_load_config.params_filename"
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
configs
.
params_filename
=
"__params__"
self
.
basic_save_load
(
layer
,
model_path
,
configs
)
self
.
basic_save_load
(
layer
,
model_path
,
params_filename
=
"__params__"
)
def
test_separate_params
(
self
):
layer
=
LinearNet
(
784
,
1
)
model_path
=
"model.save_load_config.separate_params"
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
configs
.
separate_params
=
True
self
.
basic_save_load
(
layer
,
model_path
,
configs
)
self
.
basic_save_load
(
layer
,
model_path
,
separate_params
=
True
)
def
test_output_spec
(
self
):
train_layer
=
LinearNetReturnLoss
(
8
,
8
)
...
...
@@ -428,16 +419,15 @@ class TestJitSaveLoadConfig(unittest.TestCase):
train_layer
.
clear_gradients
()
model_path
=
"model.save_load_config.output_spec"
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
configs
.
output_spec
=
[
out
]
output_spec
=
[
out
]
fluid
.
dygraph
.
jit
.
save
(
layer
=
train_layer
,
model_path
=
model_path
,
input_spec
=
[
x
],
configs
=
configs
)
output_spec
=
output_spec
)
train_layer
.
eval
()
infer_layer
=
fluid
.
dygraph
.
jit
.
load
(
model_path
,
configs
=
configs
)
infer_layer
=
fluid
.
dygraph
.
jit
.
load
(
model_path
)
x
=
fluid
.
dygraph
.
to_variable
(
np
.
random
.
random
((
4
,
8
)).
astype
(
'float32'
))
self
.
assertTrue
(
...
...
@@ -494,13 +484,12 @@ class TestJitPruneModelAndLoad(unittest.TestCase):
adam
.
minimize
(
loss
)
train_layer
.
clear_gradients
()
configs
=
fluid
.
dygraph
.
jit
.
SaveLoadConfig
()
configs
.
output_spec
=
[
hidden
]
output_spec
=
[
hidden
]
fluid
.
dygraph
.
jit
.
save
(
layer
=
train_layer
,
model_path
=
self
.
model_path
,
input_spec
=
[
x
],
configs
=
configs
)
output_spec
=
output_spec
)
return
train_layer
...
...
@@ -617,8 +606,6 @@ class TestJitSaveMultiCases(unittest.TestCase):
out
=
train_with_label
(
layer
)
model_path
=
"test_prune_to_static_after_train"
configs
=
paddle
.
SaveLoadConfig
()
configs
.
output_spec
=
[
out
]
paddle
.
jit
.
save
(
layer
,
model_path
,
...
...
@@ -626,7 +613,7 @@ class TestJitSaveMultiCases(unittest.TestCase):
InputSpec
(
shape
=
[
None
,
784
],
dtype
=
'float32'
,
name
=
"image"
)
],
configs
=
configs
)
output_spec
=
[
out
]
)
self
.
verify_inference_correctness
(
layer
,
model_path
,
True
)
...
...
@@ -634,10 +621,9 @@ class TestJitSaveMultiCases(unittest.TestCase):
layer
=
LinerNetWithLabel
(
784
,
1
)
model_path
=
"test_prune_to_static_no_train"
configs
=
paddle
.
SaveLoadConfig
()
# TODO: no train, cannot get output_spec var here
# now only can use index
configs
.
output_spec
=
layer
.
forward
.
outputs
[:
1
]
output_spec
=
layer
.
forward
.
outputs
[:
1
]
paddle
.
jit
.
save
(
layer
,
model_path
,
...
...
@@ -645,7 +631,7 @@ class TestJitSaveMultiCases(unittest.TestCase):
InputSpec
(
shape
=
[
None
,
784
],
dtype
=
'float32'
,
name
=
"image"
)
],
configs
=
configs
)
output_spec
=
output_spec
)
self
.
verify_inference_correctness
(
layer
,
model_path
,
True
)
...
...
@@ -676,10 +662,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train
(
layer
)
model_path
=
"test_not_prune_output_spec_name_warning"
configs
=
paddle
.
SaveLoadConfig
()
out
=
paddle
.
to_tensor
(
np
.
random
.
random
((
1
,
1
)).
astype
(
'float'
))
configs
.
output_spec
=
[
out
]
paddle
.
jit
.
save
(
layer
,
model_path
,
configs
=
configs
)
paddle
.
jit
.
save
(
layer
,
model_path
,
output_spec
=
[
out
])
self
.
verify_inference_correctness
(
layer
,
model_path
)
...
...
@@ -708,9 +692,7 @@ class TestJitSaveMultiCases(unittest.TestCase):
train_with_label
(
layer
)
model_path
=
"test_prune_to_static_after_train"
configs
=
paddle
.
SaveLoadConfig
()
out
=
paddle
.
to_tensor
(
np
.
random
.
random
((
1
,
1
)).
astype
(
'float'
))
configs
.
output_spec
=
[
out
]
with
self
.
assertRaises
(
ValueError
):
paddle
.
jit
.
save
(
layer
,
...
...
@@ -719,7 +701,7 @@ class TestJitSaveMultiCases(unittest.TestCase):
InputSpec
(
shape
=
[
None
,
784
],
dtype
=
'float32'
,
name
=
"image"
)
],
configs
=
configs
)
output_spec
=
[
out
]
)
class
TestJitSaveLoadEmptyLayer
(
unittest
.
TestCase
):
...
...
python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py
浏览文件 @
24a33bed
...
...
@@ -63,6 +63,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self
.
epoch_num
=
1
self
.
batch_size
=
128
self
.
batch_num
=
10
# enable static mode
paddle
.
enable_static
()
def
train_and_save_model
(
self
,
only_params
=
False
):
with
new_program_scope
():
...
...
@@ -136,13 +138,12 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self
.
params_filename
=
None
orig_param_dict
=
self
.
train_and_save_model
()
config
=
paddle
.
SaveLoadConfig
()
config
.
separate_params
=
True
config
.
model_filename
=
self
.
model_filename
load_param_dict
,
_
=
fluid
.
load_dygraph
(
self
.
save_dirname
,
config
)
load_param_dict
,
_
=
fluid
.
load_dygraph
(
self
.
save_dirname
,
model_filename
=
self
.
model_filename
)
self
.
check_load_state_dict
(
orig_param_dict
,
load_param_dict
)
new_load_param_dict
=
paddle
.
load
(
self
.
save_dirname
,
config
)
new_load_param_dict
=
paddle
.
load
(
self
.
save_dirname
,
model_filename
=
self
.
model_filename
)
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
def
test_load_with_param_filename
(
self
):
...
...
@@ -151,12 +152,12 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self
.
params_filename
=
"static_mnist.params"
orig_param_dict
=
self
.
train_and_save_model
()
config
=
paddle
.
SaveLoadConfig
()
config
.
params_filename
=
self
.
params_filename
load_param_dict
,
_
=
fluid
.
load_dygraph
(
self
.
save_dirname
,
config
)
load_param_dict
,
_
=
fluid
.
load_dygraph
(
self
.
save_dirname
,
params_filename
=
self
.
params_filename
)
self
.
check_load_state_dict
(
orig_param_dict
,
load_param_dict
)
new_load_param_dict
=
paddle
.
load
(
self
.
save_dirname
,
config
)
new_load_param_dict
=
paddle
.
load
(
self
.
save_dirname
,
params_filename
=
self
.
params_filename
)
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
def
test_load_with_model_and_param_filename
(
self
):
...
...
@@ -165,13 +166,16 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self
.
params_filename
=
"static_mnist.params"
orig_param_dict
=
self
.
train_and_save_model
()
config
=
paddle
.
SaveLoadConfig
()
config
.
params_filename
=
self
.
params_filename
config
.
model_filename
=
self
.
model_filename
load_param_dict
,
_
=
fluid
.
load_dygraph
(
self
.
save_dirname
,
config
)
load_param_dict
,
_
=
fluid
.
load_dygraph
(
self
.
save_dirname
,
params_filename
=
self
.
params_filename
,
model_filename
=
self
.
model_filename
)
self
.
check_load_state_dict
(
orig_param_dict
,
load_param_dict
)
new_load_param_dict
=
paddle
.
load
(
self
.
save_dirname
,
config
)
new_load_param_dict
=
paddle
.
load
(
self
.
save_dirname
,
params_filename
=
self
.
params_filename
,
model_filename
=
self
.
model_filename
)
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
def
test_load_state_dict_from_save_params
(
self
):
...
...
python/paddle/framework/__init__.py
浏览文件 @
24a33bed
...
...
@@ -20,8 +20,8 @@ __all__ = [
]
__all__
+=
[
'grad'
,
'LayerList'
,
'load'
,
'save'
,
'
SaveLoadConfig'
,
'to_variable
'
,
'
no_grad'
,
'
DataParallel'
'grad'
,
'LayerList'
,
'load'
,
'save'
,
'
to_variable'
,
'no_grad
'
,
'DataParallel'
]
__all__
+=
[
...
...
@@ -50,7 +50,6 @@ from ..fluid.dygraph.base import to_variable #DEFINE_ALIAS
from
..fluid.dygraph.base
import
grad
#DEFINE_ALIAS
from
.io
import
save
from
.io
import
load
from
..fluid.dygraph.jit
import
SaveLoadConfig
#DEFINE_ALIAS
from
..fluid.dygraph.parallel
import
DataParallel
#DEFINE_ALIAS
from
..fluid.dygraph.learning_rate_scheduler
import
NoamDecay
#DEFINE_ALIAS
...
...
python/paddle/framework/io.py
浏览文件 @
24a33bed
...
...
@@ -26,6 +26,7 @@ import paddle
from
paddle
import
fluid
from
paddle.fluid
import
core
from
paddle.fluid.framework
import
Variable
,
_varbase_creator
,
_dygraph_tracer
from
paddle.fluid.dygraph.jit
import
_SaveLoadConfig
from
paddle.fluid.dygraph.io
import
_construct_program_holders
,
_construct_params_and_buffers
,
EXTRA_VAR_INFO_FILENAME
__all__
=
[
...
...
@@ -116,6 +117,29 @@ def _load_state_dict_from_save_params(model_path):
return
load_param_dict
def
_parse_load_config
(
configs
):
supported_configs
=
[
'model_filename'
,
'params_filename'
,
'separate_params'
,
'keep_name_table'
]
# input check
for
key
in
configs
:
if
key
not
in
supported_configs
:
raise
ValueError
(
"The additional config (%s) of `paddle.load` is not supported."
%
key
)
# construct inner config
inner_config
=
_SaveLoadConfig
()
inner_config
.
model_filename
=
configs
.
get
(
'model_filename'
,
None
)
inner_config
.
params_filename
=
configs
.
get
(
'params_filename'
,
None
)
inner_config
.
separate_params
=
configs
.
get
(
'separate_params'
,
None
)
inner_config
.
keep_name_table
=
configs
.
get
(
'keep_name_table'
,
None
)
return
inner_config
def
save
(
obj
,
path
):
'''
Save an object to the specified path.
...
...
@@ -178,7 +202,7 @@ def save(obj, path):
pickle
.
dump
(
saved_obj
,
f
,
protocol
=
2
)
def
load
(
path
,
config
=
None
):
def
load
(
path
,
**
configs
):
'''
Load an object can be used in paddle from specified path.
...
...
@@ -197,10 +221,20 @@ def load(path, config=None):
path(str) : The path to load the target object. Generally, the path is the target
file path, when compatible with loading the saved results of
``paddle.jit.save/paddle.static.save_inference_model`` , the path is a directory.
config (SaveLoadConfig, optional): :ref:`api_imperative_jit_saveLoadConfig`
object that specifies additional configuration options, these options
are for compatibility with ``paddle.jit.save/paddle.static.save_inference_model``
formats. Default None.
configs (dict, optional): other save configuration options for compatibility. We do not
recommend using these configurations, if not necessary, DO NOT use them. Default None.
The following options are currently supported:
(1) model_filename (string): The filename to load the translated program of target Layer.
Default filename is :code:`__model__` .
(2) params_filename (string): The filename to load all persistable variables in target Layer.
Default file name is :code:`__variables__` .
(3) separate_params (bool): Configure whether to load the Layer parameters from separete files.
If True, each parameter will be loaded from a file separately, the file name is the parameter name,
and the params_filename configuration will not take effect. Default False.
(4) keep_name_table (bool): Configures whether keep ``structured_name -> parameter_name`` dict in
loaded state dict. This dict is the debugging information saved when call ``paddle.save`` .
It is generally only used for debugging and does not affect the actual training or inference.
By default, it will not be retained in ``paddle.load`` result. Default: False.
Returns:
Object(Object): a target object can be used in paddle
...
...
@@ -242,8 +276,7 @@ def load(path, config=None):
"`paddle.load('model')`."
raise
ValueError
(
error_msg
%
path
)
if
config
is
None
:
config
=
paddle
.
SaveLoadConfig
()
config
=
_parse_load_config
(
configs
)
# 2. load target
load_result
=
None
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录