Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
Paddle
提交
2eb60ddb
P
Paddle
项目概览
BaiXuePrincess
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
2eb60ddb
编写于
6月 17, 2022
作者:
Y
YuanRisheng
提交者:
GitHub
6月 17, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
cherry pick 43581 (#43596)
上级
3689a126
变更
12
展开全部
隐藏空白更改
内联
并排
Showing
12 changed file
with
501 addition
and
164 deletion
+501
-164
python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py
.../paddle/fluid/tests/unittests/test_faster_tokenizer_op.py
+7
-1
python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py
...luid/tests/unittests/test_imperative_load_static_param.py
+8
-2
python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py
...dle/fluid/tests/unittests/test_imperative_save_load_v2.py
+31
-12
python/paddle/fluid/tests/unittests/test_io_save_load.py
python/paddle/fluid/tests/unittests/test_io_save_load.py
+26
-4
python/paddle/fluid/tests/unittests/test_jit_save_load.py
python/paddle/fluid/tests/unittests/test_jit_save_load.py
+185
-51
python/paddle/fluid/tests/unittests/test_load_op.py
python/paddle/fluid/tests/unittests/test_load_op.py
+11
-2
python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py
...d/tests/unittests/test_load_state_dict_from_old_format.py
+16
-5
python/paddle/fluid/tests/unittests/test_paddle_save_load.py
python/paddle/fluid/tests/unittests/test_paddle_save_load.py
+91
-28
python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py
...dle/fluid/tests/unittests/test_paddle_save_load_binary.py
+31
-10
python/paddle/fluid/tests/unittests/test_static_save_load.py
python/paddle/fluid/tests/unittests/test_static_save_load.py
+77
-43
python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py
...addle/fluid/tests/unittests/test_static_save_load_bf16.py
+12
-3
python/paddle/fluid/tests/unittests/test_static_save_load_large.py
...ddle/fluid/tests/unittests/test_static_save_load_large.py
+6
-3
未找到文件。
python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py
浏览文件 @
2eb60ddb
...
@@ -27,6 +27,8 @@ from paddle.fluid.layer_helper import LayerHelper
...
@@ -27,6 +27,8 @@ from paddle.fluid.layer_helper import LayerHelper
from
paddle
import
_C_ops
from
paddle
import
_C_ops
import
sys
import
sys
import
tempfile
sys
.
path
.
append
(
"./tokenizer"
)
sys
.
path
.
append
(
"./tokenizer"
)
from
tokenizer.bert_tokenizer
import
BertTokenizer
from
tokenizer.bert_tokenizer
import
BertTokenizer
...
@@ -149,13 +151,17 @@ class Predictor(object):
...
@@ -149,13 +151,17 @@ class Predictor(object):
class
TestBertTokenizerOp
(
unittest
.
TestCase
):
class
TestBertTokenizerOp
(
unittest
.
TestCase
):
def
setUp
(
self
):
def
setUp
(
self
):
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
self
.
bert_tokenizer
=
BertTokenizer
.
from_pretrained
(
"bert-base-chinese"
)
self
.
bert_tokenizer
=
BertTokenizer
.
from_pretrained
(
"bert-base-chinese"
)
self
.
faster_tokenizer
=
FasterTokenizer
(
self
.
bert_tokenizer
.
vocab
)
self
.
faster_tokenizer
=
FasterTokenizer
(
self
.
bert_tokenizer
.
vocab
)
self
.
init_data
()
self
.
init_data
()
self
.
save_path
=
os
.
path
.
join
(
DATA_HOME
,
"fast_tokenizer"
)
self
.
save_path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"fast_tokenizer"
)
self
.
param_path
=
os
.
path
.
join
(
self
.
save_path
,
"model.pdparams"
)
self
.
param_path
=
os
.
path
.
join
(
self
.
save_path
,
"model.pdparams"
)
self
.
inference_path
=
os
.
path
.
join
(
self
.
save_path
,
"inference"
)
self
.
inference_path
=
os
.
path
.
join
(
self
.
save_path
,
"inference"
)
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
init_data
(
self
):
def
init_data
(
self
):
self
.
text
=
[
self
.
text
=
[
'选择珠江花园的原因就是方便,有电动扶梯直接到达海边,周围餐馆、食廊、商场、超市、摊位一应俱全。'
'选择珠江花园的原因就是方便,有电动扶梯直接到达海边,周围餐馆、食廊、商场、超市、摊位一应俱全。'
...
...
python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py
浏览文件 @
2eb60ddb
...
@@ -17,11 +17,14 @@ import paddle.fluid as fluid
...
@@ -17,11 +17,14 @@ import paddle.fluid as fluid
import
paddle.fluid.framework
as
framework
import
paddle.fluid.framework
as
framework
from
paddle.fluid.dygraph.nn
import
*
from
paddle.fluid.dygraph.nn
import
*
import
numpy
as
np
import
numpy
as
np
import
os
import
tempfile
class
TestDygraphLoadStatic
(
unittest
.
TestCase
):
class
TestDygraphLoadStatic
(
unittest
.
TestCase
):
def
testLoadStaticModel
(
self
):
def
testLoadStaticModel
(
self
):
# static mode
# static mode
temp_dir
=
tempfile
.
TemporaryDirectory
()
a
=
fluid
.
data
(
name
=
"a"
,
shape
=
[
10
,
10
])
a
=
fluid
.
data
(
name
=
"a"
,
shape
=
[
10
,
10
])
conv_in
=
fluid
.
data
(
name
=
"conv_in"
,
shape
=
[
None
,
10
,
10
,
10
])
conv_in
=
fluid
.
data
(
name
=
"conv_in"
,
shape
=
[
None
,
10
,
10
,
10
])
...
@@ -121,9 +124,11 @@ class TestDygraphLoadStatic(unittest.TestCase):
...
@@ -121,9 +124,11 @@ class TestDygraphLoadStatic(unittest.TestCase):
)
if
not
fluid
.
is_compiled_with_cuda
()
else
fluid
.
CUDAPlace
(
0
))
)
if
not
fluid
.
is_compiled_with_cuda
()
else
fluid
.
CUDAPlace
(
0
))
out
=
exe
.
run
(
framework
.
default_startup_program
())
out
=
exe
.
run
(
framework
.
default_startup_program
())
fluid
.
save
(
framework
.
default_main_program
(),
"./test_1"
)
fluid
.
save
(
framework
.
default_main_program
(),
os
.
path
.
join
(
temp_dir
.
name
,
"test_1"
))
para_dict
=
fluid
.
load_program_state
(
"./test_1"
)
para_dict
=
fluid
.
load_program_state
(
os
.
path
.
join
(
temp_dir
.
name
,
"test_1"
))
new_dict
=
{}
new_dict
=
{}
for
k
,
v
in
para_dict
.
items
():
for
k
,
v
in
para_dict
.
items
():
...
@@ -192,6 +197,7 @@ class TestDygraphLoadStatic(unittest.TestCase):
...
@@ -192,6 +197,7 @@ class TestDygraphLoadStatic(unittest.TestCase):
my_test
.
set_dict
(
new_dict
,
use_structured_name
=
False
)
my_test
.
set_dict
(
new_dict
,
use_structured_name
=
False
)
for
k
,
v
in
my_test
.
state_dict
().
items
():
for
k
,
v
in
my_test
.
state_dict
().
items
():
self
.
assertTrue
(
np
.
array_equal
(
v
.
numpy
(),
new_dict
[
v
.
name
]))
self
.
assertTrue
(
np
.
array_equal
(
v
.
numpy
(),
new_dict
[
v
.
name
]))
temp_dir
.
cleanup
()
if
__name__
==
'__main__'
:
if
__name__
==
'__main__'
:
...
...
python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py
浏览文件 @
2eb60ddb
...
@@ -29,6 +29,8 @@ import six
...
@@ -29,6 +29,8 @@ import six
import
paddle
import
paddle
from
paddle.fluid.framework
import
_test_eager_guard
from
paddle.fluid.framework
import
_test_eager_guard
import
tempfile
class
SimpleLSTMRNN
(
fluid
.
Layer
):
class
SimpleLSTMRNN
(
fluid
.
Layer
):
def
__init__
(
self
,
def
__init__
(
self
,
...
@@ -209,6 +211,12 @@ class PtbModel(fluid.Layer):
...
@@ -209,6 +211,12 @@ class PtbModel(fluid.Layer):
class
TestDygraphPtbRnn
(
unittest
.
TestCase
):
class
TestDygraphPtbRnn
(
unittest
.
TestCase
):
def
setUp
(
self
):
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
func_setUp
(
self
):
def
func_setUp
(
self
):
seed
=
90
seed
=
90
hidden_size
=
10
hidden_size
=
10
...
@@ -286,7 +294,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
...
@@ -286,7 +294,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
else
:
else
:
self
.
base_opti
[
k
]
=
v
self
.
base_opti
[
k
]
=
v
paddle
.
save
(
self
.
opti_dict
,
"./test_dy_v2.pdopt"
)
paddle
.
save
(
self
.
opti_dict
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_dy_v2.pdopt"
))
self
.
state_dict
=
ptb_model
.
state_dict
()
self
.
state_dict
=
ptb_model
.
state_dict
()
...
@@ -295,10 +304,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
...
@@ -295,10 +304,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
np_t
=
v
.
numpy
()
np_t
=
v
.
numpy
()
self
.
model_base
[
k
]
=
np_t
self
.
model_base
[
k
]
=
np_t
paddle
.
save
(
self
.
state_dict
,
"./test_dy_v2.pdparams"
)
paddle
.
save
(
self
.
state_dict
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_dy_v2.pdparams"
))
def
func_testLoadAndSetVarBase
(
self
):
def
func_testLoadAndSetVarBase
(
self
):
self
.
setUp
()
seed
=
90
seed
=
90
hidden_size
=
10
hidden_size
=
10
vocab_size
=
1000
vocab_size
=
1000
...
@@ -375,8 +384,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
...
@@ -375,8 +384,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
v
.
numpy
()))
==
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
v
.
numpy
()))
==
0
)
para_state_dict
=
paddle
.
load
(
"./test_dy_v2.pdparams"
)
para_state_dict
=
paddle
.
load
(
opti_state_dict
=
paddle
.
load
(
"./test_dy_v2.pdopt"
)
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_dy_v2.pdparams"
))
opti_state_dict
=
paddle
.
load
(
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_dy_v2.pdopt"
))
adam
.
set_state_dict
(
opti_state_dict
)
adam
.
set_state_dict
(
opti_state_dict
)
opti_dict
=
adam
.
state_dict
()
opti_dict
=
adam
.
state_dict
()
...
@@ -750,7 +761,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
...
@@ -750,7 +761,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
last_hidden
=
None
last_hidden
=
None
last_cell
=
None
last_cell
=
None
state_dict
,
opti_dict
=
fluid
.
load_dygraph
(
"./test_dy_v2"
)
state_dict
,
opti_dict
=
fluid
.
load_dygraph
(
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_dy_v2"
))
adam
.
set_state_dict
(
opti_dict
)
adam
.
set_state_dict
(
opti_dict
)
ptb_model
.
set_dict
(
state_dict
)
ptb_model
.
set_dict
(
state_dict
)
...
@@ -907,19 +919,23 @@ class TestDygraphPtbRnn(unittest.TestCase):
...
@@ -907,19 +919,23 @@ class TestDygraphPtbRnn(unittest.TestCase):
with
fluid
.
dygraph
.
guard
():
with
fluid
.
dygraph
.
guard
():
emb
=
fluid
.
dygraph
.
Embedding
([
10
,
10
])
emb
=
fluid
.
dygraph
.
Embedding
([
10
,
10
])
state_dict
=
emb
.
state_dict
()
state_dict
=
emb
.
state_dict
()
paddle
.
save
(
state_dict
,
os
.
path
.
join
(
'saved_dy'
,
'emb_dy.pdparams'
))
paddle
.
save
(
state_dict
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'saved_dy'
,
'emb_dy.pdparams'
))
para_state_dict
=
paddle
.
load
(
para_state_dict
=
paddle
.
load
(
os
.
path
.
join
(
'saved_dy'
,
'emb_dy.pdparams'
))
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'saved_dy'
,
'emb_dy.pdparams'
))
def
func_test_no_state_in_input_dict
(
self
):
def
func_test_no_state_in_input_dict
(
self
):
with
fluid
.
dygraph
.
guard
():
with
fluid
.
dygraph
.
guard
():
emb
=
fluid
.
dygraph
.
Embedding
([
10
,
10
])
emb
=
fluid
.
dygraph
.
Embedding
([
10
,
10
])
state_dict
=
emb
.
state_dict
()
state_dict
=
emb
.
state_dict
()
paddle
.
save
(
state_dict
,
os
.
path
.
join
(
'saved_dy'
,
'emb_dy.pdparams'
))
paddle
.
save
(
state_dict
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'saved_dy'
,
'emb_dy.pdparams'
))
para_state_dict
=
paddle
.
load
(
para_state_dict
=
paddle
.
load
(
os
.
path
.
join
(
'saved_dy'
,
'emb_dy.pdparams'
))
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'saved_dy'
,
'emb_dy.pdparams'
))
para_state_dict
.
pop
(
'weight'
)
para_state_dict
.
pop
(
'weight'
)
emb
.
set_state_dict
(
para_state_dict
)
emb
.
set_state_dict
(
para_state_dict
)
...
@@ -928,10 +944,13 @@ class TestDygraphPtbRnn(unittest.TestCase):
...
@@ -928,10 +944,13 @@ class TestDygraphPtbRnn(unittest.TestCase):
with
fluid
.
dygraph
.
guard
():
with
fluid
.
dygraph
.
guard
():
emb
=
fluid
.
dygraph
.
Embedding
([
10
,
10
])
emb
=
fluid
.
dygraph
.
Embedding
([
10
,
10
])
state_dict
=
emb
.
state_dict
()
state_dict
=
emb
.
state_dict
()
paddle
.
save
(
state_dict
,
os
.
path
.
join
(
'saved_dy'
,
'emb_dy.pdparams'
))
paddle
.
save
(
state_dict
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'saved_dy'
,
'emb_dy.pdparams'
))
para_state_dict
=
paddle
.
load
(
para_state_dict
=
paddle
.
load
(
os
.
path
.
join
(
'saved_dy'
,
'emb_dy.pdparams'
),
return_numpy
=
True
)
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'saved_dy'
,
'emb_dy.pdparams'
),
return_numpy
=
True
)
para_state_dict
[
'weight'
]
=
np
.
expand_dims
(
para_state_dict
[
'weight'
]
=
np
.
expand_dims
(
para_state_dict
[
'weight'
],
axis
=-
1
)
para_state_dict
[
'weight'
],
axis
=-
1
)
...
...
python/paddle/fluid/tests/unittests/test_io_save_load.py
浏览文件 @
2eb60ddb
...
@@ -19,9 +19,18 @@ import paddle
...
@@ -19,9 +19,18 @@ import paddle
import
paddle.fluid
as
fluid
import
paddle.fluid
as
fluid
from
paddle.fluid
import
core
from
paddle.fluid
import
core
from
paddle.fluid.framework
import
_test_eager_guard
,
_in_legacy_dygraph
from
paddle.fluid.framework
import
_test_eager_guard
,
_in_legacy_dygraph
import
tempfile
import
os
class
TestSaveLoadAPIError
(
unittest
.
TestCase
):
class
TestSaveLoadAPIError
(
unittest
.
TestCase
):
def
setUp
(
self
):
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
self
.
save_dir
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"fake_dir"
)
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
func_test_get_valid_program_error
(
self
):
def
func_test_get_valid_program_error
(
self
):
# case 1: CompiledProgram no program
# case 1: CompiledProgram no program
graph
=
core
.
Graph
(
core
.
ProgramDesc
())
graph
=
core
.
Graph
(
core
.
ProgramDesc
())
...
@@ -44,13 +53,13 @@ class TestSaveLoadAPIError(unittest.TestCase):
...
@@ -44,13 +53,13 @@ class TestSaveLoadAPIError(unittest.TestCase):
# case 1: main_program type error when vars None
# case 1: main_program type error when vars None
with
self
.
assertRaises
(
TypeError
):
with
self
.
assertRaises
(
TypeError
):
fluid
.
io
.
load_vars
(
fluid
.
io
.
load_vars
(
executor
=
exe
,
dirname
=
"./fake_dir"
,
main_program
=
"program"
)
executor
=
exe
,
dirname
=
self
.
save_dir
,
main_program
=
"program"
)
# case 2: main_program type error when vars not None
# case 2: main_program type error when vars not None
with
self
.
assertRaises
(
TypeError
):
with
self
.
assertRaises
(
TypeError
):
fluid
.
io
.
load_vars
(
fluid
.
io
.
load_vars
(
executor
=
exe
,
executor
=
exe
,
dirname
=
"./fake_dir"
,
dirname
=
self
.
save_dir
,
main_program
=
"program"
,
main_program
=
"program"
,
vars
=
"vars"
)
vars
=
"vars"
)
...
@@ -61,6 +70,12 @@ class TestSaveLoadAPIError(unittest.TestCase):
...
@@ -61,6 +70,12 @@ class TestSaveLoadAPIError(unittest.TestCase):
class
TestSaveInferenceModelAPIError
(
unittest
.
TestCase
):
class
TestSaveInferenceModelAPIError
(
unittest
.
TestCase
):
def
setUp
(
self
):
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
func_test_useless_feeded_var_names
(
self
):
def
func_test_useless_feeded_var_names
(
self
):
start_prog
=
fluid
.
Program
()
start_prog
=
fluid
.
Program
()
main_prog
=
fluid
.
Program
()
main_prog
=
fluid
.
Program
()
...
@@ -74,7 +89,7 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
...
@@ -74,7 +89,7 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
with
self
.
assertRaisesRegexp
(
with
self
.
assertRaisesRegexp
(
ValueError
,
"not involved in the target_vars calculation"
):
ValueError
,
"not involved in the target_vars calculation"
):
fluid
.
io
.
save_inference_model
(
fluid
.
io
.
save_inference_model
(
dirname
=
'./model'
,
dirname
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'model'
)
,
feeded_var_names
=
[
'x'
,
'y'
],
feeded_var_names
=
[
'x'
,
'y'
],
target_vars
=
[
z
],
target_vars
=
[
z
],
executor
=
exe
,
executor
=
exe
,
...
@@ -87,13 +102,20 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
...
@@ -87,13 +102,20 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
class
TestWhenTrainWithNoGrad
(
unittest
.
TestCase
):
class
TestWhenTrainWithNoGrad
(
unittest
.
TestCase
):
def
setUp
(
self
):
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
func_test_when_train_with_no_grad
(
self
):
def
func_test_when_train_with_no_grad
(
self
):
paddle
.
disable_static
()
paddle
.
disable_static
()
net
=
paddle
.
nn
.
Linear
(
1024
,
1
)
net
=
paddle
.
nn
.
Linear
(
1024
,
1
)
net
=
paddle
.
jit
.
to_static
(
net
)
net
=
paddle
.
jit
.
to_static
(
net
)
x
=
paddle
.
rand
([
1024
],
'float32'
)
x
=
paddle
.
rand
([
1024
],
'float32'
)
net
(
x
)
net
(
x
)
save_path
=
'./train_with_no_grad'
save_path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'train_with_no_grad'
)
paddle
.
jit
.
save
(
net
,
save_path
)
paddle
.
jit
.
save
(
net
,
save_path
)
net
=
paddle
.
jit
.
load
(
save_path
)
net
=
paddle
.
jit
.
load
(
save_path
)
net
.
train
()
net
.
train
()
...
...
python/paddle/fluid/tests/unittests/test_jit_save_load.py
浏览文件 @
2eb60ddb
此差异已折叠。
点击以展开。
python/paddle/fluid/tests/unittests/test_load_op.py
浏览文件 @
2eb60ddb
...
@@ -19,6 +19,8 @@ import numpy as np
...
@@ -19,6 +19,8 @@ import numpy as np
from
op_test
import
OpTest
,
randomize_probability
from
op_test
import
OpTest
,
randomize_probability
import
paddle.fluid
as
fluid
import
paddle.fluid
as
fluid
import
paddle.fluid.layers
as
layers
import
paddle.fluid.layers
as
layers
import
os
import
tempfile
class
TestLoadOp
(
unittest
.
TestCase
):
class
TestLoadOp
(
unittest
.
TestCase
):
...
@@ -26,6 +28,7 @@ class TestLoadOp(unittest.TestCase):
...
@@ -26,6 +28,7 @@ class TestLoadOp(unittest.TestCase):
"""
"""
def
setUp
(
self
):
def
setUp
(
self
):
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
self
.
ones
=
np
.
ones
((
4
,
4
)).
astype
(
'float32'
)
self
.
ones
=
np
.
ones
((
4
,
4
)).
astype
(
'float32'
)
main_prog
=
fluid
.
Program
()
main_prog
=
fluid
.
Program
()
start_prog
=
fluid
.
Program
()
start_prog
=
fluid
.
Program
()
...
@@ -41,14 +44,20 @@ class TestLoadOp(unittest.TestCase):
...
@@ -41,14 +44,20 @@ class TestLoadOp(unittest.TestCase):
exe
=
fluid
.
Executor
(
fluid
.
CPUPlace
())
exe
=
fluid
.
Executor
(
fluid
.
CPUPlace
())
exe
.
run
(
start_prog
)
exe
.
run
(
start_prog
)
fluid
.
io
.
save_persistables
(
fluid
.
io
.
save_persistables
(
exe
,
dirname
=
"./model"
,
main_program
=
main_prog
)
exe
,
dirname
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"./model"
),
main_program
=
main_prog
)
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
test_load
(
self
):
def
test_load
(
self
):
main_prog
=
fluid
.
Program
()
main_prog
=
fluid
.
Program
()
start_prog
=
fluid
.
Program
()
start_prog
=
fluid
.
Program
()
with
fluid
.
program_guard
(
main_prog
,
start_prog
):
with
fluid
.
program_guard
(
main_prog
,
start_prog
):
var
=
layers
.
create_tensor
(
dtype
=
'float32'
)
var
=
layers
.
create_tensor
(
dtype
=
'float32'
)
layers
.
load
(
var
,
file_path
=
'./model/w'
)
layers
.
load
(
var
,
file_path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'./model/w'
))
exe
=
fluid
.
Executor
(
fluid
.
CPUPlace
())
exe
=
fluid
.
Executor
(
fluid
.
CPUPlace
())
exe
.
run
(
start_prog
)
exe
.
run
(
start_prog
)
...
...
python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py
浏览文件 @
2eb60ddb
...
@@ -23,6 +23,7 @@ import paddle
...
@@ -23,6 +23,7 @@ import paddle
import
paddle.fluid
as
fluid
import
paddle.fluid
as
fluid
from
paddle.fluid
import
core
from
paddle.fluid
import
core
from
test_imperative_base
import
new_program_scope
from
test_imperative_base
import
new_program_scope
import
tempfile
def
convolutional_neural_network
(
img
):
def
convolutional_neural_network
(
img
):
...
@@ -59,6 +60,7 @@ def static_train_net(img, label):
...
@@ -59,6 +60,7 @@ def static_train_net(img, label):
class
TestLoadStateDictFromSaveInferenceModel
(
unittest
.
TestCase
):
class
TestLoadStateDictFromSaveInferenceModel
(
unittest
.
TestCase
):
def
setUp
(
self
):
def
setUp
(
self
):
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
self
.
seed
=
90
self
.
seed
=
90
self
.
epoch_num
=
1
self
.
epoch_num
=
1
self
.
batch_size
=
128
self
.
batch_size
=
128
...
@@ -66,6 +68,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
...
@@ -66,6 +68,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
# enable static mode
# enable static mode
paddle
.
enable_static
()
paddle
.
enable_static
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
train_and_save_model
(
self
,
only_params
=
False
):
def
train_and_save_model
(
self
,
only_params
=
False
):
with
new_program_scope
():
with
new_program_scope
():
startup_program
=
fluid
.
default_startup_program
()
startup_program
=
fluid
.
default_startup_program
()
...
@@ -121,7 +126,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
...
@@ -121,7 +126,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self
.
assertTrue
(
np
.
array_equal
(
value
,
load_dict
[
var_name
]))
self
.
assertTrue
(
np
.
array_equal
(
value
,
load_dict
[
var_name
]))
def
test_load_default
(
self
):
def
test_load_default
(
self
):
self
.
save_dirname
=
"static_mnist.load_state_dict.default"
self
.
save_dirname
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"static_mnist.load_state_dict.default"
)
self
.
model_filename
=
None
self
.
model_filename
=
None
self
.
params_filename
=
None
self
.
params_filename
=
None
orig_param_dict
=
self
.
train_and_save_model
()
orig_param_dict
=
self
.
train_and_save_model
()
...
@@ -133,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
...
@@ -133,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
def
test_load_with_model_filename
(
self
):
def
test_load_with_model_filename
(
self
):
self
.
save_dirname
=
"static_mnist.load_state_dict.model_filename"
self
.
save_dirname
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"static_mnist.load_state_dict.model_filename"
)
self
.
model_filename
=
"static_mnist.model"
self
.
model_filename
=
"static_mnist.model"
self
.
params_filename
=
None
self
.
params_filename
=
None
orig_param_dict
=
self
.
train_and_save_model
()
orig_param_dict
=
self
.
train_and_save_model
()
...
@@ -147,7 +154,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
...
@@ -147,7 +154,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
def
test_load_with_param_filename
(
self
):
def
test_load_with_param_filename
(
self
):
self
.
save_dirname
=
"static_mnist.load_state_dict.param_filename"
self
.
save_dirname
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"static_mnist.load_state_dict.param_filename"
)
self
.
model_filename
=
None
self
.
model_filename
=
None
self
.
params_filename
=
"static_mnist.params"
self
.
params_filename
=
"static_mnist.params"
orig_param_dict
=
self
.
train_and_save_model
()
orig_param_dict
=
self
.
train_and_save_model
()
...
@@ -161,7 +169,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
...
@@ -161,7 +169,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
def
test_load_with_model_and_param_filename
(
self
):
def
test_load_with_model_and_param_filename
(
self
):
self
.
save_dirname
=
"static_mnist.load_state_dict.model_and_param_filename"
self
.
save_dirname
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"static_mnist.load_state_dict.model_and_param_filename"
)
self
.
model_filename
=
"static_mnist.model"
self
.
model_filename
=
"static_mnist.model"
self
.
params_filename
=
"static_mnist.params"
self
.
params_filename
=
"static_mnist.params"
orig_param_dict
=
self
.
train_and_save_model
()
orig_param_dict
=
self
.
train_and_save_model
()
...
@@ -179,7 +189,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
...
@@ -179,7 +189,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
self
.
check_load_state_dict
(
orig_param_dict
,
new_load_param_dict
)
def
test_load_state_dict_from_save_params
(
self
):
def
test_load_state_dict_from_save_params
(
self
):
self
.
save_dirname
=
"static_mnist.load_state_dict.save_params"
self
.
save_dirname
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"static_mnist.load_state_dict.save_params"
)
self
.
params_filename
=
None
self
.
params_filename
=
None
orig_param_dict
=
self
.
train_and_save_model
(
True
)
orig_param_dict
=
self
.
train_and_save_model
(
True
)
...
...
python/paddle/fluid/tests/unittests/test_paddle_save_load.py
浏览文件 @
2eb60ddb
...
@@ -19,6 +19,7 @@ import numpy as np
...
@@ -19,6 +19,7 @@ import numpy as np
import
os
import
os
import
sys
import
sys
from
io
import
BytesIO
from
io
import
BytesIO
import
tempfile
import
paddle
import
paddle
import
paddle.nn
as
nn
import
paddle.nn
as
nn
...
@@ -90,7 +91,10 @@ def train(layer, loader, loss_fn, opt):
...
@@ -90,7 +91,10 @@ def train(layer, loader, loss_fn, opt):
class
TestSaveLoadLargeParameters
(
unittest
.
TestCase
):
class
TestSaveLoadLargeParameters
(
unittest
.
TestCase
):
def
setUp
(
self
):
def
setUp
(
self
):
pass
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
test_large_parameters_paddle_save
(
self
):
def
test_large_parameters_paddle_save
(
self
):
# enable dygraph mode
# enable dygraph mode
...
@@ -100,7 +104,8 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
...
@@ -100,7 +104,8 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
layer
=
LayerWithLargeParameters
()
layer
=
LayerWithLargeParameters
()
save_dict
=
layer
.
state_dict
()
save_dict
=
layer
.
state_dict
()
path
=
os
.
path
.
join
(
"test_paddle_save_load_large_param_save"
,
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_paddle_save_load_large_param_save"
,
"layer.pdparams"
)
"layer.pdparams"
)
protocol
=
4
protocol
=
4
paddle
.
save
(
save_dict
,
path
,
protocol
=
protocol
)
paddle
.
save
(
save_dict
,
path
,
protocol
=
protocol
)
...
@@ -111,6 +116,12 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
...
@@ -111,6 +116,12 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
class
TestSaveLoadPickle
(
unittest
.
TestCase
):
class
TestSaveLoadPickle
(
unittest
.
TestCase
):
def
setUp
(
self
):
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
test_pickle_protocol
(
self
):
def
test_pickle_protocol
(
self
):
# enable dygraph mode
# enable dygraph mode
paddle
.
disable_static
()
paddle
.
disable_static
()
...
@@ -118,7 +129,8 @@ class TestSaveLoadPickle(unittest.TestCase):
...
@@ -118,7 +129,8 @@ class TestSaveLoadPickle(unittest.TestCase):
layer
=
LinearNet
()
layer
=
LinearNet
()
save_dict
=
layer
.
state_dict
()
save_dict
=
layer
.
state_dict
()
path
=
os
.
path
.
join
(
"test_paddle_save_load_pickle_protocol"
,
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_paddle_save_load_pickle_protocol"
,
"layer.pdparams"
)
"layer.pdparams"
)
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
...
@@ -143,6 +155,12 @@ class TestSaveLoadPickle(unittest.TestCase):
...
@@ -143,6 +155,12 @@ class TestSaveLoadPickle(unittest.TestCase):
class
TestSaveLoadAny
(
unittest
.
TestCase
):
class
TestSaveLoadAny
(
unittest
.
TestCase
):
def
setUp
(
self
):
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
set_zero
(
self
,
prog
,
place
,
scope
=
None
):
def
set_zero
(
self
,
prog
,
place
,
scope
=
None
):
if
scope
is
None
:
if
scope
is
None
:
scope
=
fluid
.
global_scope
()
scope
=
fluid
.
global_scope
()
...
@@ -203,7 +221,8 @@ class TestSaveLoadAny(unittest.TestCase):
...
@@ -203,7 +221,8 @@ class TestSaveLoadAny(unittest.TestCase):
t
=
np
.
array
(
fluid
.
global_scope
().
find_var
(
var
.
name
)
t
=
np
.
array
(
fluid
.
global_scope
().
find_var
(
var
.
name
)
.
get_tensor
())
.
get_tensor
())
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
path
=
os
.
path
.
join
(
"test_replace_static_save_load"
,
"model"
)
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_replace_static_save_load"
,
"model"
)
# paddle.save, legacy paddle.fluid.load
# paddle.save, legacy paddle.fluid.load
self
.
replace_static_save
(
prog
,
path
)
self
.
replace_static_save
(
prog
,
path
)
self
.
set_zero
(
prog
,
place
)
self
.
set_zero
(
prog
,
place
)
...
@@ -229,7 +248,9 @@ class TestSaveLoadAny(unittest.TestCase):
...
@@ -229,7 +248,9 @@ class TestSaveLoadAny(unittest.TestCase):
for
var
in
prog
.
list_vars
():
for
var
in
prog
.
list_vars
():
if
var
.
persistable
:
if
var
.
persistable
:
tensor
=
var
.
get_value
(
fluid
.
global_scope
())
tensor
=
var
.
get_value
(
fluid
.
global_scope
())
paddle
.
save
(
tensor
,
os
.
path
.
join
(
path_vars
,
var
.
name
))
paddle
.
save
(
tensor
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
path_vars
,
var
.
name
))
with
self
.
assertRaises
(
TypeError
):
with
self
.
assertRaises
(
TypeError
):
var
.
get_value
(
'fluid.global_scope()'
)
var
.
get_value
(
'fluid.global_scope()'
)
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
...
@@ -248,7 +269,8 @@ class TestSaveLoadAny(unittest.TestCase):
...
@@ -248,7 +269,8 @@ class TestSaveLoadAny(unittest.TestCase):
for
var
in
prog
.
list_vars
():
for
var
in
prog
.
list_vars
():
if
var
.
persistable
:
if
var
.
persistable
:
tensor
=
paddle
.
load
(
tensor
=
paddle
.
load
(
os
.
path
.
join
(
path_vars
,
var
.
name
),
return_numpy
=
False
)
os
.
path
.
join
(
self
.
temp_dir
.
name
,
path_vars
,
var
.
name
),
return_numpy
=
False
)
var
.
set_value
(
tensor
)
var
.
set_value
(
tensor
)
new_t
=
np
.
array
(
fluid
.
global_scope
().
find_var
(
var
.
name
)
new_t
=
np
.
array
(
fluid
.
global_scope
().
find_var
(
var
.
name
)
.
get_tensor
())
.
get_tensor
())
...
@@ -282,7 +304,8 @@ class TestSaveLoadAny(unittest.TestCase):
...
@@ -282,7 +304,8 @@ class TestSaveLoadAny(unittest.TestCase):
y
.
mean
().
backward
()
y
.
mean
().
backward
()
adam
.
step
()
adam
.
step
()
state_dict
=
adam
.
state_dict
()
state_dict
=
adam
.
state_dict
()
path
=
'paddle_save_load_v2/model.pdparams'
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'paddle_save_load_v2/model.pdparams'
)
with
self
.
assertRaises
(
TypeError
):
with
self
.
assertRaises
(
TypeError
):
paddle
.
save
(
state_dict
,
path
,
use_binary_format
=
'False'
)
paddle
.
save
(
state_dict
,
path
,
use_binary_format
=
'False'
)
# legacy paddle.save, paddle.load
# legacy paddle.save, paddle.load
...
@@ -306,7 +329,8 @@ class TestSaveLoadAny(unittest.TestCase):
...
@@ -306,7 +329,8 @@ class TestSaveLoadAny(unittest.TestCase):
# enable dygraph mode
# enable dygraph mode
paddle
.
disable_static
()
paddle
.
disable_static
()
layer
=
LinearNet
()
layer
=
LinearNet
()
path
=
'paddle_save_load_v2/var_dygraph'
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'paddle_save_load_v2/var_dygraph'
)
tensor
=
layer
.
_linear
.
weight
tensor
=
layer
.
_linear
.
weight
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
paddle
.
save
(
tensor
,
path
,
pickle_protocol
=
'3'
)
paddle
.
save
(
tensor
,
path
,
pickle_protocol
=
'3'
)
...
@@ -348,7 +372,8 @@ class TestSaveLoadAny(unittest.TestCase):
...
@@ -348,7 +372,8 @@ class TestSaveLoadAny(unittest.TestCase):
break
break
scope
=
fluid
.
global_scope
()
scope
=
fluid
.
global_scope
()
origin_tensor
=
np
.
array
(
tensor
)
origin_tensor
=
np
.
array
(
tensor
)
path
=
'test_single_pickle_var_static/var'
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_single_pickle_var_static/var'
)
paddle
.
save
(
tensor
,
path
)
paddle
.
save
(
tensor
,
path
)
self
.
set_zero
(
prog
,
place
,
scope
)
self
.
set_zero
(
prog
,
place
,
scope
)
# static load
# static load
...
@@ -370,7 +395,8 @@ class TestSaveLoadAny(unittest.TestCase):
...
@@ -370,7 +395,8 @@ class TestSaveLoadAny(unittest.TestCase):
def
test_dygraph_save_static_load
(
self
):
def
test_dygraph_save_static_load
(
self
):
inps
=
np
.
random
.
randn
(
1
,
IMAGE_SIZE
).
astype
(
'float32'
)
inps
=
np
.
random
.
randn
(
1
,
IMAGE_SIZE
).
astype
(
'float32'
)
path
=
'test_dygraph_save_static_load/dy-static.pdparams'
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_dygraph_save_static_load/dy-static.pdparams'
)
paddle
.
disable_static
()
paddle
.
disable_static
()
with
paddle
.
utils
.
unique_name
.
guard
():
with
paddle
.
utils
.
unique_name
.
guard
():
layer
=
LinearNet
()
layer
=
LinearNet
()
...
@@ -414,10 +440,14 @@ class TestSaveLoadAny(unittest.TestCase):
...
@@ -414,10 +440,14 @@ class TestSaveLoadAny(unittest.TestCase):
})
})
obj4
=
(
np
.
random
.
randn
(
5
,
6
),
(
123
,
))
obj4
=
(
np
.
random
.
randn
(
5
,
6
),
(
123
,
))
path1
=
"test_save_load_any_complex_object_dygraph/obj1"
path1
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
path2
=
"test_save_load_any_complex_object_dygraph/obj2"
"test_save_load_any_complex_object_dygraph/obj1"
)
path3
=
"test_save_load_any_complex_object_dygraph/obj3"
path2
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
path4
=
"test_save_load_any_complex_object_dygraph/obj4"
"test_save_load_any_complex_object_dygraph/obj2"
)
path3
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_save_load_any_complex_object_dygraph/obj3"
)
path4
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_save_load_any_complex_object_dygraph/obj4"
)
paddle
.
save
(
obj1
,
path1
)
paddle
.
save
(
obj1
,
path1
)
paddle
.
save
(
obj2
,
path2
)
paddle
.
save
(
obj2
,
path2
)
paddle
.
save
(
obj3
,
path3
)
paddle
.
save
(
obj3
,
path3
)
...
@@ -584,10 +614,18 @@ class TestSaveLoadAny(unittest.TestCase):
...
@@ -584,10 +614,18 @@ class TestSaveLoadAny(unittest.TestCase):
})
})
obj4
=
(
np
.
ndarray
([
3
,
4
],
dtype
=
"float32"
),
)
obj4
=
(
np
.
ndarray
([
3
,
4
],
dtype
=
"float32"
),
)
path1
=
"test_save_load_any_complex_object_static/obj1"
path1
=
os
.
path
.
join
(
path2
=
"test_save_load_any_complex_object_static/obj2"
self
.
temp_dir
.
name
,
path3
=
"test_save_load_any_complex_object_static/obj3"
"test_save_load_any_complex_object_static/obj1"
)
path4
=
"test_save_load_any_complex_object_static/obj4"
path2
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_save_load_any_complex_object_static/obj2"
)
path3
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_save_load_any_complex_object_static/obj3"
)
path4
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_save_load_any_complex_object_static/obj4"
)
paddle
.
save
(
obj1
,
path1
)
paddle
.
save
(
obj1
,
path1
)
paddle
.
save
(
obj2
,
path2
)
paddle
.
save
(
obj2
,
path2
)
paddle
.
save
(
obj3
,
path3
)
paddle
.
save
(
obj3
,
path3
)
...
@@ -751,7 +789,8 @@ class TestSaveLoadAny(unittest.TestCase):
...
@@ -751,7 +789,8 @@ class TestSaveLoadAny(unittest.TestCase):
def
test_varbase_binary_var
(
self
):
def
test_varbase_binary_var
(
self
):
paddle
.
disable_static
()
paddle
.
disable_static
()
varbase
=
paddle
.
randn
([
3
,
2
],
dtype
=
'float32'
)
varbase
=
paddle
.
randn
([
3
,
2
],
dtype
=
'float32'
)
path
=
'test_paddle_save_load_varbase_binary_var/varbase'
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_paddle_save_load_varbase_binary_var/varbase'
)
paddle
.
save
(
varbase
,
path
,
use_binary_format
=
True
)
paddle
.
save
(
varbase
,
path
,
use_binary_format
=
True
)
load_array
=
paddle
.
load
(
path
,
return_numpy
=
True
)
load_array
=
paddle
.
load
(
path
,
return_numpy
=
True
)
load_tensor
=
paddle
.
load
(
path
,
return_numpy
=
False
)
load_tensor
=
paddle
.
load
(
path
,
return_numpy
=
False
)
...
@@ -836,6 +875,10 @@ class TestSaveLoad(unittest.TestCase):
...
@@ -836,6 +875,10 @@ class TestSaveLoad(unittest.TestCase):
# config seed
# config seed
paddle
.
seed
(
SEED
)
paddle
.
seed
(
SEED
)
paddle
.
framework
.
random
.
_manual_program_seed
(
SEED
)
paddle
.
framework
.
random
.
_manual_program_seed
(
SEED
)
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
build_and_train_model
(
self
):
def
build_and_train_model
(
self
):
# create network
# create network
...
@@ -863,8 +906,10 @@ class TestSaveLoad(unittest.TestCase):
...
@@ -863,8 +906,10 @@ class TestSaveLoad(unittest.TestCase):
layer
,
opt
=
self
.
build_and_train_model
()
layer
,
opt
=
self
.
build_and_train_model
()
# save
# save
layer_save_path
=
"test_paddle_save_load.linear.pdparams"
layer_save_path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
opt_save_path
=
"test_paddle_save_load.linear.pdopt"
"test_paddle_save_load.linear.pdparams"
)
opt_save_path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_paddle_save_load.linear.pdopt"
)
layer_state_dict
=
layer
.
state_dict
()
layer_state_dict
=
layer
.
state_dict
()
opt_state_dict
=
opt
.
state_dict
()
opt_state_dict
=
opt
.
state_dict
()
...
@@ -880,7 +925,9 @@ class TestSaveLoad(unittest.TestCase):
...
@@ -880,7 +925,9 @@ class TestSaveLoad(unittest.TestCase):
# test save load in static mode
# test save load in static mode
paddle
.
enable_static
()
paddle
.
enable_static
()
static_save_path
=
"static_mode_test/test_paddle_save_load.linear.pdparams"
static_save_path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"static_mode_test/test_paddle_save_load.linear.pdparams"
)
paddle
.
save
(
layer_state_dict
,
static_save_path
)
paddle
.
save
(
layer_state_dict
,
static_save_path
)
load_static_state_dict
=
paddle
.
load
(
static_save_path
)
load_static_state_dict
=
paddle
.
load
(
static_save_path
)
self
.
check_load_state_dict
(
layer_state_dict
,
load_static_state_dict
)
self
.
check_load_state_dict
(
layer_state_dict
,
load_static_state_dict
)
...
@@ -891,20 +938,28 @@ class TestSaveLoad(unittest.TestCase):
...
@@ -891,20 +938,28 @@ class TestSaveLoad(unittest.TestCase):
# 2. test save path format error
# 2. test save path format error
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
paddle
.
save
(
layer_state_dict
,
"test_paddle_save_load.linear.model/"
)
paddle
.
save
(
layer_state_dict
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_paddle_save_load.linear.model/"
))
# 3. test load path not exist error
# 3. test load path not exist error
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
paddle
.
load
(
"test_paddle_save_load.linear.params"
)
paddle
.
load
(
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_paddle_save_load.linear.params"
))
# 4. test load old save path error
# 4. test load old save path error
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
paddle
.
load
(
"test_paddle_save_load.linear"
)
paddle
.
load
(
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_paddle_save_load.linear"
))
class
TestSaveLoadProgram
(
unittest
.
TestCase
):
class
TestSaveLoadProgram
(
unittest
.
TestCase
):
def
test_save_load_program
(
self
):
def
test_save_load_program
(
self
):
paddle
.
enable_static
()
paddle
.
enable_static
()
temp_dir
=
tempfile
.
TemporaryDirectory
()
with
new_program_scope
():
with
new_program_scope
():
layer
=
LinearNet
()
layer
=
LinearNet
()
data
=
paddle
.
static
.
data
(
data
=
paddle
.
static
.
data
(
...
@@ -914,8 +969,12 @@ class TestSaveLoadProgram(unittest.TestCase):
...
@@ -914,8 +969,12 @@ class TestSaveLoadProgram(unittest.TestCase):
startup_program
=
paddle
.
static
.
default_startup_program
()
startup_program
=
paddle
.
static
.
default_startup_program
()
origin_main
=
main_program
.
desc
.
serialize_to_string
()
origin_main
=
main_program
.
desc
.
serialize_to_string
()
origin_startup
=
startup_program
.
desc
.
serialize_to_string
()
origin_startup
=
startup_program
.
desc
.
serialize_to_string
()
path1
=
"test_paddle_save_load_program/main_program.pdmodel"
path1
=
os
.
path
.
join
(
path2
=
"test_paddle_save_load_program/startup_program.pdmodel"
temp_dir
.
name
,
"test_paddle_save_load_program/main_program.pdmodel"
)
path2
=
os
.
path
.
join
(
temp_dir
.
name
,
"test_paddle_save_load_program/startup_program.pdmodel"
)
paddle
.
save
(
main_program
,
path1
)
paddle
.
save
(
main_program
,
path1
)
paddle
.
save
(
startup_program
,
path2
)
paddle
.
save
(
startup_program
,
path2
)
...
@@ -924,11 +983,13 @@ class TestSaveLoadProgram(unittest.TestCase):
...
@@ -924,11 +983,13 @@ class TestSaveLoadProgram(unittest.TestCase):
load_startup
=
paddle
.
load
(
path2
).
desc
.
serialize_to_string
()
load_startup
=
paddle
.
load
(
path2
).
desc
.
serialize_to_string
()
self
.
assertTrue
(
origin_main
==
load_main
)
self
.
assertTrue
(
origin_main
==
load_main
)
self
.
assertTrue
(
origin_startup
==
load_startup
)
self
.
assertTrue
(
origin_startup
==
load_startup
)
temp_dir
.
cleanup
()
class
TestSaveLoadLayer
(
unittest
.
TestCase
):
class
TestSaveLoadLayer
(
unittest
.
TestCase
):
def
test_save_load_layer
(
self
):
def
test_save_load_layer
(
self
):
paddle
.
disable_static
()
paddle
.
disable_static
()
temp_dir
=
tempfile
.
TemporaryDirectory
()
inps
=
paddle
.
randn
([
1
,
IMAGE_SIZE
],
dtype
=
'float32'
)
inps
=
paddle
.
randn
([
1
,
IMAGE_SIZE
],
dtype
=
'float32'
)
layer1
=
LinearNet
()
layer1
=
LinearNet
()
layer2
=
LinearNet
()
layer2
=
LinearNet
()
...
@@ -936,9 +997,11 @@ class TestSaveLoadLayer(unittest.TestCase):
...
@@ -936,9 +997,11 @@ class TestSaveLoadLayer(unittest.TestCase):
layer2
.
eval
()
layer2
.
eval
()
origin_layer
=
(
layer1
,
layer2
)
origin_layer
=
(
layer1
,
layer2
)
origin
=
(
layer1
(
inps
),
layer2
(
inps
))
origin
=
(
layer1
(
inps
),
layer2
(
inps
))
path
=
"test_save_load_layer_/layer.pdmodel"
path
=
os
.
path
.
join
(
temp_dir
.
name
,
"test_save_load_layer_/layer.pdmodel"
)
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
paddle
.
save
(
origin_layer
,
path
)
paddle
.
save
(
origin_layer
,
path
)
temp_dir
.
cleanup
()
if
__name__
==
'__main__'
:
if
__name__
==
'__main__'
:
...
...
python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py
浏览文件 @
2eb60ddb
...
@@ -21,6 +21,7 @@ import os
...
@@ -21,6 +21,7 @@ import os
import
sys
import
sys
import
six
import
six
import
platform
import
platform
import
tempfile
import
paddle
import
paddle
import
paddle.nn
as
nn
import
paddle.nn
as
nn
...
@@ -37,6 +38,10 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
...
@@ -37,6 +38,10 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
def
setUp
(
self
):
def
setUp
(
self
):
# enable static graph mode
# enable static graph mode
paddle
.
enable_static
()
paddle
.
enable_static
()
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
set_zero
(
self
,
prog
,
place
,
scope
=
None
):
def
set_zero
(
self
,
prog
,
place
,
scope
=
None
):
if
scope
is
None
:
if
scope
is
None
:
...
@@ -94,7 +99,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
...
@@ -94,7 +99,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
# test for replace_save_vars/io.load_vars
# test for replace_save_vars/io.load_vars
path_vars1
=
'test_replace_save_load_vars_binary1/model'
path_vars1
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_replace_save_load_vars_binary1/model'
)
self
.
replace_save_vars
(
prog
,
path_vars1
)
self
.
replace_save_vars
(
prog
,
path_vars1
)
# set var to zero
# set var to zero
self
.
set_zero
(
prog
,
place
)
self
.
set_zero
(
prog
,
place
)
...
@@ -111,7 +117,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
...
@@ -111,7 +117,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
# test for io.save_vars/replace_load_vars
# test for io.save_vars/replace_load_vars
path_vars2
=
'test_replace_save_load_vars_binary2/model/'
path_vars2
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_replace_save_load_vars_binary2/model/'
)
fluid
.
io
.
save_vars
(
fluid
.
io
.
save_vars
(
exe
,
path_vars2
,
main_program
=
prog
,
vars
=
var_list
)
exe
,
path_vars2
,
main_program
=
prog
,
vars
=
var_list
)
self
.
set_zero
(
prog
,
place
)
self
.
set_zero
(
prog
,
place
)
...
@@ -141,7 +149,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
...
@@ -141,7 +149,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
prog
=
paddle
.
static
.
default_main_program
()
prog
=
paddle
.
static
.
default_main_program
()
exe
.
run
(
fluid
.
default_startup_program
())
exe
.
run
(
fluid
.
default_startup_program
())
dirname
=
'test_save_load_lod_tensor1/tensor_'
dirname
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_save_load_lod_tensor1/tensor_'
)
for
var
in
prog
.
list_vars
():
for
var
in
prog
.
list_vars
():
if
var
.
persistable
and
list
(
if
var
.
persistable
and
list
(
var
.
shape
)
==
[
IMAGE_SIZE
,
OUTPUT_NUM
]:
var
.
shape
)
==
[
IMAGE_SIZE
,
OUTPUT_NUM
]:
...
@@ -162,12 +171,13 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
...
@@ -162,12 +171,13 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self
.
assertTrue
(
np
.
array_equal
(
origin
,
to_array
))
self
.
assertTrue
(
np
.
array_equal
(
origin
,
to_array
))
with
self
.
assertRaises
(
NotImplementedError
):
with
self
.
assertRaises
(
NotImplementedError
):
path
=
'test_save_load_error/temp'
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_save_load_error/temp'
)
paddle
.
save
({},
path
,
use_binary_format
=
True
)
paddle
.
save
({},
path
,
use_binary_format
=
True
)
# On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk.
# On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk.
if
'Windows'
!=
platform
.
system
():
if
'Windows'
!=
platform
.
system
():
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
path
=
'test_save_load_error/temp'
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_save_load_error/temp'
)
with
open
(
path
,
"w"
)
as
f
:
with
open
(
path
,
"w"
)
as
f
:
f
.
write
(
'
\0
'
)
f
.
write
(
'
\0
'
)
paddle
.
load
(
path
)
paddle
.
load
(
path
)
...
@@ -178,11 +188,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
...
@@ -178,11 +188,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
with
self
.
assertRaises
(
RuntimeError
):
with
self
.
assertRaises
(
RuntimeError
):
fluid
.
core
.
save_lod_tensor
(
fluid
.
core
.
save_lod_tensor
(
temp_lod
,
'test_save_load_error_not_exist_file/not_exist_file'
)
temp_lod
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_save_load_error_not_exist_file/not_exist_file'
))
with
self
.
assertRaises
(
RuntimeError
):
with
self
.
assertRaises
(
RuntimeError
):
fluid
.
core
.
load_lod_tensor
(
fluid
.
core
.
load_lod_tensor
(
temp_lod
,
'test_save_load_error_not_exist_file/not_exist_file'
)
temp_lod
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_save_load_error_not_exist_file/not_exist_file'
))
# save to memory
# save to memory
byio
=
BytesIO
()
byio
=
BytesIO
()
...
@@ -206,7 +222,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
...
@@ -206,7 +222,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
rows
=
[
0
,
4
,
7
]
rows
=
[
0
,
4
,
7
]
row_numel
=
12
row_numel
=
12
selected_rows
=
fluid
.
core
.
SelectedRows
(
rows
,
height
)
selected_rows
=
fluid
.
core
.
SelectedRows
(
rows
,
height
)
path
=
'test_paddle_save_load_selected_rows/sr.pdsr'
path
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_paddle_save_load_selected_rows/sr.pdsr'
)
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
paddle
.
save
(
selected_rows
,
path
,
use_binary_format
=
True
)
paddle
.
save
(
selected_rows
,
path
,
use_binary_format
=
True
)
...
@@ -227,11 +244,15 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
...
@@ -227,11 +244,15 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
with
self
.
assertRaises
(
RuntimeError
):
with
self
.
assertRaises
(
RuntimeError
):
fluid
.
core
.
save_selected_rows
(
fluid
.
core
.
save_selected_rows
(
selected_rows
,
selected_rows
,
'test_paddle_save_load_selected_rows_not_exist_file/temp'
)
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_paddle_save_load_selected_rows_not_exist_file/temp'
))
with
self
.
assertRaises
(
RuntimeError
):
with
self
.
assertRaises
(
RuntimeError
):
fluid
.
core
.
load_selected_rows
(
fluid
.
core
.
load_selected_rows
(
selected_rows
,
selected_rows
,
'test_paddle_save_load_selected_rows_not_exist_file/temp'
)
os
.
path
.
join
(
self
.
temp_dir
.
name
,
'test_paddle_save_load_selected_rows_not_exist_file/temp'
))
# save to memory
# save to memory
byio
=
BytesIO
()
byio
=
BytesIO
()
...
...
python/paddle/fluid/tests/unittests/test_static_save_load.py
浏览文件 @
2eb60ddb
...
@@ -30,6 +30,7 @@ import six
...
@@ -30,6 +30,7 @@ import six
import
pickle
import
pickle
import
os
import
os
import
errno
import
errno
import
tempfile
paddle
.
enable_static
()
paddle
.
enable_static
()
...
@@ -230,6 +231,7 @@ class TestSaveLoadBase(unittest.TestCase):
...
@@ -230,6 +231,7 @@ class TestSaveLoadBase(unittest.TestCase):
init_scale
=
0.1
init_scale
=
0.1
batch_size
=
4
batch_size
=
4
batch_num
=
200
batch_num
=
200
temp_dir
=
tempfile
.
TemporaryDirectory
()
with
new_program_scope
():
with
new_program_scope
():
fluid
.
default_startup_program
().
random_seed
=
seed
fluid
.
default_startup_program
().
random_seed
=
seed
...
@@ -297,7 +299,7 @@ class TestSaveLoadBase(unittest.TestCase):
...
@@ -297,7 +299,7 @@ class TestSaveLoadBase(unittest.TestCase):
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
fluid
.
save
(
main_program
,
"./test_1"
)
fluid
.
save
(
main_program
,
os
.
path
.
join
(
temp_dir
.
name
,
"test_1"
)
)
# set var to zero
# set var to zero
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
...
@@ -310,7 +312,8 @@ class TestSaveLoadBase(unittest.TestCase):
...
@@ -310,7 +312,8 @@ class TestSaveLoadBase(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
# make sure all the paramerter or optimizer var have been set to zero
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
fluid
.
load
(
main_program
,
"./test_1.pdparams"
,
exe
)
fluid
.
load
(
main_program
,
os
.
path
.
join
(
temp_dir
.
name
,
"test_1.pdparams"
),
exe
)
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
...
@@ -318,6 +321,7 @@ class TestSaveLoadBase(unittest.TestCase):
...
@@ -318,6 +321,7 @@ class TestSaveLoadBase(unittest.TestCase):
.
get_tensor
())
.
get_tensor
())
base_t
=
base_map
[
var
.
name
]
base_t
=
base_map
[
var
.
name
]
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
temp_dir
.
cleanup
()
class
TestSaveLoadPartial
(
unittest
.
TestCase
):
class
TestSaveLoadPartial
(
unittest
.
TestCase
):
...
@@ -334,6 +338,7 @@ class TestSaveLoadPartial(unittest.TestCase):
...
@@ -334,6 +338,7 @@ class TestSaveLoadPartial(unittest.TestCase):
init_scale
=
0.1
init_scale
=
0.1
batch_size
=
4
batch_size
=
4
batch_num
=
200
batch_num
=
200
temp_dir
=
tempfile
.
TemporaryDirectory
()
with
new_program_scope
():
with
new_program_scope
():
fluid
.
default_startup_program
().
random_seed
=
seed
fluid
.
default_startup_program
().
random_seed
=
seed
...
@@ -409,7 +414,7 @@ class TestSaveLoadPartial(unittest.TestCase):
...
@@ -409,7 +414,7 @@ class TestSaveLoadPartial(unittest.TestCase):
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
fluid
.
save
(
main_program
,
"./test_1"
)
fluid
.
save
(
main_program
,
os
.
path
.
join
(
temp_dir
.
name
,
"test_1"
)
)
# set var to zero
# set var to zero
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
...
@@ -422,7 +427,8 @@ class TestSaveLoadPartial(unittest.TestCase):
...
@@ -422,7 +427,8 @@ class TestSaveLoadPartial(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
# make sure all the paramerter or optimizer var have been set to zero
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
fluid
.
load
(
test_program
,
"./test_1.pdopt"
,
None
)
fluid
.
load
(
test_program
,
os
.
path
.
join
(
temp_dir
.
name
,
"test_1.pdopt"
),
None
)
for
var
in
test_program
.
list_vars
():
for
var
in
test_program
.
list_vars
():
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
...
@@ -430,7 +436,9 @@ class TestSaveLoadPartial(unittest.TestCase):
...
@@ -430,7 +436,9 @@ class TestSaveLoadPartial(unittest.TestCase):
.
get_tensor
())
.
get_tensor
())
base_t
=
base_map
[
var
.
name
]
base_t
=
base_map
[
var
.
name
]
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
fluid
.
load
(
test_program
,
"./test_1.pdmodel"
,
None
)
fluid
.
load
(
test_program
,
os
.
path
.
join
(
temp_dir
.
name
,
"test_1.pdmodel"
),
None
)
temp_dir
.
cleanup
()
class
TestSaveLoadSetStateDict
(
unittest
.
TestCase
):
class
TestSaveLoadSetStateDict
(
unittest
.
TestCase
):
...
@@ -447,6 +455,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
...
@@ -447,6 +455,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
init_scale
=
0.1
init_scale
=
0.1
batch_size
=
4
batch_size
=
4
batch_num
=
200
batch_num
=
200
temp_dir
=
tempfile
.
TemporaryDirectory
()
with
new_program_scope
():
with
new_program_scope
():
fluid
.
default_startup_program
().
random_seed
=
seed
fluid
.
default_startup_program
().
random_seed
=
seed
...
@@ -514,7 +523,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
...
@@ -514,7 +523,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
fluid
.
save
(
main_program
,
"./test_1"
)
fluid
.
save
(
main_program
,
os
.
path
.
join
(
temp_dir
.
name
,
"test_1"
)
)
# set var to zero
# set var to zero
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
...
@@ -527,7 +536,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
...
@@ -527,7 +536,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
# make sure all the paramerter or optimizer var have been set to zero
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
fluid
.
load
(
main_program
,
"./test_1"
,
exe
)
fluid
.
load
(
main_program
,
os
.
path
.
join
(
temp_dir
.
name
,
"test_1"
)
,
exe
)
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
...
@@ -535,6 +544,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
...
@@ -535,6 +544,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
.
get_tensor
())
.
get_tensor
())
base_t
=
base_map
[
var
.
name
]
base_t
=
base_map
[
var
.
name
]
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
temp_dir
.
cleanup
()
class
TestProgramStatePartial
(
unittest
.
TestCase
):
class
TestProgramStatePartial
(
unittest
.
TestCase
):
...
@@ -551,6 +561,7 @@ class TestProgramStatePartial(unittest.TestCase):
...
@@ -551,6 +561,7 @@ class TestProgramStatePartial(unittest.TestCase):
init_scale
=
0.1
init_scale
=
0.1
batch_size
=
4
batch_size
=
4
batch_num
=
200
batch_num
=
200
temp_dir
=
tempfile
.
TemporaryDirectory
()
with
new_program_scope
():
with
new_program_scope
():
fluid
.
default_startup_program
().
random_seed
=
seed
fluid
.
default_startup_program
().
random_seed
=
seed
...
@@ -626,7 +637,7 @@ class TestProgramStatePartial(unittest.TestCase):
...
@@ -626,7 +637,7 @@ class TestProgramStatePartial(unittest.TestCase):
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
fluid
.
save
(
main_program
,
os
.
path
.
join
(
'some_dir'
,
'test_1'
))
fluid
.
save
(
main_program
,
os
.
path
.
join
(
temp_dir
.
name
,
'test_1'
))
# set var to zero
# set var to zero
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
...
@@ -641,16 +652,16 @@ class TestProgramStatePartial(unittest.TestCase):
...
@@ -641,16 +652,16 @@ class TestProgramStatePartial(unittest.TestCase):
#fluid.load(test_program, "./test_1", None )
#fluid.load(test_program, "./test_1", None )
program_state
=
fluid
.
load_program_state
(
program_state
=
fluid
.
load_program_state
(
os
.
path
.
join
(
'some_dir'
,
'test_1'
))
os
.
path
.
join
(
temp_dir
.
name
,
'test_1'
))
program_state_1
=
fluid
.
load_program_state
(
program_state_1
=
fluid
.
load_program_state
(
os
.
path
.
join
(
'some_dir'
,
'test_1.pdparams'
))
os
.
path
.
join
(
temp_dir
.
name
,
'test_1.pdparams'
))
program_state_2
=
fluid
.
load_program_state
(
program_state_2
=
fluid
.
load_program_state
(
os
.
path
.
join
(
'some_dir'
,
'test_1.pdopt'
))
os
.
path
.
join
(
temp_dir
.
name
,
'test_1.pdopt'
))
program_state_3
=
fluid
.
load_program_state
(
program_state_3
=
fluid
.
load_program_state
(
os
.
path
.
join
(
'some_dir'
,
'test_1.pdmodel'
))
os
.
path
.
join
(
temp_dir
.
name
,
'test_1.pdmodel'
))
fluid
.
set_program_state
(
test_program
,
program_state
)
fluid
.
set_program_state
(
test_program
,
program_state
)
...
@@ -720,6 +731,7 @@ class TestProgramStatePartial(unittest.TestCase):
...
@@ -720,6 +731,7 @@ class TestProgramStatePartial(unittest.TestCase):
.
get_tensor
())
.
get_tensor
())
base_t
=
base_map
[
var
.
name
]
base_t
=
base_map
[
var
.
name
]
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
temp_dir
.
cleanup
()
class
TestVariableInit
(
unittest
.
TestCase
):
class
TestVariableInit
(
unittest
.
TestCase
):
...
@@ -737,7 +749,9 @@ class TestVariableInit(unittest.TestCase):
...
@@ -737,7 +749,9 @@ class TestVariableInit(unittest.TestCase):
exe
=
fluid
.
Executor
(
place
)
exe
=
fluid
.
Executor
(
place
)
exe
.
run
(
fluid
.
default_startup_program
())
exe
.
run
(
fluid
.
default_startup_program
())
fluid
.
save
(
fluid
.
default_main_program
(),
"./test_path"
)
temp_dir
=
tempfile
.
TemporaryDirectory
()
fluid
.
save
(
fluid
.
default_main_program
(),
os
.
path
.
join
(
temp_dir
.
name
,
"test_path"
))
def
set_var
(
var
,
ndarray
):
def
set_var
(
var
,
ndarray
):
t
=
var
.
get_tensor
()
t
=
var
.
get_tensor
()
...
@@ -763,7 +777,7 @@ class TestVariableInit(unittest.TestCase):
...
@@ -763,7 +777,7 @@ class TestVariableInit(unittest.TestCase):
fluid
.
core
.
_create_loaded_parameter
(
parameter_list
,
new_scope
,
fluid
.
core
.
_create_loaded_parameter
(
parameter_list
,
new_scope
,
exe
.
_default_executor
)
exe
.
_default_executor
)
parameter_file_name
=
"./test_path.pdparams"
parameter_file_name
=
os
.
path
.
join
(
temp_dir
.
name
,
"test_path.pdparams"
)
with
open
(
parameter_file_name
,
'rb'
)
as
f
:
with
open
(
parameter_file_name
,
'rb'
)
as
f
:
load_dict
=
pickle
.
load
(
f
)
load_dict
=
pickle
.
load
(
f
)
...
@@ -779,7 +793,7 @@ class TestVariableInit(unittest.TestCase):
...
@@ -779,7 +793,7 @@ class TestVariableInit(unittest.TestCase):
fluid
.
core
.
_create_loaded_parameter
(
opt_list
,
new_scope
,
fluid
.
core
.
_create_loaded_parameter
(
opt_list
,
new_scope
,
exe
.
_default_executor
)
exe
.
_default_executor
)
opt_file_name
=
"./test_path.pdopt"
opt_file_name
=
os
.
path
.
join
(
temp_dir
.
name
,
"test_path.pdopt"
)
with
open
(
opt_file_name
,
'rb'
)
as
f
:
with
open
(
opt_file_name
,
'rb'
)
as
f
:
load_dict
=
pickle
.
load
(
f
)
load_dict
=
pickle
.
load
(
f
)
...
@@ -805,6 +819,7 @@ class TestVariableInit(unittest.TestCase):
...
@@ -805,6 +819,7 @@ class TestVariableInit(unittest.TestCase):
base_t
=
base_map
[
var
.
name
]
base_t
=
base_map
[
var
.
name
]
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
temp_dir
.
cleanup
()
class
TestLoadFromOldInterface
(
unittest
.
TestCase
):
class
TestLoadFromOldInterface
(
unittest
.
TestCase
):
...
@@ -815,10 +830,15 @@ class TestLoadFromOldInterface(unittest.TestCase):
...
@@ -815,10 +830,15 @@ class TestLoadFromOldInterface(unittest.TestCase):
if
os
.
path
.
exists
(
"test_static_load_var_list.pdparams"
):
if
os
.
path
.
exists
(
"test_static_load_var_list.pdparams"
):
os
.
remove
(
"test_static_load_var_list.pdparams"
)
os
.
remove
(
"test_static_load_var_list.pdparams"
)
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
set_place
(
self
):
def
set_place
(
self
):
return
fluid
.
CPUPlace
()
if
not
core
.
is_compiled_with_cuda
(
return
fluid
.
CPUPlace
()
if
not
core
.
is_compiled_with_cuda
(
)
else
fluid
.
CUDAPlace
(
0
)
)
else
fluid
.
CUDAPlace
(
0
)
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
test_load_from_old_interface
(
self
):
def
test_load_from_old_interface
(
self
):
seed
=
90
seed
=
90
hidden_size
=
10
hidden_size
=
10
...
@@ -898,7 +918,9 @@ class TestLoadFromOldInterface(unittest.TestCase):
...
@@ -898,7 +918,9 @@ class TestLoadFromOldInterface(unittest.TestCase):
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
#fluid.save(main_program, "./test_1")
#fluid.save(main_program, "./test_1")
fluid
.
io
.
save_persistables
(
exe
,
"test_path"
,
main_program
)
fluid
.
io
.
save_persistables
(
exe
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_path"
),
main_program
)
# set var to zero
# set var to zero
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
...
@@ -911,7 +933,8 @@ class TestLoadFromOldInterface(unittest.TestCase):
...
@@ -911,7 +933,8 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
# make sure all the paramerter or optimizer var have been set to zero
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
fluid
.
load
(
main_program
,
"test_path"
,
exe
)
fluid
.
load
(
main_program
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_path"
),
exe
)
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
...
@@ -928,11 +951,13 @@ class TestLoadFromOldInterface(unittest.TestCase):
...
@@ -928,11 +951,13 @@ class TestLoadFromOldInterface(unittest.TestCase):
var
.
desc
.
set_shape
(
new_shape
)
var
.
desc
.
set_shape
(
new_shape
)
with
self
.
assertRaises
(
RuntimeError
):
with
self
.
assertRaises
(
RuntimeError
):
fluid
.
load
(
main_program
,
"test_path"
,
exe
)
fluid
.
load
(
main_program
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_path"
),
exe
)
# check unused parameter
# check unused parameter
fluid
.
load
(
test_clone_program
,
"test_path"
,
exe
)
fluid
.
load
(
test_clone_program
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_path"
),
exe
)
def
test_load_from_old_interface_var_list
(
self
):
def
test_load_from_old_interface_var_list
(
self
):
seed
=
90
seed
=
90
...
@@ -1013,8 +1038,10 @@ class TestLoadFromOldInterface(unittest.TestCase):
...
@@ -1013,8 +1038,10 @@ class TestLoadFromOldInterface(unittest.TestCase):
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
#fluid.save(main_program, "./test_1")
#fluid.save(main_program, "./test_1")
fluid
.
io
.
save_persistables
(
exe
,
"test_static_load_var_list"
,
fluid
.
io
.
save_persistables
(
main_program
)
exe
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_static_load_var_list"
),
main_program
)
# set var to zero
# set var to zero
var_list
=
[]
var_list
=
[]
...
@@ -1030,7 +1057,10 @@ class TestLoadFromOldInterface(unittest.TestCase):
...
@@ -1030,7 +1057,10 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
# make sure all the paramerter or optimizer var have been set to zero
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
fluid
.
load
(
main_program
,
"test_static_load_var_list"
,
exe
,
var_list
)
fluid
.
load
(
main_program
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_static_load_var_list"
),
exe
,
var_list
)
var_list_names
=
[
var
.
name
for
var
in
var_list
]
var_list_names
=
[
var
.
name
for
var
in
var_list
]
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
...
@@ -1059,6 +1089,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
...
@@ -1059,6 +1089,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
init_scale
=
0.1
init_scale
=
0.1
batch_size
=
4
batch_size
=
4
batch_num
=
200
batch_num
=
200
temp_dir
=
tempfile
.
TemporaryDirectory
()
with
new_program_scope
():
with
new_program_scope
():
fluid
.
default_startup_program
().
random_seed
=
seed
fluid
.
default_startup_program
().
random_seed
=
seed
...
@@ -1125,10 +1156,10 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
...
@@ -1125,10 +1156,10 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
# make sure all the paramerter or optimizer var have been update
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
save_dir
=
os
.
path
.
join
(
temp_dir
.
name
,
"test_path"
)
#fluid.save(main_program, "./test_1")
#fluid.save(main_program, "./test_1")
fluid
.
io
.
save_persistables
(
fluid
.
io
.
save_persistables
(
exe
,
"test_path"
,
main_program
,
filename
=
"model_single"
)
exe
,
save_dir
,
main_program
,
filename
=
"model_single"
)
# set var to zero
# set var to zero
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
...
@@ -1141,7 +1172,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
...
@@ -1141,7 +1172,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
# make sure all the paramerter or optimizer var have been set to zero
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
file_model_path
=
os
.
path
.
join
(
"test_path"
,
"model_single"
)
file_model_path
=
os
.
path
.
join
(
save_dir
,
"model_single"
)
fluid
.
load
(
main_program
,
file_model_path
,
exe
,
fluid
.
load
(
main_program
,
file_model_path
,
exe
,
fluid
.
io
.
get_program_persistable_vars
(
main_program
))
fluid
.
io
.
get_program_persistable_vars
(
main_program
))
...
@@ -1190,11 +1221,16 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
...
@@ -1190,11 +1221,16 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
all_var_list
=
list
(
main_program
.
list_vars
())
all_var_list
=
list
(
main_program
.
list_vars
())
fluid
.
load
(
main_program
,
file_model_path
,
exe
,
fluid
.
load
(
main_program
,
file_model_path
,
exe
,
all_var_list
+
[
temp_var
])
all_var_list
+
[
temp_var
])
temp_dir
.
cleanup
()
class
TestProgramStateOldSave
(
unittest
.
TestCase
):
class
TestProgramStateOldSave
(
unittest
.
TestCase
):
def
setUp
(
self
):
def
setUp
(
self
):
self
.
test_dygraph
=
True
self
.
test_dygraph
=
True
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
set_place
(
self
):
def
set_place
(
self
):
return
fluid
.
CPUPlace
()
if
not
core
.
is_compiled_with_cuda
(
return
fluid
.
CPUPlace
()
if
not
core
.
is_compiled_with_cuda
(
...
@@ -1283,8 +1319,8 @@ class TestProgramStateOldSave(unittest.TestCase):
...
@@ -1283,8 +1319,8 @@ class TestProgramStateOldSave(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
# make sure all the paramerter or optimizer var have been update
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
save_dir
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_program_1"
)
fluid
.
io
.
save_persistables
(
exe
,
"test_program_1"
,
main_program
)
fluid
.
io
.
save_persistables
(
exe
,
save_dir
,
main_program
)
# set var to zero
# set var to zero
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
...
@@ -1298,7 +1334,7 @@ class TestProgramStateOldSave(unittest.TestCase):
...
@@ -1298,7 +1334,7 @@ class TestProgramStateOldSave(unittest.TestCase):
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
# case 1: load basic
# case 1: load basic
program_state
=
fluid
.
load_program_state
(
"test_program_1"
)
program_state
=
fluid
.
load_program_state
(
save_dir
)
fluid
.
set_program_state
(
main_program
,
program_state
)
fluid
.
set_program_state
(
main_program
,
program_state
)
self
.
check_in_static
(
main_program
,
base_map
)
self
.
check_in_static
(
main_program
,
base_map
)
...
@@ -1313,24 +1349,20 @@ class TestProgramStateOldSave(unittest.TestCase):
...
@@ -1313,24 +1349,20 @@ class TestProgramStateOldSave(unittest.TestCase):
else
:
else
:
raise
e
raise
e
orig_filepath
=
'./test_program_1/fc_0.w_0'
program_state
=
fluid
.
load_program_state
(
save_dir
)
symlink_filepath
=
'./test_program_1/link_fc_0.w_0'
# create a needless link file for coverage
symlink_force
(
orig_filepath
,
symlink_filepath
)
program_state
=
fluid
.
load_program_state
(
"test_program_1"
)
fluid
.
set_program_state
(
main_program
,
program_state
)
fluid
.
set_program_state
(
main_program
,
program_state
)
self
.
check_in_static
(
main_program
,
base_map
)
self
.
check_in_static
(
main_program
,
base_map
)
# case 3: load with var_list
# case 3: load with var_list
program_state
=
fluid
.
load_program_state
(
program_state
=
fluid
.
load_program_state
(
"test_program_1"
,
main_program
.
all_parameters
())
save_dir
,
main_program
.
all_parameters
())
fluid
.
set_program_state
(
main_program
,
program_state
)
fluid
.
set_program_state
(
main_program
,
program_state
)
self
.
check_in_static
(
main_program
,
base_map
)
self
.
check_in_static
(
main_program
,
base_map
)
if
self
.
test_dygraph
:
if
self
.
test_dygraph
:
# make sure `load_program_state` can be used in dynamic graph mode
# make sure `load_program_state` can be used in dynamic graph mode
with
fluid
.
dygraph
.
guard
(
place
):
with
fluid
.
dygraph
.
guard
(
place
):
load_state
=
fluid
.
load_program_state
(
"test_program_1"
)
load_state
=
fluid
.
load_program_state
(
save_dir
)
for
k
,
v
in
load_state
.
items
():
for
k
,
v
in
load_state
.
items
():
self
.
assertTrue
(
np
.
array_equal
(
base_map
[
k
],
v
))
self
.
assertTrue
(
np
.
array_equal
(
base_map
[
k
],
v
))
...
@@ -1365,6 +1397,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
...
@@ -1365,6 +1397,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
init_scale
=
0.1
init_scale
=
0.1
batch_size
=
4
batch_size
=
4
batch_num
=
200
batch_num
=
200
temp_dir
=
tempfile
.
TemporaryDirectory
()
with
new_program_scope
():
with
new_program_scope
():
fluid
.
default_startup_program
().
random_seed
=
seed
fluid
.
default_startup_program
().
random_seed
=
seed
...
@@ -1440,8 +1473,9 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
...
@@ -1440,8 +1473,9 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
save_dir
=
os
.
path
.
join
(
temp_dir
.
name
,
"test_program_2"
)
fluid
.
io
.
save_persistables
(
fluid
.
io
.
save_persistables
(
exe
,
"test_program_2"
,
main_program
,
filename
=
"model_1"
)
exe
,
save_dir
,
main_program
,
filename
=
"model_1"
)
# set var to zero
# set var to zero
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
...
@@ -1456,7 +1490,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
...
@@ -1456,7 +1490,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
#fluid.load(test_program, "./test_1", None )
#fluid.load(test_program, "./test_1", None )
program_state
=
fluid
.
load_program_state
(
program_state
=
fluid
.
load_program_state
(
os
.
path
.
join
(
"test_program_2"
,
"model_1"
),
os
.
path
.
join
(
save_dir
,
"model_1"
),
var_list
=
fluid
.
io
.
get_program_persistable_vars
(
main_program
))
var_list
=
fluid
.
io
.
get_program_persistable_vars
(
main_program
))
fluid
.
set_program_state
(
main_program
,
program_state
)
fluid
.
set_program_state
(
main_program
,
program_state
)
...
@@ -1468,21 +1502,20 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
...
@@ -1468,21 +1502,20 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
fluid
.
load_program_state
(
fluid
.
load_program_state
(
os
.
path
.
join
(
save_dir
,
"model_1"
))
os
.
path
.
join
(
"test_program_2"
,
"model_1"
))
with
self
.
assertRaises
(
TypeError
):
with
self
.
assertRaises
(
TypeError
):
fluid
.
load_program_state
(
fluid
.
load_program_state
(
os
.
path
.
join
(
"test_program_2"
,
"model_1"
),
os
.
path
.
join
(
save_dir
,
"model_1"
),
var_list
=
[
"str"
])
var_list
=
[
"str"
])
with
self
.
assertRaises
(
RuntimeError
):
with
self
.
assertRaises
(
RuntimeError
):
fluid
.
load_program_state
(
fluid
.
load_program_state
(
os
.
path
.
join
(
"test_program_2"
,
"model_1"
),
os
.
path
.
join
(
save_dir
,
"model_1"
),
var_list
=
[
var_list
=
[
main_program
.
global_block
().
create_var
(
main_program
.
global_block
().
create_var
(
name
=
"fake_var_name"
,
persistable
=
True
)
name
=
"fake_var_name"
,
persistable
=
True
)
])
])
temp_dir
.
cleanup
()
class
TestStaticSaveLoadPickle
(
unittest
.
TestCase
):
class
TestStaticSaveLoadPickle
(
unittest
.
TestCase
):
...
@@ -1511,7 +1544,8 @@ class TestStaticSaveLoadPickle(unittest.TestCase):
...
@@ -1511,7 +1544,8 @@ class TestStaticSaveLoadPickle(unittest.TestCase):
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
path
=
os
.
path
.
join
(
"test_static_save_load_pickle"
,
temp_dir
=
tempfile
.
TemporaryDirectory
()
path
=
os
.
path
.
join
(
temp_dir
.
name
,
"test_static_save_load_pickle"
,
"pickle_protocol"
)
"pickle_protocol"
)
with
self
.
assertRaises
(
ValueError
):
with
self
.
assertRaises
(
ValueError
):
...
...
python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py
浏览文件 @
2eb60ddb
...
@@ -23,11 +23,19 @@ from paddle.fluid.optimizer import SGDOptimizer
...
@@ -23,11 +23,19 @@ from paddle.fluid.optimizer import SGDOptimizer
from
paddle.fluid.tests.unittests.test_imperative_base
import
new_program_scope
from
paddle.fluid.tests.unittests.test_imperative_base
import
new_program_scope
from
paddle.fluid.tests.unittests.test_static_save_load
import
PtbModel
from
paddle.fluid.tests.unittests.test_static_save_load
import
PtbModel
import
numpy
as
np
import
numpy
as
np
import
tempfile
import
os
@
unittest
.
skipIf
(
not
core
.
supports_bfloat16
(),
@
unittest
.
skipIf
(
not
core
.
supports_bfloat16
(),
"place does not support BF16 evaluation"
)
"place does not support BF16 evaluation"
)
class
TestSaveLoadBF16
(
unittest
.
TestCase
):
class
TestSaveLoadBF16
(
unittest
.
TestCase
):
def
setUp
(
self
):
self
.
temp_dir
=
tempfile
.
TemporaryDirectory
()
def
tearDown
(
self
):
self
.
temp_dir
.
cleanup
()
def
set_place
(
self
):
def
set_place
(
self
):
return
fluid
.
CPUPlace
()
return
fluid
.
CPUPlace
()
...
@@ -108,8 +116,8 @@ class TestSaveLoadBF16(unittest.TestCase):
...
@@ -108,8 +116,8 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
# make sure all the paramerter or optimizer var have been update
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
save_dir
=
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_1"
)
fluid
.
save
(
main_program
,
"./test_1"
)
fluid
.
save
(
main_program
,
save_dir
)
# set var to zero
# set var to zero
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
...
@@ -122,7 +130,8 @@ class TestSaveLoadBF16(unittest.TestCase):
...
@@ -122,7 +130,8 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
# make sure all the paramerter or optimizer var have been set to zero
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
new_t
))
==
0
)
fluid
.
load
(
main_program
,
"./test_1.pdparams"
,
exe
)
fluid
.
load
(
main_program
,
os
.
path
.
join
(
self
.
temp_dir
.
name
,
"test_1.pdparams"
),
exe
)
for
var
in
main_program
.
list_vars
():
for
var
in
main_program
.
list_vars
():
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
if
isinstance
(
var
,
framework
.
Parameter
)
or
var
.
persistable
:
...
...
python/paddle/fluid/tests/unittests/test_static_save_load_large.py
浏览文件 @
2eb60ddb
...
@@ -23,6 +23,7 @@ from test_imperative_base import new_program_scope
...
@@ -23,6 +23,7 @@ from test_imperative_base import new_program_scope
import
numpy
as
np
import
numpy
as
np
import
pickle
import
pickle
import
os
import
os
import
tempfile
LARGE_PARAM
=
2
**
26
LARGE_PARAM
=
2
**
26
...
@@ -51,9 +52,10 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
...
@@ -51,9 +52,10 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
# make sure all the paramerter or optimizer var have been update
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
self
.
assertTrue
(
np
.
sum
(
np
.
abs
(
t
))
!=
0
)
base_map
[
var
.
name
]
=
t
base_map
[
var
.
name
]
=
t
temp_dir
=
tempfile
.
TemporaryDirectory
()
path
=
os
.
path
.
join
(
"test_static_save_load_large_param"
,
path
=
os
.
path
.
join
(
temp_dir
.
name
,
"static_save"
)
"test_static_save_load_large_param"
)
path
=
os
.
path
.
join
(
path
,
"static_save"
)
protocol
=
4
protocol
=
4
paddle
.
fluid
.
save
(
prog
,
path
,
pickle_protocol
=
protocol
)
paddle
.
fluid
.
save
(
prog
,
path
,
pickle_protocol
=
protocol
)
# set var to zero
# set var to zero
...
@@ -93,6 +95,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
...
@@ -93,6 +95,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
.
get_tensor
())
.
get_tensor
())
base_t
=
base_map
[
var
.
name
]
base_t
=
base_map
[
var
.
name
]
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
self
.
assertTrue
(
np
.
array_equal
(
new_t
,
base_t
))
temp_dir
.
cleanup
()
if
__name__
==
'__main__'
:
if
__name__
==
'__main__'
:
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录