未验证 提交 b8814777 编写于 作者: W wangxiaoning 提交者: GitHub

Fluid clean (#48841)

* add index sample fp16 support

* remove fluid APIs in distributed_strategy.py and role_maker.py

* Revert "remove fluid APIs in distributed_strategy.py and role_maker.py"

This reverts commit 223bbee990d3bf69e252fc3c0f19e3873550a264.

* remove fluid APIs in distributed_strategy.py and role_maker.py

* remove index sample op changes

* remove fluid APIs under fleet.base

* remove fluid APIs under fleet.layers.mpu

* remove fluid APIs under fleet.meta_optimizers

* fix fluid error

* fix util_factory.py

* reset fluid.io.load_inference_model API

* remove dygraph.parallel.prepare_context

* remove fluid.dygraph.StaticModelRunner API

* remove split_lod_tensor merge_lod_tensor

* remove unittests
上级 1e0f8734
...@@ -36,9 +36,6 @@ from .checkpoint import * ...@@ -36,9 +36,6 @@ from .checkpoint import *
from . import learning_rate_scheduler from . import learning_rate_scheduler
from .learning_rate_scheduler import * from .learning_rate_scheduler import *
from . import static_runner
from .static_runner import StaticModelRunner
from . import amp from . import amp
from .amp import * from .amp import *
......
...@@ -37,51 +37,11 @@ from paddle.fluid.framework import ( ...@@ -37,51 +37,11 @@ from paddle.fluid.framework import (
in_dygraph_mode, in_dygraph_mode,
) )
__all__ = ["prepare_context", "ParallelEnv", "DataParallel"] __all__ = ["ParallelEnv", "DataParallel"]
ParallelStrategy = core.ParallelStrategy ParallelStrategy = core.ParallelStrategy
@deprecated(since="2.0.0", update_to="paddle.distributed.init_parallel_env")
def prepare_context(strategy=None):
'''
:api_attr: imperative
'''
if strategy is None:
strategy = ParallelStrategy()
strategy.nranks = Env().nranks
strategy.local_rank = Env().local_rank
strategy.trainer_endpoints = Env().trainer_endpoints
strategy.current_endpoint = Env().current_endpoint
if strategy.nranks < 2:
return
assert (
framework._non_static_mode() is True
), "dygraph.prepare_context should be used with dygraph mode."
place = framework._current_expected_place()
assert (
place is not None
), "dygraph.prepare_context should be used in fluid.dygraph.guard(place) guard."
if not parallel_helper._is_parallel_ctx_initialized():
if isinstance(place, core.CUDAPlace):
parallel_helper._set_parallel_ctx(
core.NCCLParallelContext(strategy, place)
)
elif isinstance(place, core.XPUPlace):
parallel_helper._set_parallel_ctx(
core.BKCLParallelContext(strategy, place)
)
elif isinstance(place, core.NPUPlace):
parallel_helper._set_parallel_ctx(
core.HCCLParallelContext(strategy, place)
)
else:
# TODO(Yancey1989): add Gloo Parallel Context to support CPU parallel computation
assert "Only support CUDAPlace or XPUPlace or NPUPlace for now."
parallel_helper._init_parallel_ctx()
return strategy
class ParallelEnv: class ParallelEnv:
""" """
.. note:: .. note::
......
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle.jit.api import _SaveLoadConfig
from paddle.jit.translated_layer import TranslatedLayer
# NOTE: This class will be deprecated later.
# It is kept here because PaddleHub is already using this API.
class StaticModelRunner:
"""
A Dynamic graph Layer for loading inference program and related parameters,
and then performing fine-tune training or inference.
.. note::
This is a temporary API, which will be deprecated later, please use
`fluid.dygraph.jit.load` to achieve the same function.
"""
def __new__(cls, model_dir, model_filename=None, params_filename=None):
configs = _SaveLoadConfig()
if model_filename is not None:
configs.model_filename = model_filename
if params_filename is not None:
configs.params_filename = params_filename
return TranslatedLayer._construct(model_dir, configs)
...@@ -145,130 +145,6 @@ def select_input(inputs, mask): ...@@ -145,130 +145,6 @@ def select_input(inputs, mask):
return out return out
def split_lod_tensor(input, mask, level=0):
"""
This function takes in an input that contains the complete lod information,
and takes in a mask which is used to mask certain parts of the input.
The output is the true branch and the false branch with the mask applied to
the input at a certain level in the tensor. Mainly used in IfElse to split
data into two parts.
Args:
input(Variable|tuple|list|None): The input tensor that contains complete
lod information needed to construct the output.
mask(Variable|list): A bool column vector which masks the input.
level(int): The specific lod level to split.
Returns:
tuple(Variable, Variable):
The true branch of tensor as per the mask applied to input.
The false branch of tensor as per the mask applied to input.
Examples:
.. code-block:: python
import paddle.fluid as fluid
x = fluid.layers.data(name='x', shape=[1])
x.persistable = True
y = fluid.layers.data(name='y', shape=[1])
y.persistable = True
out_true, out_false = fluid.layers.split_lod_tensor(
input=x, mask=y, level=level)
"""
check_type(
input,
'input',
(Variable, list, tuple, type(None)),
'fluid.layers.split_lod_tensor',
)
check_type(mask, 'mask', (Variable, list), 'fluid.layers.split_lod_tensor')
check_type(level, 'level', int, 'fluid.layers.split_lod_tensor')
helper = LayerHelper('split_lod_tensor', **locals())
out_true = helper.create_variable_for_type_inference(dtype=input.dtype)
out_false = helper.create_variable_for_type_inference(dtype=input.dtype)
helper.append_op(
type='split_lod_tensor',
inputs={
'X': input,
'Mask': mask,
},
outputs={'OutTrue': out_true, 'OutFalse': out_false},
attrs={'level': level},
)
return out_true, out_false
def merge_lod_tensor(in_true, in_false, x, mask, level=0):
"""
**merge_lod_tensor**
This function takes in an input :math:`x`, the True branch, the False
branch and a binary :math:`mask`. Using this information, this function
merges the True and False branches of the tensor into a single tensor as
output at a certain lod level indicated by :math:`level`. Used in IfElse
to merge the output if True block and False Block.
Args:
in_true(Variable|tuple|list|None): The True branch to be merged.
in_false(Variable|tuple|list|None): The False branch to be merged.
x(Variable|tuple|list|None): The input tensor that contains complete
lod information needed to construct the output.
mask(Variable|list): A bool column vector which masks the input.
level(int): The specific lod level to merge.
Returns:
Variable: The merged output tensor.
Examples:
.. code-block:: python
import paddle.fluid as fluid
x = layers.data(
name='x', shape=[1], dtype='float32', stop_gradient=False)
y = layers.data(
name='y', shape=[1], dtype='bool', stop_gradient=False)
level = 0
out_true, out_false = layers.split_lod_tensor(
input=x, mask=y, level=level)
out = layers.merge_lod_tensor(
in_true=out_true, in_false=out_false, mask=y, x=x, level=level)
"""
helper = LayerHelper('merge_lod_tensor', **locals())
check_type(
x,
'x',
(Variable, list, tuple, type(None)),
'fluid.layers.merge_lod_tensor',
)
check_type(mask, 'mask', (Variable, list), 'fluid.layers.merge_lod_tensor')
check_type(
in_true,
'in_true',
(Variable, list, tuple, type(None)),
'fluid.layers.merge_lod_tensor',
)
check_type(
in_false,
'in_false',
(Variable, list, tuple, type(None)),
'fluid.layers.merge_lod_tensor',
)
out = helper.create_variable_for_type_inference(dtype=in_true.dtype)
helper.append_op(
type='merge_lod_tensor',
inputs={'X': x, 'Mask': mask, 'InTrue': in_true, 'InFalse': in_false},
outputs={'Out': out},
attrs={'level': level},
)
return out
@static_only @static_only
def Print( def Print(
input, input,
......
...@@ -449,8 +449,6 @@ list(REMOVE_ITEM TEST_OPS test_basic_lstm_unit_op) ...@@ -449,8 +449,6 @@ list(REMOVE_ITEM TEST_OPS test_basic_lstm_unit_op)
list(REMOVE_ITEM TEST_OPS test_fuse_all_reduce_pass) list(REMOVE_ITEM TEST_OPS test_fuse_all_reduce_pass)
list(REMOVE_ITEM TEST_OPS test_fuse_bn_act_pass) list(REMOVE_ITEM TEST_OPS test_fuse_bn_act_pass)
list(REMOVE_ITEM TEST_OPS test_fuse_bn_add_act_pass) list(REMOVE_ITEM TEST_OPS test_fuse_bn_add_act_pass)
list(REMOVE_ITEM TEST_OPS test_imperative_static_runner_mnist)
list(REMOVE_ITEM TEST_OPS test_imperative_static_runner_while)
# disable this unittest temporarily # disable this unittest temporarily
list(REMOVE_ITEM TEST_OPS test_imperative_data_loader_exception) list(REMOVE_ITEM TEST_OPS test_imperative_data_loader_exception)
...@@ -569,12 +567,6 @@ py_test_modules( ...@@ -569,12 +567,6 @@ py_test_modules(
py_test_modules(test_install_check MODULES test_install_check ENVS py_test_modules(test_install_check MODULES test_install_check ENVS
FLAGS_cudnn_deterministic=1) FLAGS_cudnn_deterministic=1)
set_tests_properties(test_install_check PROPERTIES LABELS "RUN_TYPE=DIST") set_tests_properties(test_install_check PROPERTIES LABELS "RUN_TYPE=DIST")
py_test_modules(
test_imperative_static_runner_mnist MODULES
test_imperative_static_runner_mnist ENVS FLAGS_cudnn_deterministic=1)
py_test_modules(
test_imperative_static_runner_while MODULES
test_imperative_static_runner_while ENVS FLAGS_cudnn_deterministic=1)
if((WITH_GPU) AND (CUDA_VERSION GREATER_EQUAL 11.6)) if((WITH_GPU) AND (CUDA_VERSION GREATER_EQUAL 11.6))
py_test_modules(test_fused_gemm_epilogue_op MODULES py_test_modules(test_fused_gemm_epilogue_op MODULES
...@@ -1084,7 +1076,6 @@ set_tests_properties(test_svd_op PROPERTIES TIMEOUT 80) ...@@ -1084,7 +1076,6 @@ set_tests_properties(test_svd_op PROPERTIES TIMEOUT 80)
set_tests_properties(test_einsum_op PROPERTIES TIMEOUT 120) set_tests_properties(test_einsum_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_qr_op PROPERTIES TIMEOUT 60) set_tests_properties(test_qr_op PROPERTIES TIMEOUT 60)
set_tests_properties(test_trilinear_interp_v2_op PROPERTIES TIMEOUT 120) set_tests_properties(test_trilinear_interp_v2_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_static_runner_mnist PROPERTIES TIMEOUT 120)
set_tests_properties(test_masked_select_op PROPERTIES TIMEOUT 120) set_tests_properties(test_masked_select_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_sigmoid_cross_entropy_with_logits_op set_tests_properties(test_sigmoid_cross_entropy_with_logits_op
PROPERTIES TIMEOUT 120) PROPERTIES TIMEOUT 120)
...@@ -1167,8 +1158,6 @@ if(APPLE) ...@@ -1167,8 +1158,6 @@ if(APPLE)
PROPERTIES TIMEOUT 300) PROPERTIES TIMEOUT 300)
set_tests_properties(test_multiclass_nms_op PROPERTIES TIMEOUT 300) set_tests_properties(test_multiclass_nms_op PROPERTIES TIMEOUT 300)
set_tests_properties(test_weight_decay PROPERTIES TIMEOUT 300) set_tests_properties(test_weight_decay PROPERTIES TIMEOUT 300)
set_tests_properties(test_imperative_static_runner_mnist PROPERTIES TIMEOUT
300)
endif() endif()
if((WITH_ROCM OR WITH_GPU) AND NOT WIN32) if((WITH_ROCM OR WITH_GPU) AND NOT WIN32)
......
...@@ -676,7 +676,6 @@ class TestParallelDyGraphRunnerBase: ...@@ -676,7 +676,6 @@ class TestParallelDyGraphRunnerBase:
type(self).__name__, type(self).__name__,
"begin to prepare context in dygraph with nccl2", "begin to prepare context in dygraph with nccl2",
) )
dygraph.parallel.prepare_context(strategy)
if not args.find_unused_parameters: if not args.find_unused_parameters:
model = dygraph.parallel.DataParallel( model = dygraph.parallel.DataParallel(
model, strategy, find_unused_parameters=False model, strategy, find_unused_parameters=False
......
...@@ -19,6 +19,7 @@ import numpy as np ...@@ -19,6 +19,7 @@ import numpy as np
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.core as core import paddle.fluid.core as core
import paddle.fluid.dygraph as dygraph import paddle.fluid.dygraph as dygraph
from paddle.distributed import init_parallel_env
from paddle.nn import Linear from paddle.nn import Linear
...@@ -38,9 +39,9 @@ class MLP(fluid.Layer): ...@@ -38,9 +39,9 @@ class MLP(fluid.Layer):
class TestDataParallelStateDict(unittest.TestCase): class TestDataParallelStateDict(unittest.TestCase):
def test_data_parallel_state_dict(self): def test_data_parallel_state_dict(self):
with fluid.dygraph.guard(): with fluid.dygraph.guard():
strategy = dygraph.parallel.prepare_context() init_parallel_env()
mlp = MLP() mlp = MLP()
parallel_mlp = dygraph.parallel.DataParallel(mlp, strategy) parallel_mlp = dygraph.parallel.DataParallel(mlp)
single_state = mlp.state_dict() single_state = mlp.state_dict()
parallel_state = parallel_mlp.state_dict() parallel_state = parallel_mlp.state_dict()
......
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import numpy as np
from test_imperative_base import new_program_scope
import paddle
import paddle.fluid as fluid
from paddle.fluid import core
LOADED_VAR_SUFFIX = ".load_0"
def convolutional_neural_network(img):
conv_pool_1 = fluid.nets.simple_img_conv_pool(
input=img,
filter_size=5,
num_filters=20,
pool_size=2,
pool_stride=2,
act="relu",
)
conv_pool_1 = paddle.static.nn.batch_norm(conv_pool_1)
conv_pool_2 = fluid.nets.simple_img_conv_pool(
input=conv_pool_1,
filter_size=5,
num_filters=50,
pool_size=2,
pool_stride=2,
act="relu",
)
prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax')
return prediction
def static_train_net(img, label):
prediction = convolutional_neural_network(img)
loss = paddle.nn.functional.cross_entropy(
input=prediction, label=label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
optimizer = fluid.optimizer.SGD(learning_rate=0.001)
optimizer.minimize(avg_loss)
return prediction, avg_loss
class TestImperativeStaticModelRunnerMnist(unittest.TestCase):
def setUp(self):
self.seed = 90
self.epoch_num = 1
self.batch_size = 128
self.batch_num = 50
def reader_decorator(self, reader):
def _reader_impl():
for item in reader():
image = np.array(item[0]).reshape(1, 28, 28)
label = np.array(item[1]).astype('int64').reshape(1)
yield image, label
return _reader_impl
def train_and_save_model(self):
with new_program_scope():
startup_program = fluid.default_startup_program()
main_program = fluid.default_main_program()
img = fluid.data(
name='img', shape=[None, 1, 28, 28], dtype='float32'
)
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
prediction, avg_loss = static_train_net(img, label)
place = (
fluid.CUDAPlace(0)
if core.is_compiled_with_cuda()
else fluid.CPUPlace()
)
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[img, label], place=place)
exe.run(startup_program)
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.mnist.train(), buf_size=100
),
batch_size=self.batch_size,
)
for _ in range(0, self.epoch_num):
for batch_id, data in enumerate(train_reader()):
exe.run(
main_program,
feed=feeder.feed(data),
fetch_list=[avg_loss],
)
if batch_id > self.batch_num:
break
fluid.io.save_inference_model(
self.save_dirname,
["img"],
[prediction],
exe,
model_filename=self.model_filename,
params_filename=self.params_filename,
clip_extra=False,
)
def load_and_train_dygraph(self):
place = (
fluid.CUDAPlace(0)
if core.is_compiled_with_cuda()
else fluid.CPUPlace()
)
with fluid.dygraph.guard(place):
fluid.default_startup_program().random_seed = self.seed
fluid.default_main_program().random_seed = self.seed
fluid.set_flags({'FLAGS_sort_sum_gradient': True})
mnist = fluid.dygraph.static_runner.StaticModelRunner(
model_dir=self.save_dirname,
model_filename=self.model_filename,
params_filename=self.params_filename,
)
suffix_varname_dict = mnist._program_holder_dict[
'forward'
]._suffix_varname_dict
dict_old_new = {v: k for k, v in suffix_varname_dict.items()}
dy_param_init_value = {}
for param in mnist.parameters():
dy_param_init_value[param.name] = param.numpy()
sgd = fluid.optimizer.SGD(
learning_rate=0.001, parameter_list=mnist.parameters()
)
train_reader = paddle.batch(
self.reader_decorator(paddle.dataset.mnist.train()),
batch_size=self.batch_size,
drop_last=True,
)
train_loader = fluid.io.DataLoader.from_generator(capacity=10)
train_loader.set_sample_list_generator(train_reader, places=place)
mnist.train()
for epoch in range(self.epoch_num):
for batch_id, data in enumerate(train_loader()):
img = data[0]
label = data[1]
label.stop_gradient = True
cost = mnist(img)
loss = paddle.nn.functional.cross_entropy(
cost, label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
avg_loss.backward()
sgd.minimize(avg_loss)
mnist.clear_gradients()
if batch_id >= self.batch_num:
break
dy_x_data = img.numpy()
dy_out = avg_loss.numpy()
dy_param_value = {}
for param in mnist.parameters():
dy_param_value[param.name] = param.numpy()
return (
dy_x_data,
dy_out,
dy_param_init_value,
dy_param_value,
dict_old_new,
)
def load_and_train_static(self):
with new_program_scope():
fluid.default_startup_program().random_seed = self.seed
fluid.default_main_program().random_seed = self.seed
img = fluid.data(
name='img', shape=[None, 1, 28, 28], dtype='float32'
)
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
prediction, avg_loss = static_train_net(img, label)
place = (
fluid.CUDAPlace(0)
if core.is_compiled_with_cuda()
else fluid.CPUPlace()
)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
fluid.io.load_params(
exe,
self.save_dirname,
main_program=fluid.default_main_program(),
filename=self.params_filename,
)
static_param_init_value = {}
static_param_name_list = []
for param in fluid.default_main_program().all_parameters():
static_param_name_list.append(param.name)
static_param_init_value[param.name] = fluid.executor._fetch_var(
param.name
)
train_reader = paddle.batch(
self.reader_decorator(paddle.dataset.mnist.train()),
batch_size=self.batch_size,
drop_last=True,
)
for epoch in range(self.epoch_num):
for batch_id, data in enumerate(train_reader()):
static_x_data = np.array([x[0] for x in data])
y_data = np.array([x[1] for x in data]).reshape(
[self.batch_size, 1]
)
fetch_list = [avg_loss.name]
fetch_list.extend(static_param_name_list)
out = exe.run(
fluid.default_main_program(),
feed={"img": static_x_data, "label": y_data},
fetch_list=fetch_list,
)
if batch_id >= self.batch_num:
break
static_param_value = {}
static_out = out[0]
for i in range(1, len(out)):
static_param_value[static_param_name_list[i - 1]] = out[i]
return (
static_x_data,
static_out,
static_param_init_value,
static_param_value,
)
def load_and_infer_dygraph(self):
place = (
fluid.CUDAPlace(0)
if core.is_compiled_with_cuda()
else fluid.CPUPlace()
)
with fluid.dygraph.guard(place):
fluid.default_main_program().random_seed = self.seed
mnist = fluid.dygraph.static_runner.StaticModelRunner(
model_dir=self.save_dirname, model_filename=self.model_filename
)
train_reader = paddle.batch(
self.reader_decorator(paddle.dataset.mnist.test()),
batch_size=self.batch_size,
drop_last=True,
)
train_loader = fluid.io.DataLoader.from_generator(capacity=10)
train_loader.set_sample_list_generator(train_reader, places=place)
mnist.eval()
for batch_id, data in enumerate(train_loader()):
img = data[0]
cost = mnist(img)
if batch_id >= 1:
break
dy_x_data = img.numpy()
dy_out = cost.numpy()
return dy_x_data, dy_out
def load_and_infer_static(self):
with new_program_scope():
place = (
fluid.CUDAPlace(0)
if core.is_compiled_with_cuda()
else fluid.CPUPlace()
)
exe = fluid.Executor(place)
[
infer_program,
feed_target_names,
fetch_targets,
] = fluid.io.load_inference_model(self.save_dirname, exe)
infer_program.random_seed = self.seed
train_reader = paddle.batch(
self.reader_decorator(paddle.dataset.mnist.test()),
batch_size=self.batch_size,
drop_last=True,
)
for batch_id, data in enumerate(train_reader()):
static_x_data = np.array([x[0] for x in data])
out = exe.run(
infer_program,
feed={feed_target_names[0]: static_x_data},
fetch_list=fetch_targets,
)
if batch_id >= 1:
break
static_param_value = {}
static_out = out[0]
return static_x_data, static_out
def test_mnist_train_no_params_filename(self):
self.save_dirname = "mnist.inference.model.noname"
self.model_filename = None
self.params_filename = None
# Phase 1. run and save static model
self.train_and_save_model()
# Phase 2. load model & train dygraph
(
dy_x_data,
dy_out,
dy_param_init_value,
dy_param_value,
dict_old_new_init,
) = self.load_and_train_dygraph()
(
static_x_data,
static_out,
static_param_init_value,
static_param_value,
) = self.load_and_train_static()
# Phase 3. compare
np.testing.assert_array_equal(static_x_data, dy_x_data)
for key, value in static_param_init_value.items():
key = dict_old_new_init[key]
np.testing.assert_array_equal(value, dy_param_init_value[key])
# np.testing.assert_array_almost_equal(static_out, dy_out)
np.testing.assert_allclose(static_out, dy_out, rtol=1e-05, atol=1e-4)
for key, value in static_param_value.items():
key = dict_old_new_init[key]
np.testing.assert_allclose(
value, dy_param_value[key], rtol=1e-05, atol=1e-4
)
def test_mnist_train_with_params_filename(self):
self.save_dirname = "mnist.inference.model"
self.model_filename = "mnist.model"
self.params_filename = "mnist.params"
# Phase 1. run and save static model
self.train_and_save_model()
# Phase 2. load model & train dygraph
(
dy_x_data,
dy_out,
dy_param_init_value,
dy_param_value,
dict_old_new_init,
) = self.load_and_train_dygraph()
(
static_x_data,
static_out,
static_param_init_value,
static_param_value,
) = self.load_and_train_static()
# Phase 3. compare
np.testing.assert_array_equal(static_x_data, dy_x_data)
for key, value in static_param_init_value.items():
key = dict_old_new_init[key]
np.testing.assert_array_equal(value, dy_param_init_value[key])
# np.testing.assert_array_almost_equal(static_out, dy_out)
np.testing.assert_allclose(static_out, dy_out, rtol=1e-05, atol=1e-4)
for key, value in static_param_value.items():
key = dict_old_new_init[key]
np.testing.assert_allclose(
value, dy_param_value[key], rtol=1e-05, atol=1e-4
)
def test_mnist_infer_no_params_filename(self):
self.save_dirname = "mnist.inference.model.noname"
self.model_filename = None
self.params_filename = None
# Phase 1. run and save static model
self.train_and_save_model()
# Phase 2. load model & train dygraph
dy_x_data, dy_out = self.load_and_infer_dygraph()
static_x_data, static_out = self.load_and_infer_static()
# Phase 3. compare
np.testing.assert_array_equal(static_x_data, dy_x_data)
np.testing.assert_array_almost_equal(static_out, dy_out)
np.testing.assert_allclose(static_out, dy_out, rtol=1e-05, atol=1e-4)
if __name__ == '__main__':
unittest.main()
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import numpy as np
from jit_load_rename_var import rename_var_with_generator
from test_imperative_base import new_program_scope
import paddle
import paddle.fluid as fluid
from paddle.fluid import core, unique_name
LOADED_VAR_SUFFIX = ".load_0"
paddle.enable_static()
def while_softmax_regression(img):
def cond(i, times, pred):
return i < times
def body(i, times, pred):
pred = fluid.layers.fc(input=pred, size=10, act='softmax')
i = i + 1
return [i, times, pred]
i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0)
times = fluid.layers.fill_constant(shape=[1], dtype='int64', value=5)
pred = fluid.layers.fc(input=img, size=10, act='softmax')
i, times, pred = paddle.static.nn.while_loop(
cond=cond, body=body, loop_vars=[i, times, pred]
)
return pred
class TestImperativeStaticModelRunnerWhile(unittest.TestCase):
def setUp(self):
self.seed = 90
self.batch_size = 32
self.batch_num = 50
self.save_dirname = "while.inference.model"
self.model_filename = None
self.params_filename = None
def _random_batch_reader(self):
def _get_random_images_and_labels(image_shape, label_shape):
image = np.random.random(size=image_shape).astype('float32')
label = np.random.random(size=label_shape).astype('int64')
return image, label
def __reader__():
for _ in range(self.batch_num):
batch_image, batch_label = _get_random_images_and_labels(
[self.batch_size, 784], [self.batch_size, 1]
)
yield batch_image, batch_label
return __reader__
def train_and_save_model(self):
startup_program = fluid.default_startup_program()
main_program = fluid.default_main_program()
img = fluid.data(name='img', shape=[None, 784], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
pred = while_softmax_regression(img)
loss = paddle.nn.functional.cross_entropy(
input=pred, label=label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
optimizer = fluid.optimizer.SGD(learning_rate=0.001)
optimizer.minimize(avg_loss)
place = (
fluid.CUDAPlace(0)
if core.is_compiled_with_cuda()
else fluid.CPUPlace()
)
exe = fluid.Executor(place)
exe.run(startup_program)
loader = fluid.io.DataLoader.from_generator(
feed_list=[img, label], capacity=5, iterable=True
)
loader.set_batch_generator(self._random_batch_reader(), places=place)
for data in loader():
exe.run(main_program, feed=data, fetch_list=[avg_loss])
fluid.io.save_inference_model(
self.save_dirname,
["img"],
[pred],
exe,
model_filename=self.model_filename,
params_filename=self.params_filename,
clip_extra=False,
)
def load_and_train_dygraph(self):
place = (
fluid.CUDAPlace(0)
if core.is_compiled_with_cuda()
else fluid.CPUPlace()
)
with fluid.dygraph.guard(place):
fluid.default_startup_program().random_seed = self.seed
fluid.default_main_program().random_seed = self.seed
np.random.seed(self.seed)
fluid.set_flags({'FLAGS_sort_sum_gradient': True})
while_net = fluid.dygraph.static_runner.StaticModelRunner(
self.save_dirname
)
dy_param_init_value = {}
for param in while_net.parameters():
dy_param_init_value[param.name] = param.numpy()
sgd = fluid.optimizer.SGD(
learning_rate=0.001, parameter_list=while_net.parameters()
)
train_loader = fluid.io.DataLoader.from_generator(capacity=10)
train_loader.set_batch_generator(
self._random_batch_reader(), places=place
)
while_net.train()
for data in train_loader():
img = data[0]
label = data[1]
label.stop_gradient = True
cost = while_net(img)
loss = paddle.nn.functional.cross_entropy(
cost, label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
avg_loss.backward()
sgd.minimize(avg_loss)
while_net.clear_gradients()
dy_out = avg_loss.numpy()
dy_param_value = {}
for param in while_net.parameters():
dy_param_value[param.name] = param.numpy()
return dy_out, dy_param_init_value, dy_param_value
def load_and_train_static(self):
with new_program_scope():
fluid.default_startup_program().random_seed = self.seed
fluid.default_main_program().random_seed = self.seed
np.random.seed(self.seed)
img = fluid.data(name='img', shape=[None, 784], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
pred = while_softmax_regression(img)
loss = paddle.nn.functional.cross_entropy(
input=pred, label=label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
optimizer = fluid.optimizer.SGD(learning_rate=0.001)
optimizer.minimize(avg_loss)
place = (
fluid.CUDAPlace(0)
if core.is_compiled_with_cuda()
else fluid.CPUPlace()
)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
fluid.io.load_params(
exe,
self.save_dirname,
main_program=fluid.default_main_program(),
filename=self.params_filename,
)
static_param_init_value = {}
static_param_name_list = []
for param in fluid.default_main_program().all_parameters():
static_param_name_list.append(param.name)
static_param_init_value[param.name] = fluid.executor._fetch_var(
param.name
)
loader = fluid.io.DataLoader.from_generator(
feed_list=[img, label], capacity=5, iterable=True
)
loader.set_batch_generator(
self._random_batch_reader(), places=place
)
for data in loader():
fetch_list = [avg_loss.name]
fetch_list.extend(static_param_name_list)
out = exe.run(
fluid.default_main_program(),
feed=data,
fetch_list=[avg_loss],
)
static_param_value = {}
static_out = out[0]
for i in range(1, len(out)):
static_param_value[static_param_name_list[i - 1]] = out[i]
return static_out, static_param_init_value, static_param_value
def test_while_no_params_filename(self):
# Phase 1. run and save static model
self.train_and_save_model()
# # Phase 2. load model & train dygraph
with unique_name.guard():
(
dy_out,
dy_param_init_value,
dy_param_value,
) = self.load_and_train_dygraph()
with unique_name.guard():
(
static_out,
static_param_init_value,
static_param_value,
) = self.load_and_train_static()
# Phase 3. compare
with unique_name.guard():
dict_old_new_init = rename_var_with_generator(
static_param_init_value.keys()
)
for key, value in static_param_init_value.items():
key = dict_old_new_init[key]
np.testing.assert_array_equal(value, dy_param_init_value[key])
np.testing.assert_allclose(static_out, dy_out, rtol=1e-05)
for key, value in static_param_value.items():
key += LOADED_VAR_SUFFIX
np.testing.assert_allclose(
value, dy_param_value[key], rtol=1e-05, atol=1e-05
)
if __name__ == '__main__':
unittest.main()
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import numpy as np
import paddle
import paddle.fluid.core as core
import paddle.fluid.layers as layers
from paddle.fluid import Program, program_guard
from paddle.fluid.backward import append_backward
from paddle.fluid.executor import Executor
from paddle.fluid.layer_helper import LayerHelper
from paddle.fluid.layers.control_flow import merge_lod_tensor, split_lod_tensor
class TestCPULoDTensorArrayOps(unittest.TestCase):
def place(self):
return core.CPUPlace()
def test_split_and_merge_lod_tensor_no_lod(self):
tensor = core.LoDTensor()
tensor.set(np.arange(10).reshape(10, 1).astype('int32'), self.place())
mask_np = np.array([0, 0, 1, 1, 1, 1, 0, 0, 0, 0]).astype('bool')
mask_np = np.expand_dims(mask_np, axis=1)
mask = core.LoDTensor()
mask.set(mask_np, self.place())
expect_true_tensor = np.array([2, 3, 4, 5]).astype('int32')
expect_true_tensor = np.expand_dims(expect_true_tensor, axis=1)
expect_true = core.LoDTensor()
expect_true.set(expect_true_tensor, self.place())
expect_false_tensor = np.array([0, 1, 6, 7, 8, 9]).astype('int32')
expect_false_tensor = np.expand_dims(expect_false_tensor, axis=1)
expect_false = core.LoDTensor()
expect_false.set(expect_false_tensor, self.place())
self.main(
tensor=tensor,
mask=mask,
expect_true=expect_true,
expect_false=expect_false,
expect_out=tensor,
)
def split_and_merge_lod_tensor_level_0(self, use_merge_lod_infer=False):
tensor = core.LoDTensor()
tensor.set(np.arange(10).reshape(10, 1).astype('int32'), self.place())
tensor.set_recursive_sequence_lengths([[3, 6, 1]])
mask_np = np.array([0, 1, 0]).astype('bool')
mask_np = np.expand_dims(mask_np, axis=1)
mask = core.LoDTensor()
mask.set(mask_np, self.place())
expect_true_tensor = np.array([3, 4, 5, 6, 7, 8]).astype('int32')
expect_true_tensor = np.expand_dims(expect_true_tensor, axis=1)
expect_true = core.LoDTensor()
expect_true.set(expect_true_tensor, self.place())
expect_true.set_recursive_sequence_lengths([[6]])
expect_false_tensor = np.array([0, 1, 2, 9]).astype('int32')
expect_false_tensor = np.expand_dims(expect_false_tensor, axis=1)
expect_false_lod = [[3, 1]]
expect_false = core.LoDTensor()
expect_false.set(expect_false_tensor, self.place())
expect_false.set_recursive_sequence_lengths(expect_false_lod)
self.main(
tensor=tensor,
mask=mask,
expect_true=expect_true,
expect_false=expect_false,
expect_out=tensor,
use_merge_lod_infer=use_merge_lod_infer,
)
def test_split_and_merge_lod_tensor_1(self):
self.split_and_merge_lod_tensor_level_0()
def test_split_and_merge_lod_tensor_2(self):
self.split_and_merge_lod_tensor_level_0(True)
def main(
self,
tensor,
mask,
expect_true,
expect_false,
expect_out,
level=0,
use_merge_lod_infer=False,
):
place = self.place()
program = Program()
with program_guard(program):
x = layers.data(name='x', shape=[1])
x.persistable = True
y = layers.data(name='y', shape=[1])
y.persistable = True
out_true, out_false = split_lod_tensor(input=x, mask=y, level=level)
out_true.persistable = True
out_false.persistable = True
if use_merge_lod_infer:
input_dict = {
'X': x,
'Mask': mask,
'InTrue': out_true,
'InFalse': out_false,
'level': level,
}
helper = LayerHelper('merge_lod_tensor_infer')
out = helper.create_variable_for_type_inference(
dtype=out_true.dtype
)
helper.append_op(
type='merge_lod_tensor_infer',
inputs={
'X': x,
'Mask': y,
'InTrue': out_true,
'InFalse': out_false,
},
outputs={'Out': out},
attrs={'level': level},
)
out.persistable = True
else:
out = merge_lod_tensor(
in_true=out_true,
in_false=out_false,
mask=y,
x=x,
level=level,
)
out.persistable = True
exe = Executor(place)
scope = core.Scope()
exe.run(
program,
feed={'x': tensor, 'y': mask},
scope=scope,
return_numpy=False,
)
var_true = scope.find_var(out_true.name).get_tensor()
var_false = scope.find_var(out_false.name).get_tensor()
var_out = scope.find_var(out.name).get_tensor()
if not use_merge_lod_infer:
self.check_tensor_same(var_true, expect_true)
self.check_tensor_same(var_false, expect_false)
self.check_tensor_same(var_out, expect_out)
def check_tensor_same(self, actual, expect):
np.testing.assert_allclose(
np.array(actual), np.array(expect), rtol=1e-05
)
self.assertEqual(
actual.recursive_sequence_lengths(),
expect.recursive_sequence_lengths(),
)
class TestCPUSplitMergeLoDTensorGrad(unittest.TestCase):
def test_grad(self):
place = core.CPUPlace()
program = Program()
with program_guard(program):
x = layers.data(
name='x', shape=[1], dtype='float32', stop_gradient=False
)
y = layers.data(
name='y', shape=[1], dtype='bool', stop_gradient=False
)
level = 0
out_true, out_false = split_lod_tensor(input=x, mask=y, level=level)
out = merge_lod_tensor(
in_true=out_true, in_false=out_false, mask=y, x=x, level=level
)
mean = paddle.mean(out)
append_backward(mean)
tensor = core.LoDTensor()
tensor.set(np.arange(10).reshape(10, 1).astype('float32'), place)
tensor.set_recursive_sequence_lengths([[3, 6, 1]])
mask_np = np.array([0, 1, 0]).astype('bool')
mask_np = np.expand_dims(mask_np, axis=1)
mask = core.LoDTensor()
mask.set(mask_np, place)
exe = Executor(place)
scope = core.Scope()
g_vars = program.global_block().var(x.name + "@GRAD")
g_out = [
item.sum()
for item in map(
np.array,
exe.run(
program,
feed={'x': tensor, 'y': mask},
fetch_list=[g_vars],
scope=scope,
return_numpy=False,
),
)
]
g_out_sum = np.array(g_out).sum()
self.assertAlmostEqual(1.0, g_out_sum, delta=0.1)
class TestMergeLodTensorOpError(unittest.TestCase):
def test_errors(self):
with program_guard(Program(), Program()):
input_data = layers.data(
name='x', shape=[1], dtype='float32', stop_gradient=False
)
y = layers.data(
name='y', shape=[1], dtype='bool', stop_gradient=False
)
x_true = layers.data(
name='x_true', shape=[1], dtype='float32', stop_gradient=False
)
x_false = layers.data(
name='x_false', shape=[1], dtype='float32', stop_gradient=False
)
level = 0
def test_x():
out = merge_lod_tensor(
int_true=x_true,
in_false=x_false,
x=set(),
mask=y,
level=level,
)
self.assertRaises(TypeError, test_x)
def test_mask():
out = merge_lod_tensor(
int_true=x_true,
in_false=x_false,
x=input_data,
mask=set(),
level=level,
)
self.assertRaises(TypeError, test_mask)
def test_xtrue():
out = merge_lod_tensor(
int_true=set(),
in_false=x_false,
x=input_data,
mask=y,
level=level,
)
self.assertRaises(TypeError, test_xtrue)
def test_xfalse():
out = merge_lod_tensor(
int_true=x_true,
in_false=set(),
x=input_data,
mask=y,
level=level,
)
self.assertRaises(TypeError, test_xfalse)
class TestSplitLodTensorWithError(unittest.TestCase):
def test_error(self):
main_program = Program()
startup_program = Program()
with program_guard(main_program, startup_program):
x = layers.data(
name='x', shape=[1], dtype='float32', stop_gradient=False
)
y = layers.data(
name='y', shape=[1], dtype='bool', stop_gradient=False
)
level = 0
with self.assertRaises(TypeError):
split_lod_tensor(input=set(), mask=y, level=level)
with self.assertRaises(TypeError):
split_lod_tensor(input=x, mask=set(), level=level)
with self.assertRaises(TypeError):
split_lod_tensor(input=x, mask=set(), level=None)
if __name__ == '__main__':
unittest.main()
...@@ -72,7 +72,6 @@ HIGH_PARALLEL_JOB_NEW = [ ...@@ -72,7 +72,6 @@ HIGH_PARALLEL_JOB_NEW = [
'test_fleet_rolemaker_init', 'test_fleet_rolemaker_init',
'test_pybind_interface', 'test_pybind_interface',
'test_io_save_load', 'test_io_save_load',
'test_split_and_merge_lod_tensor_op',
'test_fusion_lstm_int8_mkldnn_op', 'test_fusion_lstm_int8_mkldnn_op',
'test_benchmark', 'test_benchmark',
'test_protobuf', 'test_protobuf',
...@@ -1164,7 +1163,6 @@ FOURTH_HIGH_PARALLEL_JOB_NEW = [ ...@@ -1164,7 +1163,6 @@ FOURTH_HIGH_PARALLEL_JOB_NEW = [
'test_proximal_gd_op', 'test_proximal_gd_op',
'test_mul_nn_grad', 'test_mul_nn_grad',
'test_full_like_op', 'test_full_like_op',
'test_imperative_static_runner_while',
'trt_instance_norm_test', 'trt_instance_norm_test',
'test_elementwise_mod_op', 'test_elementwise_mod_op',
'test_grad_clip_minimize', 'test_grad_clip_minimize',
...@@ -1245,7 +1243,6 @@ FOURTH_HIGH_PARALLEL_JOB_NEW = [ ...@@ -1245,7 +1243,6 @@ FOURTH_HIGH_PARALLEL_JOB_NEW = [
'test_adam_op', 'test_adam_op',
'test_elementwise_floordiv_op', 'test_elementwise_floordiv_op',
'test_diagonal_op', 'test_diagonal_op',
'test_imperative_static_runner_mnist',
'test_nearest_interp_op', 'test_nearest_interp_op',
'test_diag_embed', 'test_diag_embed',
'test_merge_selectedrows_op', 'test_merge_selectedrows_op',
...@@ -1714,7 +1711,6 @@ CPU_PARALLEL_JOB = [ ...@@ -1714,7 +1711,6 @@ CPU_PARALLEL_JOB = [
'test_static_shape_inferrence_for_shape_tensor', 'test_static_shape_inferrence_for_shape_tensor',
'test_static_analysis', 'test_static_analysis',
'test_squared_mat_sub_fuse_pass', 'test_squared_mat_sub_fuse_pass',
'test_split_and_merge_lod_tensor_op',
'test_spawn_and_init_parallel_env', 'test_spawn_and_init_parallel_env',
'test_slice_var', 'test_slice_var',
'test_skip_layernorm_fuse_pass', 'test_skip_layernorm_fuse_pass',
...@@ -2370,7 +2366,6 @@ TETRAD_PARALLEL_JOB = [ ...@@ -2370,7 +2366,6 @@ TETRAD_PARALLEL_JOB = [
'test_scatter_op', 'test_scatter_op',
'test_parallel_executor_pg', 'test_parallel_executor_pg',
'test_mix_precision_all_reduce_fuse', 'test_mix_precision_all_reduce_fuse',
'test_imperative_static_runner_mnist',
'test_tensorrt_engine_op', 'test_tensorrt_engine_op',
'test_zeropad2d', 'test_zeropad2d',
'test_isclose_op', 'test_isclose_op',
...@@ -2811,7 +2806,6 @@ TWO_PARALLEL_JOB = [ ...@@ -2811,7 +2806,6 @@ TWO_PARALLEL_JOB = [
'test_lstm_op', 'test_lstm_op',
'test_margin_rank_loss_op', 'test_margin_rank_loss_op',
'test_index_sample_op', 'test_index_sample_op',
'test_imperative_static_runner_while',
'test_imperative_save_load', 'test_imperative_save_load',
'test_imperative_ptb_rnn_sorted_gradient', 'test_imperative_ptb_rnn_sorted_gradient',
'test_mul_op', 'test_mul_op',
......
...@@ -478,7 +478,6 @@ STATIC_MODE_TESTING_LIST = [ ...@@ -478,7 +478,6 @@ STATIC_MODE_TESTING_LIST = [
'test_smooth_l1_loss_op', 'test_smooth_l1_loss_op',
'test_softmax_with_cross_entropy_op', 'test_softmax_with_cross_entropy_op',
'test_spectral_norm_op', 'test_spectral_norm_op',
'test_split_and_merge_lod_tensor_op',
'test_split_ids_op', 'test_split_ids_op',
'test_split_op', 'test_split_op',
'test_split_mkldnn_op', 'test_split_mkldnn_op',
...@@ -535,8 +534,6 @@ STATIC_MODE_TESTING_LIST = [ ...@@ -535,8 +534,6 @@ STATIC_MODE_TESTING_LIST = [
'test_imperative_mnist_sorted_gradient', 'test_imperative_mnist_sorted_gradient',
'test_imperative_se_resnext', 'test_imperative_se_resnext',
'test_imperative_ocr_attention_model', 'test_imperative_ocr_attention_model',
'test_imperative_static_runner_mnist',
'test_imperative_static_runner_while',
'test_recv_save_op', 'test_recv_save_op',
'test_transpiler_ops', 'test_transpiler_ops',
'test_communicator_sync', 'test_communicator_sync',
......
...@@ -39,7 +39,6 @@ disable_wingpu_test="^test_model$|\ ...@@ -39,7 +39,6 @@ disable_wingpu_test="^test_model$|\
^test_reader_reset$|\ ^test_reader_reset$|\
^test_imperative_se_resnext$|\ ^test_imperative_se_resnext$|\
^test_sync_batch_norm_op$|\ ^test_sync_batch_norm_op$|\
^test_imperative_static_runner_while$|\
^test_dataloader_keep_order$|\ ^test_dataloader_keep_order$|\
^test_dataloader_unkeep_order$|\ ^test_dataloader_unkeep_order$|\
^test_multiprocess_dataloader_iterable_dataset_static$|\ ^test_multiprocess_dataloader_iterable_dataset_static$|\
...@@ -158,7 +157,6 @@ disable_win_inference_test="^trt_quant_int8_yolov3_r50_test$|\ ...@@ -158,7 +157,6 @@ disable_win_inference_test="^trt_quant_int8_yolov3_r50_test$|\
^test_decoupled_py_reader$|\ ^test_decoupled_py_reader$|\
^test_generator_dataloader$|\ ^test_generator_dataloader$|\
^test_py_reader_using_executor$|\ ^test_py_reader_using_executor$|\
^test_imperative_static_runner_while$|\
^test_dataloader_keep_order$|\ ^test_dataloader_keep_order$|\
^test_dataloader_unkeep_order$|\ ^test_dataloader_unkeep_order$|\
^test_sync_batch_norm_op$|\ ^test_sync_batch_norm_op$|\
...@@ -195,7 +193,6 @@ disable_wincpu_test="^jit_kernel_test$|\ ...@@ -195,7 +193,6 @@ disable_wincpu_test="^jit_kernel_test$|\
^test_imperative_resnet$|\ ^test_imperative_resnet$|\
^test_imperative_resnet_sorted_gradient$|\ ^test_imperative_resnet_sorted_gradient$|\
^test_imperative_se_resnext$|\ ^test_imperative_se_resnext$|\
^test_imperative_static_runner_mnist$|\
^test_bmn$|\ ^test_bmn$|\
^test_mobile_net$|\ ^test_mobile_net$|\
^test_resnet_v2$|\ ^test_resnet_v2$|\
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册