Skip to content

P
Paddle

机器未来 / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
提交 51cb918a 编写于 作者: L Leo Chen 提交者： hong
浏览文件
操作

update layers used in transformer dygraph model, test=develop (#22051) 

* update layers, test=develop

* update layers for resnet, test=develop

* fix is_test attr, test=develop

* update cycle_gan, test=develop

* update reinforcement_learning, test=develop

* update ocr, test=develop

* fix bug, test=develop
上级 5b2e98aa
显示空白变更内容
内联 并排
Showing with 285 addition and 128 deletion
python/paddle/fluid/dygraph/nn.py
浏览文件 @ 51cb918a
......@@ -19,7 +19,7 @@ from .. import core
from ..layers import utils
from ..dygraph import dygraph_utils
from . import layers
from ..framework import Variable, in_dygraph_mode, OpProtoHolder, Parameter
from ..framework import Variable, in_dygraph_mode, OpProtoHolder, Parameter, _dygraph_tracer, _varbase_creator
from ..param_attr import ParamAttr
from ..initializer import Normal, Constant, NumpyArrayInitializer
from .. import unique_name
......@@ -1134,8 +1134,35 @@ class BatchNorm(layers.Layer):
# mean and mean_out share the same memory
mean_out = self._mean
# variance and variance out share the same memory
variance_out = self._variance
attrs = {
"momentum": self._momentum,
"epsilon": self._epsilon,
"is_test": self._is_test,
"data_layout": self._data_layout,
"use_mkldnn": False,
"fuse_with_relu": self._fuse_with_relu,
"use_global_stats": self._use_global_stats,
"trainable_statistics": self._trainable_statistics
}
inputs = {
"X": [input],
"Scale": [self.weight],
"Bias": [self.bias],
"Mean": [self._mean],
"Variance": [self._variance]
}
if in_dygraph_mode():
attrs['is_test'] = not _dygraph_tracer()._train_mode
saved_mean = _varbase_creator(dtype=self._dtype)
saved_variance = _varbase_creator(dtype=self._dtype)
batch_norm_out = _varbase_creator(dtype=self._dtype)
batch_norm_out.stop_gradient = False
# inplace is not supported currently
else:
saved_mean = self._helper.create_variable_for_type_inference(
dtype=self._dtype, stop_gradient=True)
saved_variance = self._helper.create_variable_for_type_inference(
......@@ -1143,32 +1170,21 @@ class BatchNorm(layers.Layer):
batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference(
self._dtype)
outputs = {
"Y": [batch_norm_out],
"MeanOut": [mean_out],
"VarianceOut": [variance_out],
"SavedMean": [saved_mean],
"SavedVariance": [saved_variance]
}
if in_dygraph_mode():
outs = core.ops.batch_norm(inputs, attrs, outputs)
return dygraph_utils._append_activation_in_dygraph(
batch_norm_out, act=self._act)
self._helper.append_op(
type="batch_norm",
inputs={
"X": input,
"Scale": self.weight,
"Bias": self.bias,
"Mean": self._mean,
"Variance": self._variance
},
outputs={
"Y": batch_norm_out,
"MeanOut": mean_out,
"VarianceOut": variance_out,
"SavedMean": saved_mean,
"SavedVariance": saved_variance
},
attrs={
"momentum": self._momentum,
"epsilon": self._epsilon,
"is_test": self._is_test,
"data_layout": self._data_layout,
"use_mkldnn": False,
"fuse_with_relu": self._fuse_with_relu,
"use_global_stats": self._use_global_stats,
"trainable_statistics": self._trainable_statistics
})
type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs)
# Currently, we don't support inplace in dygraph mode
return self._helper.append_activation(batch_norm_out, self._act)
......@@ -1454,11 +1470,23 @@ class LayerNorm(layers.Layer):
', expected input with shape [*, ' + str_normalized_shape[
1:] + ', but got input shape ' + str(input_shape))
inputs = dict()
inputs['X'] = input
inputs['X'] = [input]
if self._scale:
inputs['Scale'] = self.weight
inputs['Scale'] = [self.weight]
if self._shift:
inputs['Bias'] = self.bias
inputs['Bias'] = [self.bias]
attrs = {
"epsilon": self._epsilon,
"begin_norm_axis": self._begin_norm_axis
}
if in_dygraph_mode():
outs = core.ops.layer_norm(inputs, attrs)
pre_act = outs['Y'][0]
return dygraph_utils._append_activation_in_dygraph(
pre_act, act=self._act)
# create output
mean_out = self._helper.create_variable_for_type_inference(
dtype=self._dtype, stop_gradient=True)
......@@ -1623,9 +1651,22 @@ class GRUUnit(layers.Layer):
attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=True)
def forward(self, input, hidden):
inputs = {'Input': input, 'HiddenPrev': hidden, 'Weight': self.weight}
inputs = {
'Input': [input],
'HiddenPrev': [hidden],
'Weight': [self.weight]
}
if self.bias:
inputs['Bias'] = self.bias
inputs['Bias'] = [self.bias]
attrs = {
'activation': self.activation,
'gate_activation': self.gate_activation,
}
if in_dygraph_mode():
outs = core.ops.gru_unit(inputs, attrs)
return outs['Hidden'][0], outs['ResetHiddenPrev'][0], outs['Gate'][
0]
gate = self._helper.create_variable_for_type_inference(self._dtype)
reset_hidden_pre = self._helper.create_variable_for_type_inference(
......@@ -2277,21 +2318,32 @@ class Conv2DTranspose(layers.Layer):
is_bias=True)
def forward(self, input):
pre_bias = self._helper.create_variable_for_type_inference(
dtype=input.dtype)
self._helper.append_op(
type=self._op_type,
inputs={'Input': [input],
'Filter': [self.weight]},
outputs={'Output': pre_bias},
attrs={
inputs = {'Input': [input], 'Filter': [self.weight]}
attrs = {
'output_size': self._output_size,
'strides': self._stride,
'paddings': self._padding,
'dilations': self._dilation,
'groups': self._groups,
'use_cudnn': self._use_cudnn
})
}
if in_dygraph_mode():
op = getattr(core.ops, self._op_type)
outs = op(inputs, attrs)
pre_bias = outs['Output'][0]
pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
1)
return dygraph_utils._append_activation_in_dygraph(
pre_act, act=self._act)
pre_bias = self._helper.create_variable_for_type_inference(
dtype=input.dtype)
self._helper.append_op(
type=self._op_type,
inputs=inputs,
outputs={'Output': pre_bias},
attrs=attrs)
if self.bias is not None:
pre_act = self._helper.create_variable_for_type_inference(
......
python/paddle/fluid/layers/nn.py
浏览文件 @ 51cb918a
......@@ -1118,6 +1118,13 @@ def softmax(input, use_cudnn=False, name=None, axis=-1):
fetch_list=[result[0]])
print(output)
"""
inputs = {"X": [input]}
attrs = {"axis": axis, "use_cudnn": use_cudnn}
if in_dygraph_mode():
outs = core.ops.softmax(inputs, attrs)
return outs['Out'][0]
helper = LayerHelper('softmax', **locals())
check_type_and_dtype(input, 'input', Variable,
['float16', 'float32', 'float64'], 'softmax')
......@@ -1128,8 +1135,7 @@ def softmax(input, use_cudnn=False, name=None, axis=-1):
type="softmax",
inputs={"X": input},
outputs={"Out": softmax_out},
attrs={"axis": axis,
"use_cudnn": use_cudnn})
attrs=attrs)
return softmax_out
......@@ -5398,14 +5404,16 @@ def one_hot(input, depth, allow_out_of_range=False):
label = fluid.data(name="label", shape=[4, 1], dtype="int64")
one_hot_label = fluid.layers.one_hot(input=label, depth=4)
"""
helper = LayerHelper("one_hot", **locals())
if in_dygraph_mode():
inputs = {'X': [input]}
attrs = {'depth': depth, 'allow_out_of_range': allow_out_of_range}
outs = core.ops.one_hot(inputs, attrs)
outs['Out'][0].stop_gradient = True
return outs['Out'][0]
helper = LayerHelper("one_hot", **locals())
one_hot_out = helper.create_variable_for_type_inference(dtype='float32')
if in_dygraph_mode():
inputs = {'X': input}
attrs = {'depth': depth, 'allow_out_of_range': allow_out_of_range}
else:
if not isinstance(depth, Variable):
# user attribute
inputs = {'X': input}
......@@ -6266,6 +6274,15 @@ def label_smooth(label,
"""
if epsilon > 1. or epsilon < 0.:
raise ValueError("The value of epsilon must be between 0 and 1.")
if in_dygraph_mode():
inputs = {"X": [label]}
if prior_dist:
inputs["PriorDist"] = [prior_dist]
attrs = {"epsilon": float(epsilon)}
outs = core.ops.label_smooth(inputs, attrs)
return outs['Out'][0]
helper = LayerHelper("label_smooth", **locals())
label.stop_gradient = True
smooth_label = helper.create_variable_for_type_inference(dtype)
......@@ -7839,6 +7856,11 @@ def log(x, name=None):
res_val, = exe.run(fluid.default_main_program(), feed={'x':x_i}, fetch_list=[res])
print(res_val) # [[0.], [0.6931472]]
"""
inputs = {'X': [x]}
if in_dygraph_mode():
outs = core.ops.log(inputs)
return outs['Out'][0]
helper = LayerHelper('log', **locals())
dtype = helper.input_dtype(input_param_name='x')
out = helper.create_variable_for_type_inference(dtype)
......@@ -7874,6 +7896,11 @@ def relu(x, name=None):
# [[0. 0. ]
# [1. 2.6]]
"""
inputs = {'X': [x]}
if in_dygraph_mode():
outs = core.ops.relu(inputs)
return outs['Out'][0]
helper = LayerHelper('relu', **locals())
dtype = helper.input_dtype(input_param_name='x')
out = helper.create_variable_for_type_inference(dtype)
......@@ -8462,6 +8489,17 @@ def pad2d(input,
result = fluid.layers.pad2d(input=data, paddings=[1, 2, 3, 4],
mode='reflect')
"""
attrs = {'mode': mode, 'pad_value': pad_value, 'data_format': data_format}
inputs = {'X': [input]}
if isinstance(paddings, Variable):
inputs['Paddings'] = [paddings]
attrs['paddings'] = []
else:
attrs['paddings'] = paddings
if in_dygraph_mode():
outs = core.ops.pad2d(inputs, attrs)
return outs['Out'][0]
helper = LayerHelper('pad2d', **locals())
......@@ -8470,14 +8508,6 @@ def pad2d(input,
dtype = helper.input_dtype(input_param_name='input')
out = helper.create_variable_for_type_inference(dtype)
inputs = {'X': input}
attrs = {'mode': mode, 'pad_value': pad_value, 'data_format': data_format}
if isinstance(paddings, Variable):
inputs['Paddings'] = paddings
attrs['paddings'] = []
else:
attrs['paddings'] = paddings
helper.append_op(
type='pad2d', inputs=inputs, outputs={"Out": out}, attrs=attrs)
......@@ -8907,13 +8937,16 @@ def leaky_relu(x, alpha=0.02, name=None):
res_val, = exe.run(fluid.default_main_program(), feed={'x':x_i}, fetch_list=[res])
print(res_val) # [[-0.1, 2], [3, -0.4]]
"""
inputs = {'X': [x]}
attrs = {'alpha': alpha}
if in_dygraph_mode():
outs = core.ops.leaky_relu(inputs, attrs)
return outs['Out'][0]
helper = LayerHelper('leaky_relu', **locals())
out = helper.create_variable_for_type_inference(dtype=x.dtype)
helper.append_op(
type='leaky_relu',
inputs={'X': x},
outputs={'Out': out},
attrs={'alpha': alpha})
type='leaky_relu', inputs=inputs, outputs={'Out': out}, attrs=attrs)
return out
......@@ -9311,6 +9344,32 @@ def expand(x, expand_times, name=None):
expanded_2 = fluid.layers.expand(data_2, expand_times=expand_times)
# the shape of expanded_2 is [48, 56].
"""
def contain_var(expand_times):
for ele in expand_times:
if isinstance(ele, Variable):
return True
return False
inputs = {"X": [x]}
attrs = {}
if in_dygraph_mode():
if isinstance(expand_times, (list, tuple)):
contain_var = contain_var(expand_times)
if contain_var:
raise TypeError(
"The type of 'expand_times' in expand must be list[int] or tuple(int) in Dygraph mode, but "
"received %s, which contains Variable." % type(shape))
attrs['expand_times'] = expand_times
else:
raise TypeError(
"The type of 'expand_times' in expand must be list[int] or tuple(int) in Dygraph mode, but "
"received %s." % type(shape))
outs = core.ops.expand(inputs, attrs)
return outs['Out'][0]
check_type_and_dtype(x, 'x', Variable,
['bool', 'float32', 'float64', 'int32', 'int64'],
'expand')
......@@ -9320,14 +9379,6 @@ def expand(x, expand_times, name=None):
"expand op bool date type must set the stop_gradient to be False")
helper = LayerHelper('expand', input=x, **locals())
inputs = {"X": x}
attrs = {}
def contain_var(expand_times):
for ele in expand_times:
if isinstance(ele, Variable):
return True
return False
def get_attr_expand_times(list_expand_times):
attrs_expand_times = []
......@@ -10363,24 +10414,27 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None):
print(res) # [array([[ 3., 5., 7.], [ 9., 11., 13.]], dtype=float32)]
"""
helper = LayerHelper('scale', **locals())
if name is None:
out = helper.create_variable_for_type_inference(dtype=x.dtype)
else:
out = helper.create_variable(
name=name, dtype=x.dtype, persistable=False)
inputs = {'X': x}
inputs = {'X': [x]}
attrs = {
'bias': float(bias),
'bias_after_scale': bias_after_scale,
}
if isinstance(scale, Variable):
inputs['ScaleTensor'] = scale
inputs['ScaleTensor'] = [scale]
else:
attrs['scale'] = float(scale)
if in_dygraph_mode():
outs = core.ops.scale(inputs, attrs)
return dygraph_utils._append_activation_in_dygraph(outs['Out'][0])
helper = LayerHelper('scale', **locals())
if name is None:
out = helper.create_variable_for_type_inference(dtype=x.dtype)
else:
out = helper.create_variable(
name=name, dtype=x.dtype, persistable=False)
helper.append_op(
type='scale', inputs=inputs, outputs={'Out': out}, attrs=attrs)
return helper.append_activation(out)
......@@ -10817,6 +10871,9 @@ Examples:
print(z_value)#[[[[0., 0., 0., 0., 0.] .... [0., 0., 0., 0., 0.]]]]
"""
if in_dygraph_mode():
return _elementwise_op_in_dygraph(
x, y, axis=axis, act=act, op_name='elementwise_min')
return _elementwise_op(LayerHelper('elementwise_min', **locals()))
......@@ -11407,6 +11464,11 @@ def mul(x, y, x_num_col_dims=1, y_num_col_dims=1, name=None):
"""
inputs = {"X": [x], "Y": [y]}
attrs = {"x_num_col_dims": x_num_col_dims, "y_num_col_dims": y_num_col_dims}
if in_dygraph_mode():
outs = core.ops.mul(inputs, attrs)
return outs['Out'][0]
helper = LayerHelper("mul", **locals())
check_type_and_dtype(x, 'x', Variable, ['float16', 'float32', 'float64'],
......@@ -11420,14 +11482,8 @@ def mul(x, y, x_num_col_dims=1, y_num_col_dims=1, name=None):
name=name, dtype=x.dtype, persistable=False)
helper.append_op(
type="mul",
inputs={"X": x,
"Y": y},
attrs={
"x_num_col_dims": x_num_col_dims,
"y_num_col_dims": y_num_col_dims
},
outputs={"Out": out})
type="mul", inputs={"X": x,
"Y": y}, attrs=attrs, outputs={"Out": out})
return out
......
python/paddle/fluid/layers/tensor.py
浏览文件 @ 51cb918a
......@@ -16,7 +16,7 @@ from __future__ import print_function
from six.moves import reduce
from ..layer_helper import LayerHelper
from ..param_attr import ParamAttr
from ..framework import convert_np_dtype_to_dtype_, in_dygraph_mode
from ..framework import convert_np_dtype_to_dtype_, in_dygraph_mode, _varbase_creator
from ..framework import Variable
from ..initializer import Constant, force_init_on_cpu
from ..core import VarDesc
......@@ -552,6 +552,43 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None):
shape = fluid.layers.fill_constant([1,2], "int32", 2) # shape=[2,2]
data4 = fluid.layers.fill_constant(shape=shape, dtype='bool', value=True) # data4=[[True,True],[True,True]]
"""
def _contain_var(one_list):
for ele in one_list:
if isinstance(ele, Variable):
return True
return False
attrs = {
'value': float(value),
'force_cpu': force_cpu or force_init_on_cpu()
}
if convert_dtype(dtype) in ['int64', 'int32']:
attrs['str_value'] = str(int(value))
else:
attrs['str_value'] = str(float(value))
if in_dygraph_mode():
if isinstance(shape, (list, tuple)):
contain_var = _contain_var(shape)
if contain_var:
raise TypeError(
"The type of 'shape' in fill_constant must be list[int] or tuple(int) in Dygraph mode, but "
"received %s, which contains Variable." % type(shape))
attrs['shape'] = shape
else:
raise TypeError(
"The type of 'shape' in fill_constant must be list[int] or tuple(int) in Dygraph mode, but "
"received %s." % type(shape))
if out is None:
out = _varbase_creator(dtype=dtype)
attrs['dtype'] = out.dtype
outputs = {'Out': [out]}
outs = core.ops.fill_constant({}, attrs, outputs)
out.stop_gradient = True
return out
helper = LayerHelper("fill_constant", **locals())
check_dtype(dtype, 'create data type',
['bool', 'float16', 'float32', 'float64', 'int32', 'int64'],
......@@ -568,12 +605,6 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None):
else:
attrs['str_value'] = str(float(value))
def _contain_var(one_list):
for ele in one_list:
if isinstance(ele, Variable):
return True
return False
def _get_attr_shape(list_shape):
attr_shape = []
for idx, dim in enumerate(list_shape):
......
python/paddle/fluid/optimizer.py
浏览文件 @ 51cb918a
......@@ -896,21 +896,30 @@ class MomentumOptimizer(Optimizer):
velocity_acc = self._get_accumulator(self._velocity_acc_str,
param_and_grad[0])
attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov}
inputs = {
"Param": [param_and_grad[0]],
"Grad": [param_and_grad[1]],
"Velocity": [velocity_acc],
"LearningRate": [self._create_param_lr(param_and_grad)]
}
outputs = {
"ParamOut": [param_and_grad[0]],
"VelocityOut": [velocity_acc]
}
if framework.in_dygraph_mode():
core.ops.momentum(inputs, attrs, outputs)
return None
# create the momentum optimize op
momentum_op = block.append_op(
type=self.type,
inputs={
"Param": param_and_grad[0],
"Grad": param_and_grad[1],
"Velocity": velocity_acc,
"LearningRate": self._create_param_lr(param_and_grad)
},
outputs={
"ParamOut": param_and_grad[0],
"VelocityOut": velocity_acc
},
attrs={"mu": self._momentum,
"use_nesterov": self._use_nesterov},
inputs=inputs,
outputs=outputs,
attrs=attrs,
stop_gradient=True)
return momentum_op
......
python/paddle/fluid/regularizer.py
浏览文件 @ 51cb918a
......@@ -15,6 +15,7 @@
from __future__ import print_function
from . import framework
from .framework import in_dygraph_mode, _varbase_creator
from . import core
__all__ = ['L1Decay', 'L2Decay', 'L1DecayRegularizer', 'L2DecayRegularizer']
......@@ -74,10 +75,12 @@ def append_regularization_ops(parameters_and_grads, regularization=None):
lod_level=param.lod_level,
type=core.VarDesc.VarType.LOD_TENSOR)
grad.block.append_op(
type='sum',
inputs={"X": [grad, regularization_term]},
outputs={"Out": new_grad})
inputs = {"X": [grad, regularization_term]}
outputs = {"Out": [new_grad]}
if in_dygraph_mode():
core.ops.sum(inputs, {}, outputs)
else:
grad.block.append_op(type='sum', inputs=inputs, outputs=outputs)
params_and_grads.append((param, new_grad))
......@@ -165,8 +168,12 @@ class L2DecayRegularizer(WeightDecayRegularizer):
assert isinstance(param, framework.Parameter)
assert isinstance(block, framework.Block)
inputs = {"X": [param]}
attrs = {"scale": self._regularization_coeff}
if framework.in_dygraph_mode():
decay = block.create_var(dtype=param.dtype, shape=param.shape)
outs = core.ops.scale(inputs, attrs)
return outs['Out'][0]
else:
decay = block.create_var(
dtype=param.dtype, shape=param.shape, lod_level=param.lod_level)
......
python/paddle/fluid/tests/unittests/test_imperative_resnet.py
浏览文件 @ 51cb918a
......@@ -27,6 +27,8 @@ from test_imperative_base import new_program_scope
from utils import DyGraphProgramDescTracerTestHelper, is_equal_program
from paddle.fluid.dygraph import TracedLayer
#NOTE(zhiqiu): run with FLAGS_cudnn_deterministic=1
batch_size = 8
train_parameters = {
"input_size": [3, 224, 224],
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或