diff --git a/paddle/operators/adam_op.cc b/paddle/operators/adam_op.cc index e3db70ea129880434add21e71d15e5129c4551bd..3572de06bd60f7979e3bfbf39856b04942ce81c0 100644 --- a/paddle/operators/adam_op.cc +++ b/paddle/operators/adam_op.cc @@ -43,10 +43,6 @@ class AdamOp : public framework::OperatorWithKernel { "Output(Moment1Out) of AdamOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Moment2Out"), "Output(Moment2Out) of AdamOp should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("Beta1PowOut"), - "Output(Beta1PowOut) of AdamOp should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("Beta2PowOut"), - "Output(Beta2PowOut) of AdamOp should not be null."); auto lr_dims = ctx->GetInputDim("LearningRate"); PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1, @@ -72,8 +68,6 @@ class AdamOp : public framework::OperatorWithKernel { ctx->SetOutputDim("ParamOut", param_dims); ctx->SetOutputDim("Moment1Out", param_dims); ctx->SetOutputDim("Moment2Out", param_dims); - ctx->SetOutputDim("Beta1PowOut", beta1_pow_dims); - ctx->SetOutputDim("Beta2PowOut", beta2_pow_dims); } }; @@ -92,8 +86,6 @@ class AdamOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("ParamOut", "(Tensor) Output parameter"); AddOutput("Moment1Out", "(Tensor) Output first moment"); AddOutput("Moment2Out", "(Tensor) Output second moment"); - AddOutput("Beta1PowOut", "(Tensor) Output beta1 power accumulator"); - AddOutput("Beta2PowOut", "(Tensor) Output beta2 power accumulator"); AddAttr("beta1", "(float, default 0.9) " @@ -121,10 +113,8 @@ Adam updates: moment1_out = beta1 * moment1 + (1 − beta1) * grad moment2_out = beta2 * moment2 + (1 − beta2) * grad * grad -beta1_pow_out = beta1_pow * beta1 -beta2_pow_out = beta2_pow * beta2 learning_rate_t = learning_rate_t * - sqrt(1 - beta2_pow_out) / (1 - beta1_pow_out) + sqrt(1 - beta2_pow) / (1 - beta1_pow) param_out = param - learning_rate_t * moment1/ (sqrt(moment2) + epsilon) References: diff --git a/paddle/operators/adam_op.h b/paddle/operators/adam_op.h index 789c2f14b32478bf9ddc967fc5725bcf65ed2146..45938006db1231a7a134964d729df6ca114d4dbe 100644 --- a/paddle/operators/adam_op.h +++ b/paddle/operators/adam_op.h @@ -26,14 +26,10 @@ class AdamOpKernel : public framework::OpKernel { auto param_out_tensor = ctx.Output("ParamOut"); auto moment1_out_tensor = ctx.Output("Moment1Out"); auto moment2_out_tensor = ctx.Output("Moment2Out"); - auto beta1_pow_out_tensor = ctx.Output("Beta1PowOut"); - auto beta2_pow_out_tensor = ctx.Output("Beta2PowOut"); param_out_tensor->mutable_data(ctx.GetPlace()); moment1_out_tensor->mutable_data(ctx.GetPlace()); moment2_out_tensor->mutable_data(ctx.GetPlace()); - beta1_pow_out_tensor->mutable_data(ctx.GetPlace()); - beta2_pow_out_tensor->mutable_data(ctx.GetPlace()); float beta1 = ctx.Attr("beta1"); float beta2 = ctx.Attr("beta2"); @@ -56,18 +52,13 @@ class AdamOpKernel : public framework::OpKernel { auto param_out = framework::EigenVector::Flatten(*param_out_tensor); auto moment1_out = framework::EigenVector::Flatten(*moment1_out_tensor); auto moment2_out = framework::EigenVector::Flatten(*moment2_out_tensor); - auto beta1_pow_out = - framework::EigenVector::Flatten(*beta1_pow_out_tensor); - auto beta2_pow_out = - framework::EigenVector::Flatten(*beta2_pow_out_tensor); auto place = ctx.GetEigenDevice(); moment1_out.device(place) = beta1 * moment1 + (1 - beta1) * grad; moment2_out.device(place) = beta2 * moment2 + (1 - beta2) * grad.square(); - beta1_pow_out.device(place) = beta1_pow * beta1; - beta2_pow_out.device(place) = beta2_pow * beta2; + // All of these are tensors of 1 element - auto lr_t = lr * (1 - beta2_pow_out).sqrt() / (1 - beta1_pow_out); + auto lr_t = lr * (1 - beta2_pow).sqrt() / (1 - beta1_pow); // Eigen does not support automatic broadcast // Get dimensions of moment vector to broadcast lr_t Eigen::DSizes m_dsize(moment1_out_tensor->numel()); diff --git a/paddle/operators/adamax_op.cc b/paddle/operators/adamax_op.cc index e848333ef8a819648cc3056ae2f4a0e33fc58405..ff2565774115571166712b03c8990e5bf8de12a5 100644 --- a/paddle/operators/adamax_op.cc +++ b/paddle/operators/adamax_op.cc @@ -41,8 +41,6 @@ class AdamaxOp : public framework::OperatorWithKernel { "Output(MomentOut) of AdamaxOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("InfNormOut"), "Output(InfNormOut) of AdamaxOp should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("Beta1PowOut"), - "Output(Beta1PowOut) of AdamaxOp should not be null."); auto lr_dims = ctx->GetInputDim("LearningRate"); PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1, @@ -64,7 +62,6 @@ class AdamaxOp : public framework::OperatorWithKernel { ctx->SetOutputDim("ParamOut", param_dims); ctx->SetOutputDim("MomentOut", param_dims); ctx->SetOutputDim("InfNormOut", param_dims); - ctx->SetOutputDim("Beta1PowOut", beta1_pow_dims); } }; @@ -86,7 +83,6 @@ class AdamaxOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("InfNormOut", "(Tensor) " "Output exponentially weighted infinity norm"); - AddOutput("Beta1PowOut", "(Tensor) Output beta1 power accumulator"); AddAttr("beta1", "(float, default 0.9) " @@ -113,8 +109,7 @@ Adamax updates: moment_out = beta1 * moment + (1 - beta1) * grad inf_norm_out = max(beta2 * inf_norm + epsilon, abs(grad)) -beta1_pow_out = beta1_pow * beta1 -learning_rate_t = learning_rate/(1 - beta1_pow_out) +learning_rate_t = learning_rate/(1 - beta1_pow) param_out = param - learning_rate_t * moment_out/inf_norm_out The original paper does not have an epsilon attribute. diff --git a/paddle/operators/adamax_op.h b/paddle/operators/adamax_op.h index 9677b1bb786002aadfaeb571b2ba2e6aa2481ca5..2c99832ec08e9c1d9b5458c467d5238f9b1b3c37 100644 --- a/paddle/operators/adamax_op.h +++ b/paddle/operators/adamax_op.h @@ -26,12 +26,10 @@ class AdamaxOpKernel : public framework::OpKernel { auto param_out_tensor = ctx.Output("ParamOut"); auto moment_out_tensor = ctx.Output("MomentOut"); auto inf_norm_out_tensor = ctx.Output("InfNormOut"); - auto beta1_pow_out_tensor = ctx.Output("Beta1PowOut"); param_out_tensor->mutable_data(ctx.GetPlace()); moment_out_tensor->mutable_data(ctx.GetPlace()); inf_norm_out_tensor->mutable_data(ctx.GetPlace()); - beta1_pow_out_tensor->mutable_data(ctx.GetPlace()); float beta1 = ctx.Attr("beta1"); float beta2 = ctx.Attr("beta2"); @@ -53,15 +51,12 @@ class AdamaxOpKernel : public framework::OpKernel { auto moment_out = framework::EigenVector::Flatten(*moment_out_tensor); auto inf_norm_out = framework::EigenVector::Flatten(*inf_norm_out_tensor); - auto beta1_pow_out = - framework::EigenVector::Flatten(*beta1_pow_out_tensor); auto place = ctx.GetEigenDevice(); moment_out.device(place) = beta1 * moment + (1 - beta1) * grad; inf_norm_out.device(place) = grad.abs().cwiseMax((beta2 * inf_norm) + epsilon); - beta1_pow_out.device(place) = beta1_pow * beta1; - auto lr_t = lr / (1 - beta1_pow_out); + auto lr_t = lr / (1 - beta1_pow); Eigen::DSizes m_dsize(moment_out_tensor->numel()); param_out.device(place) = param - lr_t.broadcast(m_dsize) * (moment_out / inf_norm_out); diff --git a/python/paddle/v2/framework/framework.py b/python/paddle/v2/framework/framework.py index eb6867a194206c908b2a4a265a1cd983d33d6b56..30ea5c99ddc4e128732874b6cf80dd454b38d802 100644 --- a/python/paddle/v2/framework/framework.py +++ b/python/paddle/v2/framework/framework.py @@ -432,11 +432,13 @@ class Program(object): def current_block(self): return self.blocks[self.current_block_idx] - def append_backward(self, target, no_grad_set): + def append_backward(self, target, no_grad_set=None): """ return map(param_name -> (grad_name, block_index, op_index)) """ assert isinstance(target, Variable) + if no_grad_set is None: + no_grad_set = set() param_to_grad_info = self.desc.append_backward(target.desc, no_grad_set) self.sync_with_cpp() return param_to_grad_info diff --git a/python/paddle/v2/framework/layers.py b/python/paddle/v2/framework/layers.py index 329a6830b6a94b12a522f7b99feb97babb843ccc..236427efcefafd8dc15f3f184f568887fdb00992 100644 --- a/python/paddle/v2/framework/layers.py +++ b/python/paddle/v2/framework/layers.py @@ -3,7 +3,7 @@ import paddle.v2.framework.core as core from paddle.v2.framework.framework import OpProtoHolder, Variable import re -__all__ = ['fc', 'data', 'cross_entropy', 'conv2d'] +__all__ = ['fc', 'data', 'cross_entropy', 'conv2d', 'pool2d'] def fc(input, @@ -35,7 +35,10 @@ def fc(input, "Y": w, }, outputs={"Out": tmp}, - attrs={'x_num_col_dims': num_flatten_dims}) + attrs={ + 'x_num_col_dims': num_flatten_dims, + 'y_num_col_dims': len(input_shape) - num_flatten_dims + }) mul_results.append(tmp) # sum @@ -115,7 +118,6 @@ def _create_op_func_(op_type): _create_op_func_('mean') _create_op_func_('mul') -_create_op_func_('pool2d') def cross_entropy(input, label, **kwargs): @@ -170,6 +172,13 @@ def conv2d(input, raise ValueError("num_channels must be divisible by groups.") num_filter_channels = num_channels / groups + if isinstance(filter_size, int): + filter_size = [filter_size, filter_size] + if isinstance(stride, int): + stride = [stride, stride] + if isinstance(padding, int): + padding = [padding, padding] + input_shape = input.shape filter_shape = [num_filters, num_filter_channels] + filter_size filter = helper.create_parameter( @@ -190,3 +199,40 @@ def conv2d(input, pre_act = helper.append_bias_op(pre_bias) return helper.append_activation(pre_act) + + +def pool2d(input, + pool_size, + pool_type, + pool_stride=[1, 1], + pool_padding=[0, 0], + global_pooling=False, + program=None): + if pool_type not in ["max", "avg"]: + raise ValueError( + "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", + str(pool_type)) + if isinstance(pool_size, int): + pool_size = [pool_size, pool_size] + if isinstance(pool_stride, int): + pool_stride = [pool_stride, pool_stride] + if isinstance(pool_padding, int): + pool_padding = [pool_padding, pool_padding] + + helper = LayerHelper('conv2d', **locals()) + dtype = helper.input_dtype() + pool_out = helper.create_tmp_variable(dtype) + + helper.append_op( + type="pool2d", + inputs={"X": input}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "global_pooling": global_pooling, + "strides": pool_stride, + "paddings": pool_padding + }) + + return pool_out diff --git a/python/paddle/v2/framework/nets.py b/python/paddle/v2/framework/nets.py new file mode 100644 index 0000000000000000000000000000000000000000..381da55da3cd4e32fe09241a00d74e74e2de44f7 --- /dev/null +++ b/python/paddle/v2/framework/nets.py @@ -0,0 +1,24 @@ +import paddle.v2.framework.layers as layers + + +def simple_img_conv_pool(input, + filter_size, + num_filters, + pool_size, + pool_stride, + act, + program=None): + conv_out = layers.conv2d( + input=input, + num_filters=num_filters, + filter_size=filter_size, + act=act, + program=program) + + pool_out = layers.pool2d( + input=conv_out, + pool_size=pool_size, + pool_type='max', + pool_stride=pool_stride, + program=program) + return pool_out diff --git a/python/paddle/v2/framework/tests/test_adam_op.py b/python/paddle/v2/framework/tests/test_adam_op.py index ff6faafa6e2119fde11b9eb6cd2a65a75334ebe6..a0d6655d4cbcff8ed3d55df0f4e68fc6591fbb11 100644 --- a/python/paddle/v2/framework/tests/test_adam_op.py +++ b/python/paddle/v2/framework/tests/test_adam_op.py @@ -33,14 +33,12 @@ class TestAdamOp1(OpTest): self.attrs = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2} - param_out, moment1_out, moment2_out, beta1_pow_out, \ - beta2_pow_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, \ + moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, - 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out, 'ParamOut': param_out } @@ -78,14 +76,12 @@ class TestAdamOp2(OpTest): attributes = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2} - param_out, moment1_out, moment2_out, beta1_pow_out, \ - beta2_pow_out = adam_step(self.inputs, attributes) + param_out, moment1_out, \ + moment2_out = adam_step(self.inputs, attributes) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, - 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out, 'ParamOut': param_out } @@ -127,14 +123,12 @@ class TestAdamOpMultipleSteps(OpTest): def test_check_output(self): for _ in range(self.num_steps): - param_out, moment1_out, moment2_out, beta1_pow_out, \ - beta2_pow_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, \ + moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, - 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out, 'ParamOut': param_out } @@ -145,8 +139,10 @@ class TestAdamOpMultipleSteps(OpTest): self.inputs['Param'] = param_out self.inputs['Moment1'] = moment1_out self.inputs['Moment2'] = moment2_out - self.inputs['Beta1Pow'] = beta1_pow_out - self.inputs['Beta2Pow'] = beta2_pow_out + + # Update powers of Beta1 and Beta2 for next time step + self.inputs['Beta1Pow'] *= self.attrs['beta1'] + self.inputs['Beta2Pow'] *= self.attrs['beta1'] # Randomize gradient for next step self.inputs['Grad'] = np.random.uniform( @@ -175,11 +171,9 @@ def adam_step(inputs, attributes): moment1_out = beta1 * moment1 + (1 - beta1) * grad moment2_out = beta2 * moment2 + (1 - beta2) * np.square(grad) - beta1_pow_out = beta1_pow * beta1 - beta2_pow_out = beta2_pow * beta2 - lr_t = lr * np.sqrt(1 - beta2_pow_out) / (1 - beta1_pow_out) + lr_t = lr * np.sqrt(1 - beta2_pow) / (1 - beta1_pow) param_out = param - lr_t * (moment1_out / (np.sqrt(moment2_out) + epsilon)) - return param_out, moment1_out, moment2_out, beta1_pow_out, beta2_pow_out + return param_out, moment1_out, moment2_out if __name__ == "__main__": diff --git a/python/paddle/v2/framework/tests/test_adamax_op.py b/python/paddle/v2/framework/tests/test_adamax_op.py index af81075d6ad508dcd473ed596b00b036d87d894f..8e5a15aa3d12bbaae99cae6fcb627a336e48f684 100644 --- a/python/paddle/v2/framework/tests/test_adamax_op.py +++ b/python/paddle/v2/framework/tests/test_adamax_op.py @@ -31,14 +31,13 @@ class TestAdamaxOp1(OpTest): self.attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon} - param_out, moment_out, inf_norm_out, beta1_pow_out = adamax_step( - self.inputs, self.attrs) + param_out, moment_out, inf_norm_out = adamax_step(self.inputs, + self.attrs) self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, - 'InfNormOut': inf_norm_out, - 'Beta1PowOut': beta1_pow_out + 'InfNormOut': inf_norm_out } def test_check_output(self): @@ -73,14 +72,12 @@ class TestAdamaxOp2(OpTest): } attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon} - param_out, moment_out, inf_norm_out, beta1_pow_out = adamax_step( - self.inputs, attrs) + param_out, moment_out, inf_norm_out = adamax_step(self.inputs, attrs) self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, - 'InfNormOut': inf_norm_out, - 'Beta1PowOut': beta1_pow_out + 'InfNormOut': inf_norm_out } def test_check_output(self): @@ -117,19 +114,15 @@ class TestAdamaxOpMultipleSteps(OpTest): self.attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon} - param_out, moment_out, inf_norm_out, beta1_pow_out = adamax_step( - self.inputs, self.attrs) - def test_check_output(self): for _ in range(self.num_steps): - param_out, moment_out, inf_norm_out, beta1_pow_out = adamax_step( - self.inputs, self.attrs) + param_out, moment_out, inf_norm_out = adamax_step(self.inputs, + self.attrs) self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, - 'InfNormOut': inf_norm_out, - 'Beta1PowOut': beta1_pow_out + 'InfNormOut': inf_norm_out } # Verify output for this step @@ -139,7 +132,9 @@ class TestAdamaxOpMultipleSteps(OpTest): self.inputs['Param'] = param_out self.inputs['Moment'] = moment_out self.inputs['InfNorm'] = inf_norm_out - self.inputs['Beta1Pow'] = beta1_pow_out + + # Update Beta1 Power accumulator for next step + self.inputs['Beta1Pow'] *= self.attrs['beta1'] # Randomize gradient for next step self.inputs['Grad'] = np.random.uniform( @@ -167,11 +162,10 @@ def adamax_step(inputs, attributes): moment_out = beta1 * moment + (1 - beta1) * grad inf_norm_out = np.maximum(beta2 * inf_norm + epsilon, np.abs(grad)) - beta1_pow_out = beta1_pow * beta1 - lr_t = (lr / (1 - beta1_pow_out)) + lr_t = (lr / (1 - beta1_pow)) param_out = param - lr_t * np.divide(moment_out, inf_norm_out) - return param_out, moment_out, inf_norm_out, beta1_pow_out + return param_out, moment_out, inf_norm_out if __name__ == "__main__": diff --git a/python/paddle/v2/framework/tests/test_layers.py b/python/paddle/v2/framework/tests/test_layers.py index dbbb6535389e2336d156734cc672e0cc7bba175c..4ecc02b12d8db53e897dea10186bc053d05be303 100644 --- a/python/paddle/v2/framework/tests/test_layers.py +++ b/python/paddle/v2/framework/tests/test_layers.py @@ -1,4 +1,5 @@ import paddle.v2.framework.layers as layers +import paddle.v2.framework.nets as nets from paddle.v2.framework.framework import Program, g_program import paddle.v2.framework.core as core import unittest @@ -18,7 +19,7 @@ class TestBook(unittest.TestCase): avg_cost = layers.mean(x=cost, program=program) self.assertIsNotNone(avg_cost) - program.append_backward(avg_cost, set()) + program.append_backward(avg_cost) print str(program) def test_recognize_digits_mlp(self): @@ -38,24 +39,52 @@ class TestBook(unittest.TestCase): cost = layers.cross_entropy(input=predict, label=label, program=program) avg_cost = layers.mean(x=cost, program=program) self.assertIsNotNone(avg_cost) - # print str(program) + print str(program) def test_simple_conv2d(self): - pd = core.ProgramDesc.__create_program_desc__() - program = Program(desc=pd) - images = data_layer( + program = Program() + images = layers.data( name='pixel', shape=[3, 48, 48], data_type='int32', program=program) - conv2d_layer( + layers.conv2d( input=images, num_filters=3, filter_size=[4, 4], program=program) - # print str(program) + print str(program) - def test_simple_conv2d(self): + def test_recognize_digits_conv(self): program = Program() + images = layers.data( - name='pixel', shape=[3, 48, 48], data_type='int32', program=program) - layers.conv2d( - input=images, num_filters=3, filter_size=[4, 4], program=program) + name='pixel', + shape=[1, 28, 28], + data_type='float32', + program=program) + label = layers.data( + name='label', shape=[1], data_type='int32', program=program) + conv_pool_1 = nets.simple_img_conv_pool( + input=images, + filter_size=5, + num_filters=2, + pool_size=2, + pool_stride=2, + act="relu", + program=program) + conv_pool_2 = nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=4, + pool_size=2, + pool_stride=2, + act="relu", + program=program) + + predict = layers.fc(input=conv_pool_2, + size=10, + act="softmax", + program=program) + cost = layers.cross_entropy(input=predict, label=label, program=program) + avg_cost = layers.mean(x=cost, program=program) + + program.append_backward(avg_cost) print str(program)