From 94fb7ba4fa36a37db9a4b4af17a119f4c5463e40 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Thu, 26 Oct 2017 17:52:57 -0700 Subject: [PATCH] Adding L1 norm op for L1 regularization (#5058) * Adding L1 norm op for L1 regularization * Addressing code review feedback * Address code review feedback * Change variable names to match google style guide --- paddle/operators/l1_norm_op.cc | 75 +++++++++++++++++++ paddle/operators/l1_norm_op.cu | 22 ++++++ paddle/operators/l1_norm_op.h | 63 ++++++++++++++++ .../v2/framework/tests/test_l1_norm_op.py | 28 +++++++ 4 files changed, 188 insertions(+) create mode 100644 paddle/operators/l1_norm_op.cc create mode 100644 paddle/operators/l1_norm_op.cu create mode 100644 paddle/operators/l1_norm_op.h create mode 100644 python/paddle/v2/framework/tests/test_l1_norm_op.py diff --git a/paddle/operators/l1_norm_op.cc b/paddle/operators/l1_norm_op.cc new file mode 100644 index 00000000000..1d111696cf4 --- /dev/null +++ b/paddle/operators/l1_norm_op.cc @@ -0,0 +1,75 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/l1_norm_op.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; + +class L1NormOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should be not null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should be not null."); + + ctx->SetOutputDim("Out", {1}); + } +}; + +class L1NormGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should be not null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should be not null."); + PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")), + "Output(X@GRAD) should be not null."); + + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + } +}; + +class L1NormOpMaker : public framework::OpProtoAndCheckerMaker { + public: + L1NormOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : framework::OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "(Tensor) The input of l1_norm op."); + AddOutput("Out", "(Scalar) The output of l1_norm op."); + AddComment(R"DOC( +L1 Norm Operator. + +Computes the L1 norm of a tensor. + +Out = sum (abs(X)) + +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(l1_norm, ops::L1NormOp, ops::L1NormOpMaker, l1_norm_grad, + ops::L1NormGradOp); +REGISTER_OP_CPU_KERNEL(l1_norm, + ops::L1NormKernel); +REGISTER_OP_CPU_KERNEL( + l1_norm_grad, ops::L1NormGradKernel); diff --git a/paddle/operators/l1_norm_op.cu b/paddle/operators/l1_norm_op.cu new file mode 100644 index 00000000000..1c206e04ccb --- /dev/null +++ b/paddle/operators/l1_norm_op.cu @@ -0,0 +1,22 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/l1_norm_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(l1_norm, + ops::L1NormKernel); +REGISTER_OP_GPU_KERNEL( + l1_norm_grad, ops::L1NormGradKernel); diff --git a/paddle/operators/l1_norm_op.h b/paddle/operators/l1_norm_op.h new file mode 100644 index 00000000000..de459818ad8 --- /dev/null +++ b/paddle/operators/l1_norm_op.h @@ -0,0 +1,63 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +// Out = sum(abs(X)) +template +class L1NormKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &context) const override { + const framework::Tensor *X = context.Input("X"); + framework::Tensor *Out = context.Output("Out"); + Out->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto out = framework::EigenVector::Flatten(*Out); + auto place = context.GetEigenDevice(); + + out.device(place) = x.abs().sum(); + } +}; + +// dX = dout * sign(X) +template +class L1NormGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &context) const override { + const framework::Tensor *x = context.Input("X"); + const framework::Tensor *d_out = + context.Input(framework::GradVarName("Out")); + PADDLE_ENFORCE(d_out->numel() == 1, "L1 Norm Gradient should be scalar"); + framework::Tensor *dx = + context.Output(framework::GradVarName("X")); + dx->mutable_data(context.GetPlace()); + + auto x_eigen = framework::EigenVector::Flatten(*x); + auto d_out_eigen = framework::EigenVector::Flatten(*d_out); + auto dx_eigen = framework::EigenVector::Flatten(*dx); + auto place = context.GetEigenDevice(); + + Eigen::DSizes x_dsize(x->numel()); + dx_eigen.device(place) = d_out_eigen.broadcast(x_dsize) * x_eigen.sign(); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/python/paddle/v2/framework/tests/test_l1_norm_op.py b/python/paddle/v2/framework/tests/test_l1_norm_op.py new file mode 100644 index 00000000000..3a1d1689fe6 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_l1_norm_op.py @@ -0,0 +1,28 @@ +import numpy as np +import unittest +from op_test import OpTest + + +class TestL1NormOp(OpTest): + """Test l1_norm + """ + + def setUp(self): + self.op_type = "l1_norm" + self.max_relative_error = 0.005 + + X = np.random.uniform(-1, 1, (13, 19)).astype("float32") + X[np.abs(X) < self.max_relative_error] = 0.1 + self.inputs = {'X': X} + self.outputs = {'Out': np.sum(np.abs(X))} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad( + ['X'], 'Out', max_relative_error=self.max_relative_error) + + +if __name__ == "__main__": + unittest.main() -- GitLab