diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index 90c7171419888612a48d929ed85039b16384a573..f8b0bce6815ff17a60ef64b0eec34a7cc9d16e72 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -55,6 +55,13 @@ function(op_library TARGET) set(pybind_flag 1) endif() + # activation_op contains several operators + if ("${TARGET}" STREQUAL "activation_op") + set(pybind_flag 1) + # It's enough to just adding one operator to pybind + file(APPEND ${pybind_file} "USE_OP(sigmoid);\n") + endif() + # pybind USE_NO_KERNEL_OP file(READ ${TARGET}.cc TARGET_CONTENT) string(REGEX MATCH "OperatorWithKernel" regex_result "${TARGET_CONTENT}") diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..cc55767cef9552475321bcb8c06d74a8d91dc99b --- /dev/null +++ b/paddle/operators/activation_op.cc @@ -0,0 +1,306 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/activation_op.h" + +namespace paddle { +namespace operators { + +class ActivationOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + ctx.Output("Y")->Resize( + ctx.Input("X")->dims()); + } +}; + +class ActivationOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + ctx.Output(framework::GradVarName("X")) + ->Resize(ctx.Input("Y")->dims()); + } +}; + +class SigmoidOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SigmoidOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Sigmoid operator"); + AddOutput("Y", "Output of Sigmoid operator"); + AddComment("Sigmoid activation operator, sigmoid = 1 / (1 + exp(-x))"); + } +}; + +class ExpOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ExpOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Exp operator"); + AddOutput("Y", "Output of Exp operator"); + AddComment("Exp activation operator, exp(x) = e^x"); + } +}; + +class ReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ReluOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Relu operator"); + AddOutput("Y", "Output of Relu operator"); + AddComment("Relu activation operator, relu(x) = max(x, 0)"); + } +}; + +class TanhOpMaker : public framework::OpProtoAndCheckerMaker { + public: + TanhOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Tanh operator"); + AddOutput("Y", "Output of Tanh operator"); + AddComment( + "Tanh activation operator, tanh = (exp(x) - exp(-x)) / (exp(x) + " + "exp(-x))"); + } +}; + +class SqrtOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SqrtOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Sqrt operator"); + AddOutput("Y", "Output of Sqrt operator"); + AddComment("Sqrt activation operator, sqrt(x) = x^(1/2)"); + } +}; + +class AbsOpMaker : public framework::OpProtoAndCheckerMaker { + public: + AbsOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Abs operator"); + AddOutput("Y", "Output of Abs operator"); + AddComment("Abs activation operator, abs(x) = |x|"); + } +}; + +class ReciprocalOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ReciprocalOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Reciprocal operator"); + AddOutput("Y", "Output of Reciprocal operator"); + AddComment("Reciprocal activation operator, reciprocal(x) = 1 / x"); + } +}; + +class LogOpMaker : public framework::OpProtoAndCheckerMaker { + public: + LogOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Log operator"); + AddOutput("Y", "Output of Log operator"); + AddComment("Log activation operator, log(x) = natural logarithm of x"); + } +}; + +class SquareOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SquareOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Square operator"); + AddOutput("Y", "Output of Square operator"); + AddComment("Square activation operator, square(x) = x^2"); + } +}; + +template +class BReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + BReluOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of BRelu operator"); + AddOutput("Y", "Output of BRelu operator"); + AddComment("BRelu activation operator, brelu = max(min(x, t_min), t_max)"); + AddAttr("t_min", "The min marginal value of BRelu") + .SetDefault(static_cast(0)); + AddAttr("t_max", "The max marginal value of BRelu") + .SetDefault(static_cast(24)); + } +}; + +template +class SoftReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SoftReluOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of SoftRelu operator"); + AddOutput("Y", "Output of SoftRelu operator"); + AddComment( + "SoftRelu activation operator, soft_relu = log(1 + exp(max(min(x, " + "threshold), threshold)))"); + AddAttr("threshold", "The threshold value of SoftRelu") + .SetDefault(static_cast(40)); + } +}; + +template +class PowOpMaker : public framework::OpProtoAndCheckerMaker { + public: + PowOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Pow operator"); + AddOutput("Y", "Output of Pow operator"); + AddComment("Pow activation operator, pow(x, factor) = x^factor"); + AddAttr("factor", "The exponential factor of Pow") + .SetDefault(static_cast(1)); + } +}; + +template +class STanhOpMaker : public framework::OpProtoAndCheckerMaker { + public: + STanhOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of STanh operator"); + AddOutput("Y", "Output of STanh operator"); + AddComment("STanh activation operator, stanh = b * tanh(a * x)"); + AddAttr("scale_a", "The scale parameter of a for the input") + .SetDefault(static_cast(2 / 3)); + AddAttr("scale_b", "The scale parameter of b for the input") + .SetDefault(static_cast(1.7159)); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(sigmoid, ops::ActivationOp, ops::SigmoidOpMaker, sigmoid_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(sigmoid, + ops::ActivationKernel>); +REGISTER_OP_CPU_KERNEL( + sigmoid_grad, ops::ActivationGradKernel>); + +REGISTER_OP(exp, ops::ActivationOp, ops::ExpOpMaker, exp_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + exp, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL(exp_grad, + ops::ActivationGradKernel); + +REGISTER_OP(relu, ops::ActivationOp, ops::ReluOpMaker, relu_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(relu, + ops::ActivationKernel>); +REGISTER_OP_CPU_KERNEL( + relu_grad, ops::ActivationGradKernel>); + +REGISTER_OP(tanh, ops::ActivationOp, ops::TanhOpMaker, tanh_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + tanh, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + tanh_grad, ops::ActivationGradKernel>); + +REGISTER_OP(sqrt, ops::ActivationOp, ops::SqrtOpMaker, sqrt_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + sqrt, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + sqrt_grad, ops::ActivationGradKernel>); + +REGISTER_OP(abs, ops::ActivationOp, ops::AbsOpMaker, abs_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + abs, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL(abs_grad, + ops::ActivationGradKernel); + +REGISTER_OP(reciprocal, ops::ActivationOp, ops::ReciprocalOpMaker, + reciprocal_grad, ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(reciprocal, + ops::ActivationKernel>); +REGISTER_OP_CPU_KERNEL( + reciprocal_grad, + ops::ActivationGradKernel>); + +REGISTER_OP(log, ops::ActivationOp, ops::LogOpMaker, log_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + log, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + log_grad, ops::ActivationGradKernel>); + +REGISTER_OP(square, ops::ActivationOp, ops::SquareOpMaker, square_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(square, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + square_grad, ops::ActivationGradKernel>); + +REGISTER_OP(brelu, ops::ActivationOp, ops::BReluOpMaker, brelu_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(brelu, + ops::BReluKernel); +REGISTER_OP_CPU_KERNEL(brelu_grad, + ops::BReluGradKernel); + +REGISTER_OP(soft_relu, ops::ActivationOp, ops::SoftReluOpMaker, + soft_relu_grad, ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(soft_relu, + ops::SoftReluKernel); +REGISTER_OP_CPU_KERNEL( + soft_relu_grad, ops::SoftReluGradKernel); + +REGISTER_OP(pow, ops::ActivationOp, ops::PowOpMaker, pow_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(pow, ops::PowKernel); +REGISTER_OP_CPU_KERNEL(pow_grad, + ops::PowGradKernel); + +REGISTER_OP(stanh, ops::ActivationOp, ops::STanhOpMaker, stanh_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(stanh, + ops::STanhKernel); +REGISTER_OP_CPU_KERNEL(stanh_grad, + ops::STanhGradKernel); diff --git a/paddle/operators/activation_op.cu b/paddle/operators/activation_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..feed1302b292a546f88fa35457c86aa2cfdaa307 --- /dev/null +++ b/paddle/operators/activation_op.cu @@ -0,0 +1,100 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/activation_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL(sigmoid, + ops::ActivationKernel>); +REGISTER_OP_GPU_KERNEL( + sigmoid_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + exp, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL(exp_grad, + ops::ActivationGradKernel); +REGISTER_OP_GPU_KERNEL(relu, + ops::ActivationKernel>); +REGISTER_OP_GPU_KERNEL( + relu_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + tanh, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + tanh_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + sqrt, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + sqrt_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + abs, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL(abs_grad, + ops::ActivationGradKernel); + +REGISTER_OP_GPU_KERNEL(reciprocal, + ops::ActivationKernel>); +REGISTER_OP_GPU_KERNEL( + reciprocal_grad, + ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + log, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + log_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL(square, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + square_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL(brelu, + ops::BReluKernel); +REGISTER_OP_GPU_KERNEL(brelu_grad, + ops::BReluGradKernel); + +REGISTER_OP_GPU_KERNEL(soft_relu, + ops::SoftReluKernel); +REGISTER_OP_GPU_KERNEL( + soft_relu_grad, ops::SoftReluGradKernel); + +REGISTER_OP_GPU_KERNEL(pow, ops::PowKernel); +REGISTER_OP_GPU_KERNEL(pow_grad, + ops::PowGradKernel); + +REGISTER_OP_GPU_KERNEL(stanh, + ops::STanhKernel); +REGISTER_OP_GPU_KERNEL(stanh_grad, + ops::STanhGradKernel); diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h new file mode 100644 index 0000000000000000000000000000000000000000..15f8afb4ba45cc989fe7576b82b8bf853b1df7de --- /dev/null +++ b/paddle/operators/activation_op.h @@ -0,0 +1,353 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +class ActivationKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + Functor functor; + functor(place, x, y); + } +}; + +template +class ActivationGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Input("Y"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + Functor functor; + functor(place, x, y, dy, dx); + } +}; + +// sigmoid(x) = 1 / (1 + exp(-x)) +template +struct SigmoidFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = static_cast(1) / (static_cast(1) + (-x).exp()); + } +}; + +template +struct SigmoidGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * y * (static_cast(1) - y); + } +}; + +// exp(x) = e^x +struct ExpFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.exp(); + } +}; + +struct ExpGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * y; + } +}; + +// relu(x) = max(x, 0) +template +struct ReluFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.cwiseMax(static_cast(0)); + } +}; + +template +struct ReluGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * (x > static_cast(0)).template cast(); + } +}; + +// tanh(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x)) +struct TanhFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.tanh(); + } +}; + +template +struct TanhGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * (static_cast(1) - y * y); + } +}; + +// sqrt(x) = x^(1/2) +struct SqrtFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.sqrt(); + } +}; + +template +struct SqrtGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + const Y y_conj = Eigen::numext::conj(y); + dx.device(d) = static_cast(0.5) * dy / y_conj; + } +}; + +// abs(x) = |x| +struct AbsFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.abs(); + } +}; + +struct AbsGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * x.sign(); + } +}; + +// reciprocal(x) = 1 / x +template +struct ReciprocalFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = static_cast(1) / x; + } +}; + +template +struct ReciprocalGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * static_cast(-1) * y * y; + } +}; + +// log(x) = natural logarithm of x +struct LogFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.log(); + } +}; + +template +struct LogGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * (static_cast(1) / x); + } +}; + +// square(x) = x^2 +struct SquareFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.square(); + } +}; + +template +struct SquareGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * static_cast(2) * x; + } +}; + +template +class BReluKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto t_min = static_cast(context.Attr("t_min")); + auto t_max = static_cast(context.Attr("t_max")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + y.device(place) = x.cwiseMax(t_min).cwiseMin(t_max); + } +}; + +template +class BReluGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto t_min = static_cast(context.Attr("t_min")); + auto t_max = static_cast(context.Attr("t_max")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + + dx.device(place) = dy * ((x > t_min) * (x < t_max)).template cast(); + } +}; + +template +class SoftReluKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto threshold = static_cast(context.Attr("threshold")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + auto temp = x.cwiseMax(-threshold).cwiseMin(threshold).eval(); + y.device(place) = (static_cast(1) + temp.exp()).log(); + } +}; + +template +class SoftReluGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Input("Y"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto threshold = static_cast(context.Attr("threshold")); + dX->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto dy = framework::EigenVector::Flatten(*dY); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + auto temp = ((x > -threshold) * (x < threshold)).template cast().eval(); + dx.device(place) = dy * (static_cast(1) - (-y).exp()) * temp; + } +}; + +template +class PowKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto factor = static_cast(context.Attr("factor")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + y.device(place) = x.pow(factor); + } +}; + +template +class PowGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto factor = static_cast(context.Attr("factor")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + + dx.device(place) = dy * factor * x.pow(factor - static_cast(1)); + } +}; + +template +class STanhKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto scale_a = static_cast(context.Attr("scale_a")); + auto scale_b = static_cast(context.Attr("scale_b")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + y.device(place) = scale_b * (scale_a * x).tanh(); + } +}; + +template +class STanhGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto scale_a = static_cast(context.Attr("scale_a")); + auto scale_b = static_cast(context.Attr("scale_b")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + + auto temp = (scale_a * x).tanh() * (scale_a * x).tanh(); + dx.device(place) = dy * scale_a * scale_b * (static_cast(1) - temp); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/sigmoid_op.cc b/paddle/operators/sigmoid_op.cc deleted file mode 100644 index 992b19965e0ca9ce7dba1b8b3c5b7780af06eb45..0000000000000000000000000000000000000000 --- a/paddle/operators/sigmoid_op.cc +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "paddle/operators/sigmoid_op.h" - -namespace paddle { -namespace operators { - -class SigmoidOp : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of SigmoidOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Y"), - "Output(Y) of SigmoidOp should not be null."); - - ctx.Output("Y")->Resize( - ctx.Input("X")->dims()); - } -}; - -class SigmoidOpMaker : public framework::OpProtoAndCheckerMaker { - public: - SigmoidOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "sigmoid input"); - AddOutput("Y", "sigmoid output"); - AddComment("Sigmoid function"); - } -}; - -class SigmoidOpGrad : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - ctx.Output(framework::GradVarName("X")) - ->Resize(ctx.Input("Y")->dims()); - } -}; - -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -REGISTER_OP(sigmoid, ops::SigmoidOp, ops::SigmoidOpMaker, sigmoid_grad, - ops::SigmoidOpGrad); -REGISTER_OP_CPU_KERNEL(sigmoid, - ops::SigmoidKernel); -REGISTER_OP_CPU_KERNEL( - sigmoid_grad, ops::SigmoidGradKernel); diff --git a/paddle/operators/sigmoid_op.cu b/paddle/operators/sigmoid_op.cu deleted file mode 100644 index 1a50dfe14a7b9e2614aadb7729de9f9e461e9905..0000000000000000000000000000000000000000 --- a/paddle/operators/sigmoid_op.cu +++ /dev/null @@ -1,23 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#define EIGEN_USE_GPU -#include "paddle/operators/sigmoid_op.h" - -namespace ops = paddle::operators; - -REGISTER_OP_GPU_KERNEL(sigmoid, - ops::SigmoidKernel); -REGISTER_OP_GPU_KERNEL( - sigmoid_grad, ops::SigmoidGradKernel); diff --git a/paddle/operators/sigmoid_op.h b/paddle/operators/sigmoid_op.h deleted file mode 100644 index b01a9b3f23283471f8846325075719ba0e75ed35..0000000000000000000000000000000000000000 --- a/paddle/operators/sigmoid_op.h +++ /dev/null @@ -1,62 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#pragma once -#include "paddle/framework/eigen.h" -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { - -using Tensor = framework::Tensor; -template -using EigenVector = framework::EigenVector; - -template -class SigmoidKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto input = context.Input("X"); - auto output = context.Output("Y"); - output->mutable_data(context.GetPlace()); - - // The clipping is used in Paddle's raw implenmention - auto X = EigenVector::Flatten(*input); - auto Y = EigenVector::Flatten(*output); - auto place = context.GetEigenDevice(); - - Y.device(place) = 1. / (1. + (-X).exp()); - } -}; - -template -class SigmoidGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto Y_t = context.Input("Y"); - auto dY_t = context.Input(framework::GradVarName("Y")); - auto dX_t = context.Output(framework::GradVarName("X")); - - dX_t->mutable_data(context.GetPlace()); - - auto dX = EigenVector::Flatten(*dX_t); - auto Y = EigenVector::Flatten(*Y_t); - auto dY = EigenVector::Flatten(*dY_t); - dX.device(context.GetEigenDevice()) = dY * Y * (1. - Y); - } -}; - -} // namespace operators -} // namespace paddle diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py new file mode 100644 index 0000000000000000000000000000000000000000..8f6d2be17758b7f6604d2db74fe466fb30695bd5 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_activation_op.py @@ -0,0 +1,223 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestExp(OpTest): + def setUp(self): + self.op_type = "exp" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.exp(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestSigmoid(OpTest): + def setUp(self): + self.op_type = "sigmoid" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': 1 / (1 + np.exp(-self.inputs['X']))} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.008) + + +class TestTanh(OpTest): + def setUp(self): + self.op_type = "tanh" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.tanh(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestSqrt(OpTest): + def setUp(self): + self.op_type = "sqrt" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.sqrt(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestAbs(OpTest): + def setUp(self): + self.op_type = "abs" + x = np.random.uniform(-1, 1, [4, 4]).astype("float32") + # Because we set delta = 0.005 in caculating numeric gradient, + # if x is too small, such as 0.002, x_neg will be -0.003 + # x_pos will be 0.007, so the numeric gradient is unaccurate. + # we should avoid this + x[np.abs(x) < 0.005] = 0.02 + self.inputs = {'X': x} + self.outputs = {'Y': np.abs(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestRelu(OpTest): + def setUp(self): + self.op_type = "relu" + x = np.random.uniform(-1, 1, [11, 17]).astype("float32") + # The same reason with TestAbs + x[np.abs(x) < 0.005] = 0.02 + self.inputs = {'X': x} + self.outputs = {'Y': np.maximum(self.inputs['X'], 0)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestBRelu(OpTest): + def setUp(self): + self.op_type = "brelu" + x = np.random.uniform(-1, 1, [4, 4]).astype("float32") + t_min = 1 + t_max = 4 + # The same with TestAbs + x[np.abs(x - t_min) < 0.005] = t_min + 0.02 + x[np.abs(x - t_max) < 0.005] = t_max + 0.02 + + self.inputs = {'X': x} + self.attrs = {'t_min': t_min, 't_max': t_max} + t = np.copy(x) + t[t < t_min] = t_min + t[t > t_max] = t_max + self.outputs = {'Y': t} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.02) + + +class TestSoftRelu(OpTest): + def setUp(self): + self.op_type = "soft_relu" + x = np.random.uniform(-3, 3, [4, 4]).astype("float32") + threshold = 2 + # The same reason with TestAbs + x[np.abs(x - threshold) < 0.005] = threshold + 0.02 + x[np.abs(x + threshold) < 0.005] = -threshold + 0.02 + self.inputs = {'X': x} + self.attrs = {'threshold': threshold} + t = np.copy(x) + t[t < -threshold] = -threshold + t[t > threshold] = threshold + self.outputs = {'Y': np.log((np.exp(t) + 1))} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.02) + + +class TestReciprocal(OpTest): + def setUp(self): + self.op_type = "reciprocal" + self.inputs = {'X': np.random.uniform(1, 2, [11, 17]).astype("float32")} + self.outputs = {'Y': np.reciprocal(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.01) + + +class TestLog(OpTest): + def setUp(self): + self.op_type = "log" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.log(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestSquare(OpTest): + def setUp(self): + self.op_type = "square" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.square(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestPow(OpTest): + def setUp(self): + self.op_type = "pow" + self.inputs = {'X': np.random.uniform(1, 2, [11, 17]).astype("float32")} + self.attrs = {'factor': 3} + self.outputs = {'Y': np.power(self.inputs['X'], 3)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.02) + + +class TestSTanh(OpTest): + def setUp(self): + self.op_type = "stanh" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + scale_a = 2.0 / 3.0 + scale_b = 1.7159 + self.attrs = {'scale_a': scale_a, 'scale_b': scale_b} + self.outputs = {'Y': scale_b * np.tanh(self.inputs['X'] * scale_a)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_sigmoid_op.py b/python/paddle/v2/framework/tests/test_sigmoid_op.py deleted file mode 100644 index d65d887db4af58c40e4e78fdbfd8e8ee668b7ee3..0000000000000000000000000000000000000000 --- a/python/paddle/v2/framework/tests/test_sigmoid_op.py +++ /dev/null @@ -1,22 +0,0 @@ -import unittest -import numpy as np -from op_test import OpTest - - -class TestSigmoidOp(OpTest): - def setUp(self): - self.op_type = "sigmoid" - self.inputs = { - 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") - } - self.outputs = {'Y': 1 / (1 + np.exp(-self.inputs['X']))} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(["X"], "Y", max_relative_error=0.007) - - -if __name__ == '__main__': - unittest.main()