From f196ad0210aadb715c12cafea2798ca235d84940 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Tue, 5 Sep 2017 11:33:40 +0000 Subject: [PATCH] Port fully connected operator, the FCOp c++ implementation and python unittest. --- paddle/operators/CMakeLists.txt | 5 +- paddle/operators/fc_op.cc | 107 ++++++++++++++++++ paddle/operators/scale_op.cc | 1 + paddle/pybind/CMakeLists.txt | 2 +- paddle/pybind/pybind.cc | 1 + .../paddle/v2/framework/tests/CMakeLists.txt | 1 + .../paddle/v2/framework/tests/test_fc_op.py | 30 +++++ 7 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 paddle/operators/fc_op.cc create mode 100644 python/paddle/v2/framework/tests/test_fc_op.py diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index e5efcccb0e..2a8beda2c8 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -47,17 +47,20 @@ endfunction() add_subdirectory(math) list(REMOVE_ITEM GENERAL_OPS + fc_op net_op minus_op mul_op recurrent_op scale_op) +op_library(fc_op SRCS fc_op.cc + DEPS mul_op rowwise_add_op scale_op softmax_op sigmoid_op) op_library(net_op SRCS net_op.cc) op_library(minus_op SRCS minus_op.cc minus_op.cu DEPS scale_op) op_library(mul_op SRCS mul_op.cc mul_op.cu DEPS math_function) op_library(recurrent_op SRCS recurrent_op.cc rnn/recurrent_op_utils.cc - DEPS framework_proto tensor operator net_op) + DEPS framework_proto tensor operator net_op) op_library(scale_op SRCS scale_op.cc scale_op.cu DEPS net_op) foreach(src ${GENERAL_OPS}) diff --git a/paddle/operators/fc_op.cc b/paddle/operators/fc_op.cc new file mode 100644 index 0000000000..ebf8908db7 --- /dev/null +++ b/paddle/operators/fc_op.cc @@ -0,0 +1,107 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_registry.h" +#include "paddle/operators/net_op.h" + +namespace paddle { +namespace operators { + +class FCOp : public NetOp { + public: + FCOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : NetOp(type, inputs, outputs, attrs) { + AppendOp(framework::OpRegistry::CreateOp( + "mul", {{"X", {Input("X")}}, {"Y", {Input("W")}}}, + {{"Out", {Output("mul_out")}}}, {})); + auto b = Input("b"); + if (b != framework::kEmptyVarName) { + AppendOp(framework::OpRegistry::CreateOp( + "rowwise_add", {{"X", {Output("mul_out")}}, {"b", {Input("b")}}}, + {{"Out", {Output("mul_out")}}}, {})); + } + + auto activation = GetAttr("activation"); + AppendOp(framework::OpRegistry::CreateOp( + activation, {{"X", {Output("mul_out")}}}, {{"Y", {Output("Y")}}}, {})); + CompleteAddOp(false); + } +}; + +class FCOpMaker : public framework::OpProtoAndCheckerMaker { + public: + FCOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The 2D input matrix of FC operator."); + AddInput("W", "The 2D weight matrix of FC operator."); + AddInput("b", "The 1D bias vector of FC operator"); + + AddOutput("Y", "The activated output matrix of FC operator"); + AddOutput("mul_out", "The non-actived output of FC operator, X * W + b") + .AsIntermediate(); + AddAttr("activation", "The activation type of FC operator.") + .SetDefault("identity") + .InEnum({"identity", "sigmoid", "softmax"}); + + AddComment(R"DOC( +Fully Connected Operator, known as Fully Connected Layer or Inner Product Layer +in Convolutional Neural Networks. Neurons in a fully connected layer have +full connections to all activations in the previous layer. +It computes an inner product of a set of +learned weights with a matrix multiplication followed by a bias offset +(optionally). + +Equation: + Y = Act(sum_n{X_i * W_i} + b) + +where X_i is a 2D matrix of size (M x K), usually M is the minibatch size and +K is the number of features. W_i is also a 2D matrix of size (K x N), +where N means the number of neurons in the fully connected layer. +b is a 1D vector of size N. Thus, the output Y is a 2D matrix of size (M x N). +Activation type can be set to `identity` (default), `sigmoid` or `softmax`. + + The config api is `paddle.v2.layer.fc`. +)DOC"); + } +}; + +class FCGradOp : public NetOp { + public: + FCGradOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : NetOp(type, inputs, outputs, attrs) { + auto y_grad = Input(framework::GradVarName("Y")); + auto mul_out_grad = Input(framework::GradVarName("mul_out")); + auto x_grad = Output(framework::GradVarName("X")); + auto w_grad = Output(framework::GradVarName("W")); + auto b_grad = Output(framework::GradVarName("b")); + + CompleteAddOp(false); + } +}; + +} // namespace operators +} // namespace paddle + +USE_OP(mul); +USE_OP(rowwise_add); +USE_NO_KERNEL_OP(identity); +USE_OP(sigmoid); +USE_OP(softmax); + +namespace ops = paddle::operators; +REGISTER_OP(fc, ops::FCOp, ops::FCOpMaker, fc_grad, ops::FCGradOp); diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc index 8e96a74c94..ffc2f02b0b 100644 --- a/paddle/operators/scale_op.cc +++ b/paddle/operators/scale_op.cc @@ -89,6 +89,7 @@ class IdentityOp : public NetOp { AppendOp(framework::OpRegistry::CreateOp( "scale", {{"X", {Input("X")}}}, {{"Out", {Output("Out")}}}, {{"scale", static_cast(1)}})); + CompleteAddOp(false); } }; diff --git a/paddle/pybind/CMakeLists.txt b/paddle/pybind/CMakeLists.txt index 0003005070..4f05406c7f 100644 --- a/paddle/pybind/CMakeLists.txt +++ b/paddle/pybind/CMakeLists.txt @@ -1,5 +1,5 @@ if(WITH_PYTHON) -cc_library(paddle_pybind SHARED + cc_library(paddle_pybind SHARED SRCS pybind.cc DEPS pybind python backward ${GLOB_OP_LIB}) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 6896422617..ff6bae8f85 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -45,6 +45,7 @@ USE_OP(uniform_random); USE_OP(lookup_table); USE_OP(scale); USE_NO_KERNEL_OP(identity); +USE_NO_KERNEL_OP(fc); USE_OP(minus); USE_CPU_ONLY_OP(gather); USE_CPU_ONLY_OP(scatter); diff --git a/python/paddle/v2/framework/tests/CMakeLists.txt b/python/paddle/v2/framework/tests/CMakeLists.txt index 661ebd8964..807ca2961e 100644 --- a/python/paddle/v2/framework/tests/CMakeLists.txt +++ b/python/paddle/v2/framework/tests/CMakeLists.txt @@ -16,6 +16,7 @@ py_test(test_cross_entropy_op SRCS test_cross_entropy_op.py) py_test(test_gather_op SRCS test_gather_op.py) py_test(test_scatter_op SRCS test_scatter_op.py) py_test(test_fill_zeros_like_op SRCS test_fill_zeros_like_op.py) +py_test(test_fc_op SRCS test_fc_op.py) py_test(gradient_checker SRCS gradient_checker.py) diff --git a/python/paddle/v2/framework/tests/test_fc_op.py b/python/paddle/v2/framework/tests/test_fc_op.py new file mode 100644 index 0000000000..bc469a5f47 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_fc_op.py @@ -0,0 +1,30 @@ +import unittest +import numpy as np +from gradient_checker import GradientChecker, create_op +from op_test_util import OpTestMeta + + +class TestFCOp(unittest.TestCase): + __metaclass__ = OpTestMeta + + def setUp(self): + self.type = "fc" + self.inputs = { + "X": np.random.random((32, 784)).astype("float32"), + "W": np.random.random((784, 1000)).astype("float32"), + "b": np.random.random(1000).astype("float32") + } + self.attrs = {"activation": "sigmoid"} + mul_out = np.dot(self.inputs["X"], self.inputs["W"]) + add_out = np.add(mul_out, self.inputs["b"]) + sigmoid_out = 1 / (1 + np.exp(-add_out)) + self.outputs = {"mul_out": add_out, "Y": sigmoid_out} + + +class TestFCGradOp(GradientChecker): + def test_normal(self): + print "nothing" + + +if __name__ == '__main__': + unittest.main() -- GitLab