diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index cf83d4cec312ac16366d84f897e7dc4784596ae8..6fcfe6de25737b66a2ea6c1a438636f072a513bb 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -106,7 +106,7 @@ enum DataType { message LoDTensorDesc { required DataType data_type = 1; - repeated int32 dims = 2; // [UNK, 640, 480] is saved as [-1, 640, 480] + repeated int64 dims = 2; // [UNK, 640, 480] is saved as [-1, 640, 480] optional int32 lod_level = 3 [ default = 0 ]; } diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 908a1f2fd0abe0aa4016c72dbcbc18dcc144232c..3c349637cdbe59b2cf9a1ea28e7715f4181f9293 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -72,20 +72,16 @@ bool operator==(const LoD& a, const LoD& b) { return true; } -void LoDTensor::SliceLevels(size_t level_begin, size_t level_end) { +void LoDTensor::ShrinkLevels(size_t level_begin, size_t level_end) { auto new_lod = framework::SliceLevels(lod_, level_begin, level_end); lod_ = new_lod; } -void LoDTensor::SliceInLevel(size_t level, size_t elem_begin, size_t elem_end) { - PADDLE_ENFORCE(level < NumLevels(), "level [%d] out of range [%d]", level, - NumLevels()); - PADDLE_ENFORCE(elem_begin < NumElements(level), - "element begin [%d] out of range [%d]", elem_begin, - NumElements(level)); - PADDLE_ENFORCE(elem_end < NumElements(level) + 1, - "element end [%d] out of range [%d]", elem_end, - NumElements(level)); +void LoDTensor::ShrinkInLevel(size_t level, size_t elem_begin, + size_t elem_end) { + PADDLE_ENFORCE_LT(level, NumLevels()); + PADDLE_ENFORCE_LT(elem_begin, NumElements(level)); + PADDLE_ENFORCE_LT(elem_end, NumElements(level) + 1); auto new_lod = framework::SliceInLevel(lod_, level, elem_begin, elem_end); lod_ = new_lod; diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index fac5cd20aa7f9db0792f8102bb442192ab1ad63f..82f58464264c6871b51251e0feae3d5ca076cd2b 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -89,15 +89,15 @@ class LoDTensor : public Tensor { } /* - * Slice of levels[level_begin:level_end] + * Shrink levels[level_begin:level_end] */ - void SliceLevels(size_t level_begin, size_t level_end); + void ShrinkLevels(size_t level_begin, size_t level_end); /* - * Slice of elements of a level, [elem_begin: elem_end] + * Shrink elements of a level, [elem_begin: elem_end] * @note: low performance in slice lod_. */ - void SliceInLevel(size_t level, size_t elem_begin, size_t elem_end); + void ShrinkInLevel(size_t level, size_t elem_begin, size_t elem_end); private: LoD lod_; diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index 7915326b27a22e9280e3f09d9bbfc2a58f46aff7..486b839738ec077545163bc47e6a97ef188c3c2f 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -56,11 +56,11 @@ TEST_F(LoDTensorTester, NumElements) { ASSERT_EQ(lod_tensor_.NumElements(2), 8UL); } -TEST_F(LoDTensorTester, SliceLevels) { +TEST_F(LoDTensorTester, ShrinkLevels) { // slice 1 level for (size_t level = 0; level < 3UL; ++level) { LoDTensor new_lod_tensor = lod_tensor_; - new_lod_tensor.SliceLevels(level, level + 1); + new_lod_tensor.ShrinkLevels(level, level + 1); ASSERT_EQ(new_lod_tensor.NumLevels(), 1UL); ASSERT_EQ(new_lod_tensor.NumElements(0), lod_tensor_.NumElements(level)); ASSERT_EQ(new_lod_tensor.data(), lod_tensor_.data()); @@ -68,7 +68,7 @@ TEST_F(LoDTensorTester, SliceLevels) { // slice 2 level for (size_t level = 0; level < 2UL; ++level) { LoDTensor new_lod_tensor = lod_tensor_; - new_lod_tensor.SliceLevels(level, level + 2); + new_lod_tensor.ShrinkLevels(level, level + 2); ASSERT_EQ(new_lod_tensor.NumLevels(), 2UL); ASSERT_EQ(new_lod_tensor.NumElements(0), lod_tensor_.NumElements(level)); ASSERT_EQ(new_lod_tensor.NumElements(1), @@ -77,10 +77,10 @@ TEST_F(LoDTensorTester, SliceLevels) { } } -TEST_F(LoDTensorTester, SliceInLevel) { +TEST_F(LoDTensorTester, ShrinkInLevel) { size_t level = 0; LoDTensor new_lod_tensor = lod_tensor_; - new_lod_tensor.SliceInLevel(level, 0, 2); + new_lod_tensor.ShrinkInLevel(level, 0, 2); EXPECT_EQ(new_lod_tensor.NumLevels(), 3UL); EXPECT_EQ(new_lod_tensor.NumElements(0), 2UL); EXPECT_EQ(new_lod_tensor.NumElements(1), 4UL); @@ -89,7 +89,7 @@ TEST_F(LoDTensorTester, SliceInLevel) { level = 1; new_lod_tensor = lod_tensor_; - new_lod_tensor.SliceInLevel(level, 0, 2); + new_lod_tensor.ShrinkInLevel(level, 0, 2); ASSERT_EQ(new_lod_tensor.NumLevels(), 2UL); ASSERT_EQ(new_lod_tensor.NumElements(0), 2UL); ASSERT_EQ(new_lod_tensor.NumElements(1), 4UL); diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index a75fd49a470fa5f9ce7a47ecd03c1084f9b48c27..fcbfc3e4377edd0ea55c8d4328c325fa18663001 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -60,8 +60,8 @@ std::string OperatorBase::Output(const std::string& name) const { const std::vector& OperatorBase::Outputs( const std::string& name) const { auto it = outputs_.find(name); - PADDLE_ENFORCE(it != outputs_.end(), "Op %s does not have output %s", type_, - name); + PADDLE_ENFORCE(it != outputs_.end(), "Op %s does not have output called %s", + type_, name); return it->second; } diff --git a/paddle/framework/tensor.h b/paddle/framework/tensor.h index 4b5a2ae523f2f7fde5445f0534cd99969ad9d59e..f040c09c089ec75c9773d752685be5e232e8f4b7 100644 --- a/paddle/framework/tensor.h +++ b/paddle/framework/tensor.h @@ -29,16 +29,19 @@ limitations under the License. */ namespace paddle { -namespace framework { +namespace pybind { namespace details { template struct CastToPyBufferImpl; } +} // namespace pybind + +namespace framework { class Tensor { public: template - friend struct details::CastToPyBufferImpl; + friend struct pybind::details::CastToPyBufferImpl; template friend struct EigenTensor; @@ -165,12 +168,6 @@ class Tensor { /*! points to dimensions of memory block. */ DDim dims_; - /** - * A cache of the number of elements in a tensor. - * Would be 0 for an uninitialized tensor. - */ - int64_t numel_; - /** * @brief A PlaceHolder may be shared by more than one tensor. * diff --git a/paddle/framework/tensor_impl.h b/paddle/framework/tensor_impl.h index 6d2c14f4c47afb755b1c74f6dc4dd10ab25ed191..a5405f9c31543b5733f9db923c2a6f8b968cfc2d 100644 --- a/paddle/framework/tensor_impl.h +++ b/paddle/framework/tensor_impl.h @@ -147,13 +147,12 @@ inline Tensor Tensor::Slice(const int& begin_idx, const int& end_idx) const { inline Tensor& Tensor::Resize(const DDim& dims) { dims_ = dims; - numel_ = product(dims_); return *this; } inline const DDim& Tensor::dims() const { return dims_; } -inline int64_t Tensor::numel() const { return numel_; } +inline int64_t Tensor::numel() const { return product(dims_); } template inline Tensor ReshapeToMatrix(const Tensor& src, int num_col_dims) { diff --git a/paddle/operators/elementwise_add_op.h b/paddle/operators/elementwise_add_op.h index 42a7a29e991adcce98c74c62bb359a2cc4a7c30d..9e9f1ffba6fb23f5394713c67aa4363b85717f50 100644 --- a/paddle/operators/elementwise_add_op.h +++ b/paddle/operators/elementwise_add_op.h @@ -12,6 +12,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +#pragma once + #include "paddle/operators/elementwise_op.h" namespace paddle { diff --git a/paddle/operators/elementwise_div_op.h b/paddle/operators/elementwise_div_op.h index 6ef60cdf8dfc01f7cf51bb68f18900effb444fca..9bd7c8ea548c46ec9b4c5a085e4e70d5dd162f3a 100644 --- a/paddle/operators/elementwise_div_op.h +++ b/paddle/operators/elementwise_div_op.h @@ -12,6 +12,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +#pragma once + #include "paddle/operators/elementwise_op.h" namespace paddle { diff --git a/paddle/operators/elementwise_sub_op.h b/paddle/operators/elementwise_sub_op.h index faa38cf4014e98b2a28f51b32585ca5c10289841..f6bc66cd0e1594a8bc7070e2f182401b92d1c88e 100644 --- a/paddle/operators/elementwise_sub_op.h +++ b/paddle/operators/elementwise_sub_op.h @@ -12,6 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +#pragma once #include "paddle/operators/elementwise_op.h" namespace paddle { diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index ad985839f5908d9235a4dbefc9b841362810114e..e7deaf9940699b938e4f36358c2c7f3ba15e918b 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -80,7 +80,6 @@ void RecurrentAlgorithm::CreateScopes(const Scope& scope) const { // Now all variables in scope must be created outside of op. PADDLE_ENFORCE_NOT_NULL(stepnet_); PADDLE_ENFORCE(!(*stepnet_)->Outputs().empty(), "stepnet_ op has no outputs"); - PADDLE_ENFORCE(!(*stepnet_)->Outputs().empty(), "net_op has no outputs"); if (seq_len_ > step_scopes->size()) { for (size_t i = step_scopes->size(); i < seq_len_; ++i) { @@ -129,8 +128,8 @@ const rnn::ArgumentName RecurrentOp::kArgName{ "memories", "pre_memories", "boot_memories"}; const rnn::ArgumentName RecurrentGradientOp::kArgName{ - "step_net", "step_scopes", "outlink@grad", "inlink@grad", - "memories", "pre_memories", "boot_memories@grad"}; + "step_net", "step_scopes@GRAD", "outlinks@GRAD", "inlinks@GRAD", + "memories", "pre_memories", "boot_memories@GRAD"}; RecurrentOp::RecurrentOp(const std::string& type, const framework::VariableNameMap& inputs, @@ -226,13 +225,13 @@ RecurrentGradientOp::RecurrentGradientOp( const framework::VariableNameMap& outputs, const framework::AttributeMap& attrs) : OperatorBase(type, inputs, outputs, attrs) { - rnn::InitArgument(kArgName, &arg_, *this); + rnn::InitArgument(kArgName, &arg_, *this, true /*is grad*/); alg_.Init(&arg_, &stepnet_); } } // namespace operators } // namespace paddle -REGISTER_OP_WITHOUT_GRADIENT( - recurrent, paddle::operators::RecurrentOp, - paddle::operators::RecurrentAlgorithmProtoAndCheckerMaker); +REGISTER_OP(recurrent, paddle::operators::RecurrentOp, + paddle::operators::RecurrentAlgorithmProtoAndCheckerMaker, + recurrent_grad, paddle::operators::RecurrentGradientOp); diff --git a/paddle/operators/recurrent_op.h b/paddle/operators/recurrent_op.h index 1033d657a3a8f96c8b3dae8dd93d3f1f6840b59b..ad4df9e55b91dbe89c34762945cd9edefde86e08 100644 --- a/paddle/operators/recurrent_op.h +++ b/paddle/operators/recurrent_op.h @@ -22,7 +22,7 @@ namespace paddle { namespace operators { // The sequence format in RecurrentOp is Tensor now. -// TODO(Yan Chunwei): +// TODO(Superjom) // 1. No-padding computing for sequences with indifinite length in one batch. // 2. Hierarchical RNN for sequence with sub-sequence. // 3. Internal Memory. @@ -177,6 +177,9 @@ class RecurrentGradientOp : public framework::OperatorBase { static const rnn::ArgumentName kArgName; + /* + * set a stepnet that is created according to a RecurrentOp's stepnet. + */ void set_stepnet(std::unique_ptr net) { stepnet_ = std::move(net); } diff --git a/paddle/operators/rnn/recurrent_op_utils.cc b/paddle/operators/rnn/recurrent_op_utils.cc index ca7219b26d83eb6b8db75a5ed9cd360c5ac1d5df..a767009d2366e20d2ebd35f562b8df7d408f2d4e 100644 --- a/paddle/operators/rnn/recurrent_op_utils.cc +++ b/paddle/operators/rnn/recurrent_op_utils.cc @@ -109,15 +109,14 @@ void LinkMemories(const std::vector& scopes, } void InitArgument(const ArgumentName& name, Argument* arg, - const framework::OperatorBase& op) { - arg->step_scopes = op.Output(name.step_scopes); - + const framework::OperatorBase& op, bool is_grad) { + arg->step_scopes = + is_grad ? op.Input(name.step_scopes) : op.Output(name.step_scopes); arg->inlinks = op.Inputs(name.inlinks); - arg->outlinks = op.Outputs(name.outlinks); - auto boot_memories = op.Inputs(name.boot_memories); - + auto boot_memories = + is_grad ? op.Outputs(name.boot_memories) : op.Inputs(name.boot_memories); // attributes auto memories = op.Attr>(name.memories); auto pre_memories = op.Attr>(name.pre_memories); diff --git a/paddle/operators/rnn/recurrent_op_utils.h b/paddle/operators/rnn/recurrent_op_utils.h index 7dafe5d0088c4c8bf2cad163654e7e4f28eebe2e..9c777f1e9067a3e2ceb9d23f7bf7d3c73343c91f 100644 --- a/paddle/operators/rnn/recurrent_op_utils.h +++ b/paddle/operators/rnn/recurrent_op_utils.h @@ -78,7 +78,7 @@ void LinkMemories(const std::vector& step_scopes, const int offset, bool infer_shape_mode); void InitArgument(const ArgumentName& name, Argument* arg, - const framework::OperatorBase& op); + const framework::OperatorBase& op, bool is_grad = false); } // namespace rnn } // namespace operators diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index c7009a604f60cda11434ad33b6c7d7caee1befdd..fbe074188e5870de4b00fa4fff733035739974ea 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -34,12 +34,7 @@ limitations under the License. */ namespace py = pybind11; namespace paddle { -namespace framework { - -using Tensor = framework::Tensor; -using LoDTensor = framework::LoDTensor; -using LoD = framework::LoD; - +namespace pybind { static size_t UniqueIntegerGenerator() { static std::atomic generator; return generator.fetch_add(1); @@ -56,6 +51,10 @@ bool IsCompileGPU() { PYBIND11_PLUGIN(core) { py::module m("core", "C++ core of PaddlePaddle"); + // using framework in this function. Since it is inside a function, it will + // not cause namespace pollution. + using namespace paddle::framework; // NOLINT + py::class_(m, "Tensor", py::buffer_protocol()) .def_buffer( [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); }) @@ -107,7 +106,7 @@ PYBIND11_PLUGIN(core) { #ifdef PADDLE_ONLY_CPU new (&instance) LoDTensor(lod); #else - paddle::framework::LoD new_lod; + LoD new_lod; new_lod.reserve(lod.size()); std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod)); new (&instance) LoDTensor(new_lod); @@ -118,7 +117,7 @@ PYBIND11_PLUGIN(core) { #ifdef PADDLE_ONLY_CPU self.set_lod(lod); #else - paddle::framework::LoD new_lod; + LoD new_lod; new_lod.reserve(lod.size()); std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod)); self.set_lod(new_lod); @@ -132,7 +131,7 @@ PYBIND11_PLUGIN(core) { std::vector> new_lod; new_lod.reserve(lod.size()); std::transform(lod.begin(), lod.end(), std::back_inserter(new_lod), - [](paddle::framework::Vector item) -> + [](Vector item) -> std::vector { std::vector v; v.reserve(item.size()); @@ -317,5 +316,5 @@ All parameter, weight, gradient are variables in Paddle. return m.ptr(); } -} // namespace framework +} // namespace pybind } // namespace paddle diff --git a/paddle/pybind/tensor_py.h b/paddle/pybind/tensor_py.h index 95171acf729a513e5c92d1e0cba15cb12b38561a..bcfba84a1aa6e646cf255dc4612dfda42169fc44 100644 --- a/paddle/pybind/tensor_py.h +++ b/paddle/pybind/tensor_py.h @@ -23,7 +23,7 @@ namespace py = pybind11; namespace paddle { -namespace framework { +namespace pybind { namespace details { diff --git a/python/paddle/v2/framework/tests/test_recurrent_op.py b/python/paddle/v2/framework/tests/test_recurrent_op.py index 79eda70021b76cd06e4c40740b1ca49476f4c503..cc3d4776e26a9dcaf9cf8403e0a1d0fca1d2ebae 100644 --- a/python/paddle/v2/framework/tests/test_recurrent_op.py +++ b/python/paddle/v2/framework/tests/test_recurrent_op.py @@ -3,6 +3,7 @@ import paddle.v2.framework.core as core import unittest import numpy as np from paddle.v2.framework.op import Operator, RecurrentOp +from op_test import get_numeric_gradient def py_sigmoid(x): @@ -47,7 +48,7 @@ class PySimpleRNN(object): else: pre_mem = self.h_boot xW = np.matmul(x, self.W) - hU = np.matmul(mem, self.U) + hU = np.matmul(pre_mem, self.U) sum = xW + hU self.mems[step_id] = py_sigmoid(sum) @@ -68,7 +69,7 @@ def create_tensor(scope, name, shape, np_data): return tensor -class TestRecurrentOp(unittest.TestCase): +class RecurrentOpTest(unittest.TestCase): ''' Test RNNOp @@ -158,6 +159,42 @@ class TestRecurrentOp(unittest.TestCase): print print 'py_output', py_output self.assertEqual(pd_output.shape, py_output.shape) + self.assertTrue(np.isclose(pd_output, py_output, rtol=0.1).all()) + + +class RecurrentGradientOpTest(unittest.TestCase): + def create_forward_op(self): + self.forward_op = RecurrentOp( + # inputs + inlinks=["x"], + boot_memories=["h_boot"], + step_net="stepnet", + # outputs + outlinks=["h"], + step_scopes="step_scopes", + # attributes + pre_memories=["h@pre"], + memories=["h@alias"]) + + # create a stepnet for RNN + stepnet = core.Net.create() + x_fc_op = Operator("mul", X="x@alias", Y="W", Out="Wx") + h_fc_op = Operator("mul", X="h@pre", Y="U", Out="Uh") + sum_op = Operator("add", X="Wx", Y="Uh", Out="sum") + sig_op = Operator("sigmoid", X="sum", Y="h@alias") + + for op in [x_fc_op, h_fc_op, sum_op, sig_op]: + stepnet.append_op(op) + stepnet.complete_add_op(True) + self.forward_op.set_stepnet(stepnet) + + def create_gradient_op(self): + a = set() + backward_op = core.RecurrentOp.backward(self.forward_op, a) + + def test_grad(self): + self.create_forward_op() + self.create_gradient_op() if __name__ == '__main__':