From 74849158e3613131460d05bec50dcafd276ed891 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 3 Nov 2017 13:55:32 -0700 Subject: [PATCH] Add LoDRankTable (#5349) * Add LoDRankTable LoD Rank Table stores the `level` of `lod` which is ordered by sequence length in descending order. It is useful when implement dynamic RNN and is shared by dynamic RNN memory, dynamic RNN slice input and dynamic RNN slice output operators. * Add InferVarType --- paddle/framework/CMakeLists.txt | 3 +- paddle/framework/executor.cc | 5 +- paddle/framework/framework.proto | 1 + paddle/framework/lod_rank_table.cc | 43 ++++++++++ paddle/framework/lod_rank_table.h | 55 +++++++++++++ paddle/framework/var_desc.h | 1 + paddle/operators/CMakeLists.txt | 2 + paddle/operators/lod_rank_table_op.cc | 80 +++++++++++++++++++ paddle/pybind/protobuf.cc | 3 +- paddle/pybind/pybind.cc | 13 +++ python/paddle/v2/framework/framework.py | 4 + python/paddle/v2/framework/layers.py | 13 +++ .../v2/framework/tests/test_lod_rank_table.py | 29 +++++++ 13 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 paddle/framework/lod_rank_table.cc create mode 100644 paddle/framework/lod_rank_table.h create mode 100644 paddle/operators/lod_rank_table_op.cc create mode 100644 python/paddle/v2/framework/tests/test_lod_rank_table.py diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 2be21e825ae..1afc5242081 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -45,8 +45,9 @@ add_custom_command(TARGET framework_py_proto POST_BUILD cc_library(backward SRCS backward.cc DEPS net_op) cc_test(backward_test SRCS backward_test.cc DEPS backward recurrent_op device_context fill_constant_op) +cc_library(lod_rank_table SRCS lod_rank_table.cc DEPS lod_tensor) -cc_library(executor SRCS executor.cc DEPS op_registry device_context scope framework_proto backward glog) +cc_library(executor SRCS executor.cc DEPS op_registry device_context scope framework_proto backward glog lod_rank_table) cc_library(prune SRCS prune.cc DEPS framework_proto) cc_test(prune_test SRCS prune_test.cc DEPS op_info prune recurrent_op device_context) diff --git a/paddle/framework/executor.cc b/paddle/framework/executor.cc index 52fefe4ea30..c1a009f131c 100644 --- a/paddle/framework/executor.cc +++ b/paddle/framework/executor.cc @@ -21,6 +21,7 @@ limitations under the License. */ #include #include "paddle/framework/feed_fetch_type.h" +#include "paddle/framework/lod_rank_table.h" #include "paddle/framework/lod_tensor.h" #include "paddle/framework/op_registry.h" #include "paddle/framework/scope.h" @@ -70,10 +71,12 @@ static void CreateTensor(Variable* var, VarDesc::VarType var_type) { var->GetMutable(); } else if (var_type == VarDesc::STEP_SCOPES) { var->GetMutable>(); + } else if (var_type == VarDesc::LOD_RANK_TABLE) { + var->GetMutable(); } else { PADDLE_THROW( "Variable type %d is not in " - "[LoDTensor, SelectedRows, FEED_MINIBATCH, FETCH_LIST]", + "[LoDTensor, SelectedRows, FEED_MINIBATCH, FETCH_LIST, LOD_RANK_TABLE]", var_type); } } diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index 8f2df3dc0e2..54ce461ce81 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -116,6 +116,7 @@ message VarDesc { FEED_MINIBATCH = 3; FETCH_LIST = 4; STEP_SCOPES = 5; + LOD_RANK_TABLE = 6; } required string name = 1; required VarType type = 2; diff --git a/paddle/framework/lod_rank_table.cc b/paddle/framework/lod_rank_table.cc new file mode 100644 index 00000000000..f9abf902a13 --- /dev/null +++ b/paddle/framework/lod_rank_table.cc @@ -0,0 +1,43 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/lod_rank_table.h" + +namespace paddle { +namespace framework { +void LoDRankTable::Reset(const LoD& lod, size_t level) { + this->coarse_lod_.clear(); + this->items_.clear(); + PADDLE_ENFORCE(level < lod.size(), + "Cannot rank lod since the level %d is less than lod size %d", + level, lod.size()); + coarse_lod_.reserve(level); + for (size_t i = 0; i < level; ++i) { + coarse_lod_.push_back(lod[i]); + } + auto& vec = lod[level]; + for (size_t i = 0; i < vec.size() - 1; ++i) { + TableItem item; + item.index = i; + item.length = vec[i + 1] - vec[i]; + items_.emplace_back(item); + } + std::sort(items_.begin(), items_.end(), + [](const TableItem& a, const TableItem& b) { + return a.length > b.length; + }); +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/lod_rank_table.h b/paddle/framework/lod_rank_table.h new file mode 100644 index 00000000000..9faa3a4d7bd --- /dev/null +++ b/paddle/framework/lod_rank_table.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/lod_tensor.h" + +namespace paddle { +namespace framework { + +// LoD Rank Table stores the `level` of `lod` which is ordered by sequence +// length in descending order. It is useful when implement dynamic RNN and is +// shared by dynamic RNN memory, dynamic RNN slice input and dynamic RNN slice +// output operators. +// +// The table item contains two element. The length of sequence and the index of +// sequence in that level. +// +// LoDRankTable also stores the coarse_lod, which is the lod information whose +// level is less than input level, in order to restore the output LoD +// information. +class LoDRankTable { + public: + struct TableItem { + size_t index; + size_t length; + }; + + LoDRankTable() {} + + void Reset(const LoD& lod, size_t level); + + const std::vector& items() const { return this->items_; } + + const LoD& coarse_lod() const { return this->coarse_lod_; } + + size_t level() const { return coarse_lod_.size(); } + + private: + LoD coarse_lod_; + std::vector items_; +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/var_desc.h b/paddle/framework/var_desc.h index 70daa20e8d9..5cf4608944c 100644 --- a/paddle/framework/var_desc.h +++ b/paddle/framework/var_desc.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include +#include "glog/logging.h" #include "paddle/framework/framework.pb.h" namespace paddle { diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index 81d92ec6f4f..13ebb0ad65f 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -141,6 +141,7 @@ set(DEPS_OPS pool_with_index_op nccl_op sequence_conv_op + lod_rank_table_op lstm_op) op_library(cond_op SRCS cond_op.cc DEPS framework_proto tensor operator net_op) @@ -149,6 +150,7 @@ op_library(softmax_with_cross_entropy_op DEPS cross_entropy softmax) op_library(sum_op DEPS net_op selected_rows_functor) op_library(pool_op DEPS pooling) op_library(pool_with_index_op DEPS pooling) +op_library(lod_rank_table_op SRCS lod_rank_table_op.cc DEPS lod_rank_table) if(WITH_GPU) op_library(nccl_op DEPS nccl_common) endif() diff --git a/paddle/operators/lod_rank_table_op.cc b/paddle/operators/lod_rank_table_op.cc new file mode 100644 index 00000000000..be198951c24 --- /dev/null +++ b/paddle/operators/lod_rank_table_op.cc @@ -0,0 +1,80 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include "paddle/framework/lod_rank_table.h" +#include "paddle/framework/op_registry.h" +namespace paddle { +namespace operators { + +class LoDRankTableOp : public framework::OperatorBase { + public: + LoDRankTableOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto x = scope.FindVar(Input("X"))->Get(); + auto *out = + scope.FindVar(Output("Out"))->GetMutable(); + out->Reset(x.lod(), static_cast(Attr("level"))); + } +}; + +class LoDRankTableOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + LoDRankTableOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(LoDTensor) input lod tensor, must contain lod information."); + AddOutput("Out", "(LoDRankTable) The rank table of specific level."); + AddAttr("level", "(int) the specific lod level to rank.") + .SetDefault(0) + .EqualGreaterThan(0); + AddComment(R"DOC(Create LoDRanTable by LoDTensor + +LoD Rank Table stores the `level` of `lod` which is ordered by sequence +length in descending order. It is useful when implement dynamic RNN and is +shared by dynamic RNN memory, dynamic RNN slice input and dynamic RNN slice +output operators. +)DOC"); + } +}; + +class LoDRankTableInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X"), "LoDRankTable must has input X"); + } +}; + +class LoDRankTableInferVarType : public framework::VarTypeInference { + public: + void operator()(const framework::OpDescBind &op_desc, + framework::BlockDescBind *block) const override { + for (auto &o : op_desc.Output("Out")) { + block->Var(o)->SetType(framework::VarDesc::LOD_RANK_TABLE); + } + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(lod_rank_table, paddle::operators::LoDRankTableOp, + paddle::operators::LoDRankTableOpProtoMaker, + paddle::operators::LoDRankTableInferShape, + paddle::operators::LoDRankTableInferVarType, + paddle::framework::EmptyGradOpMaker); diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index dcae426c7e2..d3fc544ec71 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -238,7 +238,8 @@ void BindVarDsec(py::module &m) { .value("SELECTED_ROWS", VarDesc::SELECTED_ROWS) .value("FEED_MINIBATCH", VarDesc::FEED_MINIBATCH) .value("FETCH_LIST", VarDesc::FETCH_LIST) - .value("STEP_SCOPES", VarDesc::STEP_SCOPES); + .value("STEP_SCOPES", VarDesc::STEP_SCOPES) + .value("LOD_RANK_TABLE", VarDesc::LOD_RANK_TABLE); } void BindOpDesc(py::module &m) { diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index aab08a759b0..78dc7943b36 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -21,6 +21,7 @@ limitations under the License. */ #include "paddle/framework/executor.h" #include "paddle/framework/feed_fetch_method.h" #include "paddle/framework/framework.pb.h" +#include "paddle/framework/lod_rank_table.h" #include "paddle/framework/lod_tensor.h" #include "paddle/framework/prune.h" #include "paddle/framework/selected_rows.h" @@ -224,6 +225,9 @@ All parameter, weight, gradient are variables in Paddle. return self.GetMutable(); }, py::return_value_policy::reference) + .def("get_lod_rank_table", + [](Variable &self) { return self.GetMutable(); }, + py::return_value_policy::reference) .def("get_selected_rows", [](Variable &self) -> SelectedRows * { return self.GetMutable(); @@ -492,6 +496,15 @@ All parameter, weight, gradient are variables in Paddle. BindVarDsec(m); BindOpDesc(m); + py::class_(m, "LodRankTable") + .def("items", [](framework::LoDRankTable &table) { + std::vector> res; + for (auto &item : table.items()) { + res.push_back({item.index, item.length}); + } + return res; + }); + m.def("op_support_gpu", OpSupportGPU); #ifdef PADDLE_WITH_CUDA m.def("get_cuda_device_count", platform::GetCUDADeviceCount); diff --git a/python/paddle/v2/framework/framework.py b/python/paddle/v2/framework/framework.py index a890bbf5985..4e737549c92 100644 --- a/python/paddle/v2/framework/framework.py +++ b/python/paddle/v2/framework/framework.py @@ -101,6 +101,10 @@ class Variable(object): def persistable(self): return self.desc.persistable() + @persistable.setter + def persistable(self, p): + self.desc.set_persistable(p) + @property def name(self): return self.desc.name() diff --git a/python/paddle/v2/framework/layers.py b/python/paddle/v2/framework/layers.py index a98b4e554f9..d6b5be94582 100644 --- a/python/paddle/v2/framework/layers.py +++ b/python/paddle/v2/framework/layers.py @@ -729,3 +729,16 @@ class StaticRNN(object): 'states': memories, 'step_block': rnn_block }) + + +def lod_rank_table(x, level=0, program=None): + helper = LayerHelper("lod_rank_table", **locals()) + table = helper.create_variable( + type=core.VarDesc.VarType.LOD_RANK_TABLE, + name=unique_name("lod_rank_table")) + helper.append_op( + type='lod_rank_table', + inputs={'X': x}, + outputs={'Out': table}, + attrs={'level': level}) + return table diff --git a/python/paddle/v2/framework/tests/test_lod_rank_table.py b/python/paddle/v2/framework/tests/test_lod_rank_table.py new file mode 100644 index 00000000000..f635e716bcd --- /dev/null +++ b/python/paddle/v2/framework/tests/test_lod_rank_table.py @@ -0,0 +1,29 @@ +from paddle.v2.framework.layers import lod_rank_table, data +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.framework import g_program +import paddle.v2.framework.core as core +import numpy +import unittest + + +class TestLoDRankTable(unittest.TestCase): + def test_lod_rank_table(self): + x = data(name='x', shape=[100]) + cpu = core.CPUPlace() + rank_table = lod_rank_table(x=x, level=1) + rank_table.persistable = True + exe = Executor(cpu) + scope = core.Scope() + + tensor = core.LoDTensor() + tensor.set(numpy.random.random(size=(17, 100)), cpu) + tensor.set_lod([[0, 1, 3], [0, 5, 6, 7], [0, 3, 4, 9, 10, 13, 16, 17]]) + + exe.run(g_program, scope=scope, feed={'x': tensor}) + var = scope.find_var(rank_table.name) + table = var.get_lod_rank_table() + self.assertEqual([(0, 5), (1, 1), (2, 1)], table.items()) + + +if __name__ == '__main__': + unittest.main() -- GitLab