diff --git a/doc/fluid/new_docs/user_guides/howto/debug/visualdl.md b/doc/fluid/new_docs/user_guides/howto/debug/visualdl.md index 84987ea5daee9abd0fe2fe71bdfde62ea3388ab5..99f8bee5ca1519ccf5d7c35ad2a64da4a8841ada 100644 --- a/doc/fluid/new_docs/user_guides/howto/debug/visualdl.md +++ b/doc/fluid/new_docs/user_guides/howto/debug/visualdl.md @@ -104,6 +104,7 @@ visualDL --logdir=scratch_log --port=8080 # 访问 http://127.0.0.1:8080 ``` +如果出现`TypeError: __init__() got an unexpected keyword argument 'file'`, 是因为protobuf不是3.5以上，运行`pip install --upgrade protobuf`就能解决。 如果在虚拟环境下仍然遇到安装问题，请尝试以下方法。 diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index 6b1aee5667e92a45a19226fa33058df5e0cb89e2..c2694144d708161a3bed214ceca745505656456f 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -43,6 +43,7 @@ paddle.fluid.Executor.run ArgSpec(args=['self', 'program', 'feed', 'fetch_list', paddle.fluid.global_scope ArgSpec(args=[], varargs=None, keywords=None, defaults=None) paddle.fluid.scope_guard ArgSpec(args=[], varargs='args', keywords='kwds', defaults=None) paddle.fluid.Trainer.__init__ ArgSpec(args=['self', 'train_func', 'optimizer_func', 'param_path', 'place', 'parallel', 'checkpoint_config'], varargs=None, keywords=None, defaults=(None, None, False, None)) +paddle.fluid.Trainer.save_inference_model ArgSpec(args=['self', 'param_path', 'feeded_var_names', 'target_var_indexes'], varargs=None, keywords=None, defaults=None) paddle.fluid.Trainer.save_params ArgSpec(args=['self', 'param_path'], varargs=None, keywords=None, defaults=None) paddle.fluid.Trainer.stop ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None) paddle.fluid.Trainer.test ArgSpec(args=['self', 'reader', 'feed_order'], varargs=None, keywords=None, defaults=None) @@ -376,7 +377,7 @@ paddle.fluid.optimizer.DecayedAdagradOptimizer.__init__ ArgSpec(args=['self', 'l paddle.fluid.optimizer.DecayedAdagradOptimizer.minimize ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set'], varargs=None, keywords=None, defaults=(None, None, None)) paddle.fluid.optimizer.FtrlOptimizer.__init__ ArgSpec(args=['self', 'learning_rate', 'l1', 'l2', 'lr_power'], varargs=None, keywords='kwargs', defaults=(0.0, 0.0, -0.5)) paddle.fluid.optimizer.FtrlOptimizer.minimize ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set'], varargs=None, keywords=None, defaults=(None, None, None)) -paddle.fluid.optimizer.RMSPropOptimizer.__init__ ArgSpec(args=['self', 'learning_rate', 'rho', 'epsilon', 'momentum'], varargs=None, keywords='kwargs', defaults=(0.95, 1e-06, 0.0)) +paddle.fluid.optimizer.RMSPropOptimizer.__init__ ArgSpec(args=['self', 'learning_rate', 'rho', 'epsilon', 'momentum', 'centered'], varargs=None, keywords='kwargs', defaults=(0.95, 1e-06, 0.0, False)) paddle.fluid.optimizer.RMSPropOptimizer.minimize ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set'], varargs=None, keywords=None, defaults=(None, None, None)) paddle.fluid.optimizer.AdadeltaOptimizer.__init__ ArgSpec(args=['self', 'learning_rate', 'epsilon', 'rho'], varargs=None, keywords='kwargs', defaults=(1e-06, 0.95)) paddle.fluid.optimizer.AdadeltaOptimizer.minimize ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set'], varargs=None, keywords=None, defaults=(None, None, None)) diff --git a/paddle/fluid/framework/ir/fc_lstm_fuse_pass.cc b/paddle/fluid/framework/ir/fc_lstm_fuse_pass.cc index 0d69dfa79aa26940f8f56f84b35ffed34f29f703..9512fd056e73836cdc34a9e409ab2d7809a6aff6 100644 --- a/paddle/fluid/framework/ir/fc_lstm_fuse_pass.cc +++ b/paddle/fluid/framework/ir/fc_lstm_fuse_pass.cc @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + #include "paddle/fluid/framework/ir/fc_lstm_fuse_pass.h" #include #include "paddle/fluid/framework/lod_tensor.h" diff --git a/paddle/fluid/framework/ir/graph_viz_pass.cc b/paddle/fluid/framework/ir/graph_viz_pass.cc index 4c7ffe69e933de3d52c8f762a1eeb73de17e0561..31ed98db72c8fd4af8c970861d386687962001ce 100644 --- a/paddle/fluid/framework/ir/graph_viz_pass.cc +++ b/paddle/fluid/framework/ir/graph_viz_pass.cc @@ -50,20 +50,37 @@ std::unique_ptr GraphVizPass::ApplyImpl( Dot dot; - std::vector op_attrs({Dot::Attr("style", "filled"), - Dot::Attr("shape", "box"), - Dot::Attr("fillcolor", "red")}); - std::vector var_attrs({Dot::Attr("style", "filled,rounded"), - // Dot::Attr("shape", "diamond"), - Dot::Attr("fillcolor", "yellow")}); - - std::vector marked_op_attrs({Dot::Attr("style", "filled"), - Dot::Attr("shape", "box"), - Dot::Attr("fillcolor", "lightgray")}); - std::vector marked_var_attrs( - {Dot::Attr("style", "filled,rounded"), - // Dot::Attr("shape", "diamond"), - Dot::Attr("fillcolor", "lightgray")}); + const std::vector op_attrs({ + Dot::Attr("style", "rounded,filled,bold"), // + Dot::Attr("shape", "box"), // + Dot::Attr("color", "#303A3A"), // + Dot::Attr("fontcolor", "#ffffff"), // + Dot::Attr("width", "1.3"), // + Dot::Attr("height", "0.84"), // + Dot::Attr("fontname", "Arial"), // + }); + const std::vector arg_attrs({ + Dot::Attr("shape", "box"), // + Dot::Attr("style", "rounded,filled,bold"), // + Dot::Attr("fontname", "Arial"), // + Dot::Attr("fillcolor", "#999999"), // + Dot::Attr("color", "#dddddd"), // + }); + + const std::vector param_attrs({ + Dot::Attr("shape", "box"), // + Dot::Attr("style", "rounded,filled,bold"), // + Dot::Attr("fontname", "Arial"), // + Dot::Attr("color", "#148b97"), // + Dot::Attr("fontcolor", "#ffffff"), // + }); + + const std::vector marked_op_attrs( + {Dot::Attr("style", "rounded,filled,bold"), Dot::Attr("shape", "box"), + Dot::Attr("fillcolor", "yellow")}); + const std::vector marked_var_attrs( + {Dot::Attr("style", "filled,rounded"), Dot::Attr("shape", "box"), + Dot::Attr("fillcolor", "yellow")}); auto marked_nodes = ConsumeMarkedNodes(graph.get()); // Create nodes @@ -74,9 +91,17 @@ std::unique_ptr GraphVizPass::ApplyImpl( marked_nodes.count(n) ? marked_op_attrs : op_attrs; dot.AddNode(node_id, attr, node_id); } else if (n->IsVar()) { - decltype(op_attrs) attr = - marked_nodes.count(n) ? marked_var_attrs : var_attrs; - dot.AddNode(node_id, attr, node_id); + decltype(op_attrs)* attr; + if (marked_nodes.count(n)) { + attr = &marked_var_attrs; + } else if (const_cast(n)->Var() && + const_cast(n)->Var()->Persistable()) { + attr = ¶m_attrs; + } else { + attr = &arg_attrs; + } + + dot.AddNode(node_id, *attr, node_id); } node2dot[n] = node_id; } diff --git a/paddle/fluid/inference/analysis/CMakeLists.txt b/paddle/fluid/inference/analysis/CMakeLists.txt index f2e18a461fd221252e4a10262a13bc8e942f5988..226645058e85da55b47e26efe5a199f50aef3847 100644 --- a/paddle/fluid/inference/analysis/CMakeLists.txt +++ b/paddle/fluid/inference/analysis/CMakeLists.txt @@ -105,6 +105,6 @@ if (NOT EXISTS ${TEXT_CLASSIFICATION_INSTALL_DIR} AND WITH_TESTING AND WITH_INFE inference_download_and_uncompress(${TEXT_CLASSIFICATION_INSTALL_DIR} ${TEXT_CLASSIFICATION_MODEL_URL} "text-classification-Senta.tar.gz") endif() -inference_analysis_test(test_text_classification SRCS test_text_classification.cc +inference_analysis_test(test_text_classification SRCS analyzer_text_classification_tester.cc EXTRA_DEPS paddle_inference_api paddle_fluid_api analysis_predictor ARGS --infer_model=${TEXT_CLASSIFICATION_INSTALL_DIR}/text-classification-Senta) diff --git a/paddle/fluid/inference/analysis/test_text_classification.cc b/paddle/fluid/inference/analysis/analyzer_text_classification_tester.cc similarity index 94% rename from paddle/fluid/inference/analysis/test_text_classification.cc rename to paddle/fluid/inference/analysis/analyzer_text_classification_tester.cc index 191b41e98834f4a63cee9aa54b422896266152da..f5d938c1cca5c7343724176dd624c3efde51ceab 100644 --- a/paddle/fluid/inference/analysis/test_text_classification.cc +++ b/paddle/fluid/inference/analysis/analyzer_text_classification_tester.cc @@ -12,14 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "paddle/fluid/inference/analysis/analyzer.h" #include #include // use glog instead of PADDLE_ENFORCE to avoid importing other paddle header files. #include #include "paddle/fluid/framework/ir/pass.h" -#include "paddle/fluid/inference/analysis/analyzer.h" #include "paddle/fluid/inference/analysis/ut_helper.h" #include "paddle/fluid/inference/api/helper.h" #include "paddle/fluid/inference/api/paddle_inference_api.h" +#include "paddle/fluid/inference/api/paddle_inference_pass.h" +#include "paddle/fluid/inference/api/timer.h" DEFINE_string(infer_model, "", "Directory of the inference model."); DEFINE_string(infer_data, "", "Path of the dataset."); @@ -86,10 +88,3 @@ TEST(text_classification, basic) { Main(FLAGS_batch_size); } } // namespace inference } // namespace paddle - -USE_PASS(fc_fuse_pass); -USE_PASS(seq_concat_fc_fuse_pass); -USE_PASS(fc_lstm_fuse_pass); -USE_PASS(graph_viz_pass); -USE_PASS(infer_clean_graph_pass); -USE_PASS(attention_lstm_fuse_pass); diff --git a/paddle/fluid/inference/api/CMakeLists.txt b/paddle/fluid/inference/api/CMakeLists.txt index ea00bf364951b0a4304b380df492d00e84451136..6b8278a0395c9ae71e32337d9735409de7ba0c96 100644 --- a/paddle/fluid/inference/api/CMakeLists.txt +++ b/paddle/fluid/inference/api/CMakeLists.txt @@ -44,19 +44,7 @@ function(inference_api_test TARGET_NAME) endfunction(inference_api_test) cc_library(paddle_inference_api SRCS api.cc api_impl.cc helper.cc DEPS lod_tensor) -cc_library(analysis_predictor SRCS analysis_predictor.cc DEPS paddle_inference_api - analysis - ir_pass_manager - pass - fc_fuse_pass - fc_lstm_fuse_pass - seq_concat_fc_fuse_pass - graph_viz_pass - infer_clean_graph_pass - graph_pattern_detector - infer_clean_graph_pass - attention_lstm_fuse_pass - ) +cc_library(analysis_predictor SRCS analysis_predictor.cc DEPS paddle_inference_api analysis) cc_test(test_paddle_inference_api SRCS api_tester.cc diff --git a/paddle/fluid/operators/fake_quantize_op.cu b/paddle/fluid/operators/fake_quantize_op.cu index 7c65d6dba7d67b5d31720bae1f4877dd22210138..a0ff6396210c2b3a7f8bd6b9f274b875d7fd4933 100644 --- a/paddle/fluid/operators/fake_quantize_op.cu +++ b/paddle/fluid/operators/fake_quantize_op.cu @@ -119,7 +119,8 @@ struct FindRangeAbsMaxFunctor { const framework::Tensor& last_scale, const framework::Tensor& iter, const int window_size, framework::Tensor* scales_arr, framework::Tensor* out_scale) { - auto& gpu_place = boost::get(ctx.GetPlace()); + const auto gpu_place = boost::get(ctx.GetPlace()); + T* scale_arr = scales_arr->mutable_data(gpu_place); T* out_scale_data = out_scale->mutable_data(gpu_place); diff --git a/paddle/fluid/operators/flatten_op.cc b/paddle/fluid/operators/flatten_op.cc index fdda01381e117cecffb2a05f8399f3ad82a46339..8e80dc0e641c443923076c31e269689b5bc134a7 100644 --- a/paddle/fluid/operators/flatten_op.cc +++ b/paddle/fluid/operators/flatten_op.cc @@ -157,6 +157,116 @@ class FlattenGradOp : public framework::OperatorBase { } }; +// FIXME(zcd): flatten2 adds an intermediate output(XShape) based on flatten, +// the XShape is used to carry the shape and lod of X which will be used in +// flatten_grad, in this way, the framework can reuse the memory of X +// immediately the flatten2_op is finished. +// Considering compatibility issues, we could not fix flatten2_op +class Flatten2OpInferShape : public FlattenOpInferShape { + public: + void operator()(framework::InferShapeContext *ctx) const override { + FlattenOpInferShape::operator()(ctx); + PADDLE_ENFORCE(ctx->HasOutput("XShape"), + "Output (XShape) of Flatten op should not be null."); + const auto &in_dims = ctx->GetInputDim("X"); + std::vector xshape_dims(in_dims.size() + 1); + xshape_dims[0] = 0; + for (int i = 0; i < in_dims.size(); ++i) { + xshape_dims[i + 1] = in_dims[i]; + } + ctx->SetOutputDim("XShape", framework::make_ddim(xshape_dims)); + ctx->ShareLoD("X", "XShape"); + } +}; + +class Flatten2Op : public framework::OperatorBase { + public: + using OperatorBase::OperatorBase; + + private: + void RunImpl(const framework::Scope &scope, + const platform::Place &place) const override { + auto &axis = Attr("axis"); + auto in_dims = + scope.FindVar(Input("X"))->Get().dims(); + const auto &out_dims = FlattenOpInferShape::GetOutputShape(axis, in_dims); + + framework::AttributeMap attrs; + attrs["shape"] = out_dims; + attrs["inplace"] = false; + // Invoke Reshape Op + auto reshape_op = framework::OpRegistry::CreateOp( + "reshape2", {{"X", {Input("X")}}, {"Shape", {}}}, + {{"Out", {Output("Out")}}, {"XShape", {Output("XShape")}}}, attrs); + reshape_op->Run(scope, place); + } +}; + +class Flatten2OpMaker : public FlattenOpMaker { + public: + void Make() override { + FlattenOpMaker::Make(); + AddOutput("XShape", + "XShape is just used to store the shape and lod of X, which will " + "be used in FlattenGradOp.") + .AsIntermediate(); + } +}; + +class Flatten2GradOpMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); + grad_op->SetType("flatten2_grad"); + grad_op->SetInput("XShape", Output("XShape")); + grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +class Flatten2GradInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("XShape"), + "Input(XShape) shouldn't be null."); + PADDLE_ENFORCE(context->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) shouldn't be null."); + auto xshape_dims = context->GetInputDim("XShape"); + auto x_dims = framework::slice_ddim(xshape_dims, 1, xshape_dims.size()); + context->SetOutputDim(framework::GradVarName("X"), x_dims); + context->ShareLoD("XShape", framework::GradVarName("X")); + } +}; + +class Flatten2GradOp : public framework::OperatorBase { + public: + using OperatorBase::OperatorBase; + + private: + void RunImpl(const framework::Scope &scope, + const platform::Place &place) const override { + auto dx_name = Output(framework::GradVarName("X")); + auto dout_name = Input(framework::GradVarName("Out")); + auto xshape_name = Input("XShape"); + auto xshape_dims = + scope.FindVar(xshape_name)->Get().dims(); + auto x_dims = framework::slice_ddim(xshape_dims, 1, xshape_dims.size()); + + framework::AttributeMap attrs; + attrs["shape"] = framework::vectorize2int(x_dims); + attrs["inplace"] = false; + + auto reshape_op = framework::OpRegistry::CreateOp( + "reshape2", {{"X", {dout_name}}, {"Shape", {}}}, + {{"Out", {dx_name}}, {"XShape", {xshape_name}}}, attrs); + reshape_op->Run(scope, place); + } +}; + } // namespace operators } // namespace paddle @@ -167,3 +277,8 @@ REGISTER_OPERATOR(flatten, ops::FlattenOp, ops::FlattenOpMaker, ops::FlattenOpInferShape, paddle::framework::DefaultGradOpDescMaker); REGISTER_OPERATOR(flatten_grad, ops::FlattenGradOp, ops::FlattenGradInferShape); + +REGISTER_OPERATOR(flatten2, ops::Flatten2Op, ops::Flatten2OpMaker, + ops::Flatten2OpInferShape, ops::Flatten2GradOpMaker); +REGISTER_OPERATOR(flatten2_grad, ops::Flatten2GradOp, + ops::Flatten2GradInferShape); diff --git a/paddle/fluid/operators/fusion_gru_op.cc b/paddle/fluid/operators/fusion_gru_op.cc index 582c75872ab2818cdf834f9a46278db1d6f91d54..916f84cb4a78c3721cb67bd3cf8d3759a8eaf1bf 100644 --- a/paddle/fluid/operators/fusion_gru_op.cc +++ b/paddle/fluid/operators/fusion_gru_op.cc @@ -30,14 +30,7 @@ void FusionGRUOp::InferShape(framework::InferShapeContext* ctx) const { "Input(WeightX) of GRU should not be null."); PADDLE_ENFORCE(ctx->HasInput("WeightH"), "Input(WeightH) of GRU should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("XX"), "Output(XX) of GRU should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("ReorderedH0"), - "Output(ReorderedH0) of GRU should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("BatchedInput"), - "Output(BatchedInput) of GRU should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("BatchedOut"), - "Output(BatchedOut) of GRU should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Hidden"), "Output(Hidden) of GRU should not be null."); @@ -80,15 +73,20 @@ void FusionGRUOp::InferShape(framework::InferShapeContext* ctx) const { } framework::DDim out_dims({x_dims[0], frame_size}); ctx->SetOutputDim("Hidden", out_dims); - ctx->SetOutputDim("BatchedInput", {x_dims[0], wx_dims[1]}); - ctx->SetOutputDim("BatchedOut", out_dims); ctx->ShareLoD("X", "Hidden"); - int xx_width; if (ctx->Attrs().Get("use_seq")) { xx_width = wx_dims[1]; } else { xx_width = x_dims[1] > wx_dims[1] ? wx_dims[1] : x_dims[1]; + PADDLE_ENFORCE(ctx->HasOutput("ReorderedH0"), + "Output(ReorderedH0) of GRU should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("BatchedInput"), + "Output(BatchedInput) of GRU should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("BatchedOut"), + "Output(BatchedOut) of GRU should not be null."); + ctx->SetOutputDim("BatchedInput", {x_dims[0], wx_dims[1]}); + ctx->SetOutputDim("BatchedOut", out_dims); } ctx->SetOutputDim("XX", {x_dims[0], xx_width}); ctx->ShareLoD("X", "XX"); diff --git a/paddle/fluid/operators/fusion_lstm_op.cc b/paddle/fluid/operators/fusion_lstm_op.cc index 104e160e2d7069ec247cc51e927ce8824f1b69e8..ef23ab3f981786d33567619ad0194d21f31bdc8e 100644 --- a/paddle/fluid/operators/fusion_lstm_op.cc +++ b/paddle/fluid/operators/fusion_lstm_op.cc @@ -38,16 +38,6 @@ void FusionLSTMOp::InferShape(framework::InferShapeContext* ctx) const { "Output(Hidden) of LSTM should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Cell"), "Output(Cell) of LSTM should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("BatchedInput"), - "Output(BatchedInput) of LSTM should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("BatchedHidden"), - "Output(BatchedHidden) of LSTM should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("BatchedCell"), - "Output(BatchedCell) of LSTM should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("ReorderedH0"), - "Output(ReorderedH0) of LSTM should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("ReorderedC0"), - "Output(ReorderedC0) of LSTM should not be null."); auto x_dims = ctx->GetInputDim("X"); PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank must be 2."); @@ -88,28 +78,36 @@ void FusionLSTMOp::InferShape(framework::InferShapeContext* ctx) const { PADDLE_ENFORCE_EQ(b_dims.size(), 2, "The rank of Input(Bias) should be 2."); PADDLE_ENFORCE_EQ(b_dims[0], 1, "The first dimension of Input(Bias) should be 1."); - - auto use_peepholes = ctx->Attrs().Get("use_peepholes"); - PADDLE_ENFORCE_EQ(b_dims[1], (use_peepholes ? 7 : 4) * frame_size, - "The second dimension of Input(Bias) should be " - "7 * %d if enable peepholes connection or" - "4 * %d if disable peepholes", - frame_size, frame_size); + PADDLE_ENFORCE_EQ( + b_dims[1], (ctx->Attrs().Get("use_peepholes") ? 7 : 4) * frame_size, + "The second dimension of Input(Bias) should be " + "7 * %d if enable peepholes connection or" + "4 * %d if disable peepholes", + frame_size, frame_size); framework::DDim out_dims({x_dims[0], frame_size}); ctx->SetOutputDim("Hidden", out_dims); ctx->SetOutputDim("Cell", out_dims); - ctx->SetOutputDim("BatchedInput", {x_dims[0], wx_dims[1]}); - ctx->SetOutputDim("BatchedHidden", out_dims); - ctx->SetOutputDim("BatchedCell", out_dims); ctx->ShareLoD("X", "Hidden"); ctx->ShareLoD("X", "Cell"); - int xx_width; if (ctx->Attrs().Get("use_seq")) { xx_width = wx_dims[1]; } else { xx_width = x_dims[1] > wx_dims[1] ? wx_dims[1] : x_dims[1]; + PADDLE_ENFORCE(ctx->HasOutput("BatchedInput"), + "Output(BatchedInput) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("BatchedHidden"), + "Output(BatchedHidden) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("BatchedCell"), + "Output(BatchedCell) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("ReorderedH0"), + "Output(ReorderedH0) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("ReorderedC0"), + "Output(ReorderedC0) of LSTM should not be null."); + ctx->SetOutputDim("BatchedInput", {x_dims[0], wx_dims[1]}); + ctx->SetOutputDim("BatchedHidden", out_dims); + ctx->SetOutputDim("BatchedCell", out_dims); } ctx->SetOutputDim("XX", {x_dims[0], xx_width}); ctx->ShareLoD("X", "XX"); @@ -232,18 +230,18 @@ class FuisonLSTMKernel : public framework::OpKernel { act_cand = act_functor(act_cand_str); \ } -#define INIT_BASE_INPUT_OUTPUT \ - auto* x = ctx.Input("X"); \ - auto* h0 = ctx.Input("H0"); \ - auto* c0 = ctx.Input("C0"); \ - auto* wx = ctx.Input("WeightX"); \ - auto* wh = ctx.Input("WeightH"); \ - auto* bias = ctx.Input("Bias"); \ - auto* xx = ctx.Output("XX"); \ - auto* hidden_out = ctx.Output("Hidden"); \ - auto* cell_out = ctx.Output("Cell"); \ - bool use_peepholes = ctx.Attr("use_peepholes"); \ - bool is_reverse = ctx.Attr("is_reverse"); +#define INIT_BASE_INPUT_OUTPUT \ + auto* x = ctx.Input("X"); \ + auto* h0 = ctx.Input("H0"); \ + auto* c0 = ctx.Input("C0"); \ + auto* wx = ctx.Input("WeightX"); \ + auto* wh = ctx.Input("WeightH"); \ + auto* bias = ctx.Input("Bias"); \ + auto* xx = ctx.Output("XX"); \ + auto* hidden_out = ctx.Output("Hidden"); \ + auto* cell_out = ctx.Output("Cell"); \ + bool is_reverse = ctx.Attr("is_reverse"); \ + bool use_peepholes = ctx.Attr("use_peepholes"); #define INIT_BASE_SIZES \ auto x_dims = x->dims(); /* T x M*/ \ @@ -254,172 +252,183 @@ class FuisonLSTMKernel : public framework::OpKernel { const int D3 = D * 3; \ const int D4 = wh_dims[1]; +#define INIT_BASE_INPUT_DATAS \ + const T* x_data = x->data(); \ + const T* wx_data = wx->data(); \ + const T* wh_data = wh->data(); \ + /* diagonal weight*/ \ + const T* wc_data = bias->data() + D4; \ + /* for peephole only*/ \ + Tensor checked_cell; \ + T* checked_cell_data = nullptr; \ + auto place = ctx.GetPlace(); \ + if (use_peepholes) { \ + /* w_ic * Ct-1, w_fc * Ct-1 ; w_oc * Ct => ih*/ \ + checked_cell_data = checked_cell.mutable_data({2, D}, place); \ + } + +/// Compute LSTM +#define GEMM_WH_ADDON(bs, prev, out) \ + blas.GEMM(CblasNoTrans, CblasNoTrans, bs, D4, D, static_cast(1), prev, D, \ + wh_data, D4, static_cast(1), out, D4) + +// gates: W_ch, W_ih, W_fh, W_oh +#define GET_Ct(ct_1, gates, ct) \ + /* C_t = C_t-1 * fgated + cand_gated * igated*/ \ + act_cand(D, gates, gates); \ + blas.VMUL(D, gates, gates + D, gates + D); \ + blas.VMUL(D, ct_1, gates + D2, gates + D2); \ + blas.VADD(D, gates + D, gates + D2, ct) + +#define GET_Ht(ct, gates, ht) \ + /* H_t = act_cell(C_t) * ogated */ \ + act_cell(D, ct, gates + D2); \ + blas.VMUL(D, gates + D2, gates + D3, ht) + +#define GET_Ct_NOH0C0(gates, ct) \ + /* C_t = igated * cgated*/ \ + act_gate(D, gates + D, gates + D); \ + act_cand(D, gates, gates); \ + blas.VMUL(D, gates, gates + D, ct) + +#define COMPUTE_CtHt_NOH0C0(gates, ct, ht) \ + GET_Ct_NOH0C0(gates, ct); \ + act_gate(D, gates + D3, gates + D3); \ + GET_Ht(ct, gates, ht) + +#define COMPUTE_CtHt_PEEPHOLE_NOH0C0(gates, ct, ht) \ + GET_Ct_NOH0C0(gates, ct); \ + /* get outgated, put W_oc * C_t on igated */ \ + blas.VMUL(D, wc_data + D2, ct, gates + D); \ + blas.VADD(D, gates + D, gates + D3, gates + D3); \ + act_gate(D, gates + D3, gates + D3); \ + GET_Ht(ct, gates, ht) + +#define COMPUTE_CtHt(gates, ct_1, ct, ht) \ + act_gate(D3, gates + D, gates + D); \ + GET_Ct(ct_1, gates, ct); \ + GET_Ht(ct, gates, ht) + +#define COMPUTE_CtHt_PEEPHOLE(gates, ct_1, ct, ht) \ + /* get fgated and igated*/ \ + blas.VMUL(D, wc_data, ct_1, checked_cell_data); \ + blas.VMUL(D, wc_data + D, ct_1, checked_cell_data + D); \ + blas.VADD(D2, checked_cell_data, gates + D, gates + D); \ + act_gate(D2, gates + D, gates + D); \ + GET_Ct(ct_1, gates, ct); \ + /* get ogated*/ \ + blas.VMUL(D, wc_data + D2, ct, gates + D); \ + blas.VADD(D, gates + D, gates + D3, gates + D3); \ + act_gate(D, gates + D3, gates + D3); \ + GET_Ht(ct, gates, ht) + void SeqCompute(const framework::ExecutionContext& ctx) const { using DeviceContext = paddle::platform::CPUDeviceContext; INIT_BASE_INPUT_OUTPUT INIT_BASE_SIZES INIT_VEC_FUNC + INIT_BASE_INPUT_DATAS auto x_lod = x->lod(); const int total_T = x_dims[0]; - const int N = x_lod[0].size() - 1; // batch size - - const T* x_data = x->data(); + const int N = x_lod[0].size() - 1; const T* h0_data = h0 ? h0->data() : nullptr; const T* c0_data = c0 ? c0->data() : nullptr; - const T* bias_data = bias->data(); - const T* wc_data = bias_data + D4; // w_ic, w_fc, w_oc - const T* wx_data = wx->data(); - const T* wh_data = wh->data(); - - T* xx_data = xx->mutable_data(ctx.GetPlace()); - T* hidden_out_data = hidden_out->mutable_data(ctx.GetPlace()); - T* cell_out_data = cell_out->mutable_data(ctx.GetPlace()); - - // use local variable - framework::DDim check_dims({3, D}); - Tensor checked_cell; // w_ic * Ct-1, w_fc * Ct-1, w_oc * Ct - auto checked_cell_data = - checked_cell.mutable_data(check_dims, ctx.GetPlace()); - + T* xx_data = xx->mutable_data(place); + T* h_out_data = hidden_out->mutable_data(place); + T* c_out_data = cell_out->mutable_data(place); auto blas = math::GetBlas(ctx); math::FCCompute(blas, total_T, D4, M, x_data, wx_data, xx_data, bias->data()); + int xx_offset = D4; int gate_offset = D; if (is_reverse) { const int offset = (total_T - 1) * D; xx_data = xx_data + offset * 4; - hidden_out_data = hidden_out_data + offset; - cell_out_data = cell_out_data + offset; + h_out_data = h_out_data + offset; + c_out_data = c_out_data + offset; xx_offset = -D4; gate_offset = -D; } - auto move_step = [&]() { - xx_data = xx_data + xx_offset; - hidden_out_data = hidden_out_data + gate_offset; - cell_out_data = cell_out_data + gate_offset; - }; - - for (int i = 0; i < N; ++i) { - int bid = is_reverse ? N - 1 - i : i; - int seq_len = x_lod[0][bid + 1] - x_lod[0][bid]; - const T* prev_c_data = nullptr; - const T* prev_h_data = nullptr; - - int tstart = 0; - if (h0_data) { - prev_h_data = h0_data + bid * D; - prev_c_data = c0_data + bid * D; - } else { - // If step == 0 and there is no initialized hidden state, that is to say - // the H0 is zeros. Then W_h * H_t-1 can be skipped - - // ~C_t - act_cand(D, xx_data, xx_data); - if (use_peepholes) { - // I_t, F_t - act_gate(D2, xx_data + D, xx_data + D); - } else { - // I_t, F_t, O_t - act_gate(D3, xx_data + D, xx_data + D); - } - // C_t = I_t * ~C_t - blas.VMUL(D, xx_data, xx_data + D, cell_out_data); - - if (use_peepholes) { - // + W_oc * C_t for peephole connection - blas.VMUL(D, wc_data + D2, cell_out_data, checked_cell_data + D2); - blas.VADD(D, xx_data + D3, checked_cell_data + D2, xx_data + D3); - // O_t - act_gate(D, xx_data + D3, xx_data + D3); - } - - // hidden out= act_state(cellout) * outgate - act_cell(D, cell_out_data, xx_data + D2); - // H_t = O_t * act_state(C_t) - blas.VMUL(D, xx_data + D2, xx_data + D3, hidden_out_data); - - // prev - prev_h_data = hidden_out_data; - prev_c_data = cell_out_data; - - tstart = 1; - move_step(); - } - - for (int step = tstart; step < seq_len; ++step) { - // + W_h * H_t-1 - blas.GEMM(CblasNoTrans, CblasNoTrans, 1, D4, D, static_cast(1), - prev_h_data, D, wh_data, D4, static_cast(1), xx_data, D4); +#define MOVE_ONE_STEP \ + prev_h_data = h_out_data; \ + prev_c_data = c_out_data; \ + xx_data = xx_data + xx_offset; \ + h_out_data = h_out_data + gate_offset; \ + c_out_data = c_out_data + gate_offset + +#define PROCESS_H0C0_DEFINES \ + int bid = is_reverse ? N - 1 - i : i; \ + int seq_len = x_lod[0][bid + 1] - x_lod[0][bid]; \ + const T* prev_c_data = nullptr; \ + const T* prev_h_data = nullptr; \ + int tstart = 0 + +#define PROCESS_H0C0_PEEPHOLE \ + PROCESS_H0C0_DEFINES; \ + if (h0_data) { \ + prev_h_data = h0_data + bid * D; \ + prev_c_data = c0_data + bid * D; \ + } else { \ + COMPUTE_CtHt_PEEPHOLE_NOH0C0(xx_data, c_out_data, h_out_data); \ + MOVE_ONE_STEP; \ + tstart = 1; \ + } - // ~C_t - act_cand(D, xx_data, xx_data); +#define PROCESS_H0C0 \ + PROCESS_H0C0_DEFINES; \ + if (h0_data) { \ + prev_h_data = h0_data + bid * D; \ + prev_c_data = c0_data + bid * D; \ + } else { \ + COMPUTE_CtHt_NOH0C0(xx_data, c_out_data, h_out_data); \ + MOVE_ONE_STEP; \ + tstart = 1; \ + } - if (use_peepholes) { - // + W_ic|W_fc * C_t-1 for peephole connection - blas.VMUL(D, wc_data, prev_c_data, checked_cell_data); - blas.VMUL(D, wc_data + D, prev_c_data, checked_cell_data + D); - blas.VADD(D2, xx_data + D, checked_cell_data, xx_data + D); - // I_t, F_t - act_gate(D2, xx_data + D, xx_data + D); - } else { - // I_t, F_t, O_t - act_gate(D3, xx_data + D, xx_data + D); + if (use_peepholes) { + for (int i = 0; i < N; ++i) { + PROCESS_H0C0_PEEPHOLE + for (int step = tstart; step < seq_len; ++step) { + GEMM_WH_ADDON(1, prev_h_data, xx_data); + COMPUTE_CtHt_PEEPHOLE(xx_data, prev_c_data, c_out_data, h_out_data); + MOVE_ONE_STEP; } - - // F_t * C_t-1 - blas.VMUL(D, xx_data + D2, prev_c_data, xx_data + D2); - // I_t * ~C_t - blas.VMUL(D, xx_data, xx_data + D, xx_data + D); - // C_t = F_t * C_t-1 + I_t * ~C_t - blas.VADD(D, xx_data + D, xx_data + D2, cell_out_data); - - if (use_peepholes) { - // + W_oc * C_t for peephole connection - blas.VMUL(D, wc_data + D2, cell_out_data, checked_cell_data + D2); - blas.VADD(D, xx_data + D3, checked_cell_data + D2, xx_data + D3); - // O_t - act_gate(D, xx_data + D3, xx_data + D3); + } + } else { + for (int i = 0; i < N; ++i) { + PROCESS_H0C0 + for (int step = tstart; step < seq_len; ++step) { + GEMM_WH_ADDON(1, prev_h_data, xx_data); + COMPUTE_CtHt(xx_data, prev_c_data, c_out_data, h_out_data); + MOVE_ONE_STEP; } - - // hidden out= act_state(cellout) * outgate - act_cell(D, cell_out_data, xx_data + D2); - // H_t = O_t * act_state(C_t) - blas.VMUL(D, xx_data + D2, xx_data + D3, hidden_out_data); - - // prev - prev_h_data = hidden_out_data; - prev_c_data = cell_out_data; - - move_step(); - } // for each step in batch - } // for each batch + } + } +#undef PROCESS_H0C0_DEFINES +#undef PROCESS_H0C0_PEEPHOLE +#undef PROCESS_H0C0 +#undef MOVE_ONE_STEP } void BatchCompute(const framework::ExecutionContext& ctx) const { using DeviceContext = platform::CPUDeviceContext; INIT_BASE_INPUT_OUTPUT - if (x->lod()[0].size() == 2) { // batch size == 1 + if (x->lod()[0].size() == 2) { SeqCompute(ctx); return; } INIT_BASE_SIZES INIT_VEC_FUNC + INIT_BASE_INPUT_DATAS auto* reordered_h0 = ctx.Output("ReorderedH0"); auto* reordered_c0 = ctx.Output("ReorderedC0"); auto* batched_input = ctx.Output("BatchedInput"); auto* batched_c_out = ctx.Output("BatchedCell"); auto* batched_h_out = ctx.Output("BatchedHidden"); - - const T* x_data = x->data(); - const T* wx_data = wx->data(); - const T* wh_data = wh->data(); - const T* bias_data = bias->data(); - const T* wc_data = bias_data + D4; // w_ic, w_fc, w_oc - auto place = ctx.GetPlace(); T* xx_data = xx->mutable_data(place); T* batched_input_data = batched_input->mutable_data(place); T* batched_c_out_data = batched_c_out->mutable_data(place); @@ -427,12 +436,6 @@ class FuisonLSTMKernel : public framework::OpKernel { hidden_out->mutable_data(place); cell_out->mutable_data(place); - // use local variable - framework::DDim check_dims({3, D}); - Tensor checked_cell; // w_ic * Ct-1, w_fc * Ct-1, w_oc * Ct - auto checked_cell_data = - checked_cell.mutable_data(check_dims, ctx.GetPlace()); - math::LoDTensor2BatchFunctor to_batch; auto& dev_ctx = ctx.template device_context(); auto blas = math::GetBlas(dev_ctx); @@ -454,27 +457,17 @@ class FuisonLSTMKernel : public framework::OpKernel { reordered_h0->Resize({max_bs, D}); reordered_c0->Resize({max_bs, D}); - T* prev_batch_h_data = nullptr; - T* prev_batch_c_data = nullptr; - T* cur_batch_in_data = batched_input_data; - T* cur_batch_h_out_data = batched_h_out_data; - T* cur_batch_c_out_data = batched_c_out_data; - - auto move_step = [&](int bs) { - cur_batch_in_data += bs * D4; - cur_batch_c_out_data += bs * D; - cur_batch_h_out_data += bs * D; - }; - int tstart = 0; + T* prev_h_data = nullptr; + T* prev_c_data = nullptr; if (h0) { // reorder h0, c0 T* reordered_h0_data = reordered_h0->mutable_data(place); T* reordered_c0_data = reordered_c0->mutable_data(place); const T* h0_data = h0->data(); const T* c0_data = c0->data(); - prev_batch_h_data = reordered_h0_data; - prev_batch_c_data = reordered_c0_data; + prev_h_data = reordered_h0_data; + prev_c_data = reordered_c0_data; size_t sz = sizeof(T) * D; for (int i = 0; i < max_bs; ++i) { std::memcpy(reordered_h0_data, h0_data + seq_order[i] * D, sz); @@ -483,123 +476,80 @@ class FuisonLSTMKernel : public framework::OpKernel { reordered_c0_data += D; } } else { - // Compute with no H0/C0 - T* cur_in_data = cur_batch_in_data; - T* cur_c_out_data = cur_batch_c_out_data; - T* cur_h_out_data = cur_batch_h_out_data; - - // If step == 0 and there is no initialized hidden state, that is to say - // the H0 is zeros. Then W_h * H_t-1 can be skiped - - for (int i = 0; i < max_bs; ++i) { // iterate each data in 1st batch - // ~C_t - act_cand(D, cur_in_data, cur_in_data); - - if (use_peepholes) { - // I_t, F_t - act_gate(D2, cur_in_data + D, cur_in_data + D); - } else { - // I_t, F_t, O_t - act_gate(D3, cur_in_data + D, cur_in_data + D); - } - - // C_t = I_t * ~C_t - blas.VMUL(D, cur_in_data, cur_in_data + D, cur_c_out_data); - + // compute without h0, c0 + T* cur_in_data = batched_input_data; + T* cur_h_out_data = batched_h_out_data; + T* cur_c_out_data = batched_c_out_data; + for (int i = 0; i < max_bs; ++i) { + GET_Ct_NOH0C0(cur_in_data, cur_c_out_data); if (use_peepholes) { - // + W_oc * C_t for peephole connection - blas.VMUL(D, wc_data + D2, cur_c_out_data, checked_cell_data + D2); - blas.VADD(D, cur_in_data + D3, checked_cell_data + D2, - cur_in_data + D3); - // O_t - act_gate(D, cur_in_data + D3, cur_in_data + D3); + blas.VMUL(D, wc_data + D2, cur_c_out_data, cur_in_data + D); + blas.VADD(D, cur_in_data + D, cur_in_data + D3, cur_in_data + D3); } - - // hidden out= act_state(cellout) * outgate - act_cell(D, cur_c_out_data, cur_in_data + D2); - // H_t = O_t * act_state(C_t) - blas.VMUL(D, cur_in_data + D2, cur_in_data + D3, cur_h_out_data); - - // move to next data in the same batch + act_gate(D, cur_in_data + D3, cur_in_data + D3); + GET_Ht(cur_c_out_data, cur_in_data, cur_h_out_data); cur_in_data += D4; cur_c_out_data += D; cur_h_out_data += D; } - - // move to data for next timestep - prev_batch_h_data = cur_batch_h_out_data; - prev_batch_c_data = cur_batch_c_out_data; - move_step(max_bs); tstart = 1; + prev_h_data = batched_h_out_data; + prev_c_data = batched_c_out_data; } - const auto& batch_starts = batched_lod[0]; const int max_seq_len = batch_starts.size() - 1; - for (int step = tstart; step < max_seq_len; ++step) { - const int cur_bs = batch_starts[step + 1] - batch_starts[step]; - // + W_h * H_t-1 - blas.GEMM(CblasNoTrans, CblasNoTrans, cur_bs, D4, D, static_cast(1), - prev_batch_h_data, D, wh_data, D4, static_cast(1), - cur_batch_in_data, D4); - - T* cur_in_data = cur_batch_in_data; - T* cur_c_out_data = cur_batch_c_out_data; - T* cur_h_out_data = cur_batch_h_out_data; - T* prev_c_data = prev_batch_c_data; // NULL if no C0 in step0 - T* prev_h_data = prev_batch_h_data; // NULL if no H0 in step0 - auto next_data_in_batch = [&]() { - cur_in_data += D4; - cur_c_out_data += D; - cur_h_out_data += D; - prev_c_data = prev_c_data ? prev_c_data + D : nullptr; - prev_h_data = prev_h_data ? prev_h_data + D : nullptr; - }; - - for (int i = 0; i < cur_bs; ++i) { // iterate each data in same batch - // ~C_t - act_cand(D, cur_in_data, cur_in_data); - - if (use_peepholes) { - // + W_ic|W_fc * C_t-1 for peephole connection - blas.VMUL(D, wc_data, prev_c_data, checked_cell_data); - blas.VMUL(D, wc_data + D, prev_c_data, checked_cell_data + D); - blas.VADD(D2, cur_in_data + D, checked_cell_data, cur_in_data + D); - // I_t, F_t - act_gate(D2, cur_in_data + D, cur_in_data + D); - } else { - // I_t, F_t, O_t - act_gate(D3, cur_in_data + D, cur_in_data + D); + const int offset = tstart * max_bs * D; + batched_input_data = batched_input_data + offset * 4; + batched_h_out_data = batched_h_out_data + offset; + batched_c_out_data = batched_c_out_data + offset; + +#define DEFINE_CUR \ + T* cur_in_data = batched_input_data; \ + T* cur_prev_c_data = prev_c_data; \ + T* cur_c_out_data = batched_c_out_data; \ + T* cur_h_out_data = batched_h_out_data + +#define MOVE_ONE_BATCH \ + cur_in_data += D4; \ + cur_prev_c_data += D; \ + cur_c_out_data += D; \ + cur_h_out_data += D + +#define MOVE_ONE_STEP \ + prev_c_data = batched_c_out_data; \ + prev_h_data = batched_h_out_data; \ + batched_c_out_data = cur_c_out_data; \ + batched_h_out_data = cur_h_out_data; \ + batched_input_data = cur_in_data + + if (use_peepholes) { + for (int step = tstart; step < max_seq_len; ++step) { + const int cur_bs = batch_starts[step + 1] - batch_starts[step]; + GEMM_WH_ADDON(cur_bs, prev_h_data, batched_input_data); + DEFINE_CUR; + for (int i = 0; i < cur_bs; ++i) { + COMPUTE_CtHt_PEEPHOLE(cur_in_data, cur_prev_c_data, cur_c_out_data, + cur_h_out_data); + MOVE_ONE_BATCH; } - - // F_t * C_t-1 - blas.VMUL(D, cur_in_data + D2, prev_c_data, cur_in_data + D2); - // I_t * ~C_t - blas.VMUL(D, cur_in_data, cur_in_data + D, cur_in_data + D); - // C_t = F_t * C_t-1 + I_t * ~C_t - blas.VADD(D, cur_in_data + D, cur_in_data + D2, cur_c_out_data); - - if (use_peepholes) { - // + W_oc * C_t for peephole connection - blas.VMUL(D, wc_data + D2, cur_c_out_data, checked_cell_data + D2); - blas.VADD(D, cur_in_data + D3, checked_cell_data + D2, - cur_in_data + D3); - // O_t - act_gate(D, cur_in_data + D3, cur_in_data + D3); + MOVE_ONE_STEP; + } + } else { + for (int step = tstart; step < max_seq_len; ++step) { + const int cur_bs = batch_starts[step + 1] - batch_starts[step]; + GEMM_WH_ADDON(cur_bs, prev_h_data, batched_input_data); + DEFINE_CUR; + for (int i = 0; i < cur_bs; ++i) { + COMPUTE_CtHt(cur_in_data, cur_prev_c_data, cur_c_out_data, + cur_h_out_data); + MOVE_ONE_BATCH; } - - // hidden out= act_state(cellout) * outgate - act_cell(D, cur_c_out_data, cur_in_data + D2); - // H_t = O_t * act_state(C_t) - blas.VMUL(D, cur_in_data + D2, cur_in_data + D3, cur_h_out_data); - - // move to next data in same batch - next_data_in_batch(); + MOVE_ONE_STEP; } - // move to data for next timestep - prev_batch_h_data = cur_batch_h_out_data; - prev_batch_c_data = cur_batch_c_out_data; - move_step(cur_bs); } +#undef MOVE_ONE_STEP +#undef MOVE_ONE_BATCH +#undef DEFINE_CUR math::Batch2LoDTensorFunctor to_seq; batched_h_out->set_lod(batched_lod); @@ -615,6 +565,16 @@ class FuisonLSTMKernel : public framework::OpKernel { BatchCompute(ctx); } } + +#undef COMPUTE_CtHt_PEEPHOLE +#undef COMPUTE_CtHt +#undef GET_Ct_NOH0C0 +#undef COMPUTE_CtHt_NOH0C0 +#undef COMPUTE_CtHt_PEEPHOLE_NOH0C0 +#undef GET_Ht +#undef GET_Ct +#undef GEMM_WH_ADDON +#undef INIT_BASE_INPUT_DATAS #undef INIT_BASE_SIZES #undef INIT_BASE_INPUT_OUTPUT #undef INIT_VEC_FUNC diff --git a/paddle/fluid/operators/layer_norm_op.cu b/paddle/fluid/operators/layer_norm_op.cu index 0886c41a1b582881faf24f5531d414db4e4db71c..22343d7724b2f0dc01bff8c2274e3dd914bf70ef 100644 --- a/paddle/fluid/operators/layer_norm_op.cu +++ b/paddle/fluid/operators/layer_norm_op.cu @@ -67,27 +67,27 @@ template __global__ void LayerNormForward(const T *x, const T *scale, const T *bias, T *y, T *mean, T *var, float epsilon, int feature_size) { - using BlockReduce = cub::BlockReduce, BlockDim>; + using BlockReduce = cub::BlockReduce, BlockDim>; __shared__ typename BlockReduce::TempStorage temp_storage; int beg_idx = blockIdx.x * feature_size + threadIdx.x; int end_idx = (blockIdx.x + 1) * feature_size; // Step 1: Reduce to calculate mean and var - T mean_val = static_cast(0); - T var_val = static_cast(0); + double mean_val = 0; + double var_val = 0; for (int i = beg_idx; i < end_idx; i += BlockDim) { T tmp = x[i]; mean_val += tmp; var_val += (tmp * tmp); } auto pair = BlockReduce(temp_storage) - .Reduce(PairForLayerNorm(mean_val, var_val), - PairForLayerNormAddFunctor()); + .Reduce(PairForLayerNorm(mean_val, var_val), + PairForLayerNormAddFunctor()); if (threadIdx.x == 0) { auto tmp = pair.first_ / feature_size; - mean[blockIdx.x] = tmp; - var[blockIdx.x] = pair.second_ / feature_size - tmp * tmp; + mean[blockIdx.x] = static_cast(tmp); + var[blockIdx.x] = static_cast(pair.second_ / feature_size - tmp * tmp); } __syncthreads(); mean_val = mean[blockIdx.x]; diff --git a/paddle/fluid/operators/reshape_op.cc b/paddle/fluid/operators/reshape_op.cc index a1dfe39c3a4f84f5e4aaa2306813a7decf0e49ea..d72f85f2c44db2fa887732cfc05e1376a6a79e4a 100644 --- a/paddle/fluid/operators/reshape_op.cc +++ b/paddle/fluid/operators/reshape_op.cc @@ -246,6 +246,88 @@ class ReshapeGradKernel { } }; +// FIXME(zcd): reshape2 adds an intermediate output(XShape) based on reshape, +// the XShape is used to carry the shape and lod of X which will be used in +// reshape_grad, in this way, the framework can reuse the memory of X +// immediately the reshape_op is finished. +// Considering compatibility issues, we could not fix reshape_op +class Reshape2Op : public ReshapeOp { + public: + Reshape2Op(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ReshapeOp(type, inputs, outputs, attrs) {} + + void InferShape(framework::InferShapeContext *ctx) const override { + ReshapeOp::InferShape(ctx); + PADDLE_ENFORCE(ctx->HasOutput("XShape"), + "Output(XShape) of ReshapeOp should not be null."); + const auto &x_dims = ctx->GetInputDim("X"); + std::vector xshape_dims(x_dims.size() + 1); + xshape_dims[0] = 0; + for (int i = 0; i < x_dims.size(); ++i) { + xshape_dims[i + 1] = x_dims[i]; + } + ctx->SetOutputDim("XShape", framework::make_ddim(xshape_dims)); + ctx->ShareLoD("X", /*->*/ "XShape"); + } +}; + +class Reshape2OpMaker : public ReshapeOpMaker { + public: + void Make() override { + ReshapeOpMaker::Make(); + AddOutput("XShape", + "XShape is just used to store the shape and lod of X, which will " + "be used in FlattenGradOp.") + .AsIntermediate(); + } +}; + +class Reshape2GradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); + grad_op->SetType("reshape2_grad"); + grad_op->SetInput("XShape", Output("XShape")); + grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +class Reshape2GradOp : public framework::OperatorWithKernel { + public: + Reshape2GradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorWithKernel(type, inputs, outputs, attrs) {} + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("XShape"), "Input(XShape) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) shouldn't be null."); + auto xshape_dims = ctx->GetInputDim("XShape"); + auto x_dims = framework::slice_ddim(xshape_dims, 1, xshape_dims.size()); + ctx->SetOutputDim(framework::GradVarName("X"), x_dims); + ctx->ShareLoD("XShape", framework::GradVarName("X")); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType( + ctx.Input(framework::GradVarName("Out")) + ->type()), + ctx.device_context()); + } +}; + } // namespace operators } // namespace paddle namespace ops = paddle::operators; @@ -261,6 +343,17 @@ REGISTER_OP_CPU_KERNEL_FUNCTOR(reshape_grad, float, ops::ReshapeGradKernel, ops::ReshapeGradKernel, int64_t, ops::ReshapeGradKernel); +REGISTER_OPERATOR(reshape2, ops::Reshape2Op, ops::Reshape2OpMaker, + ops::Reshape2GradMaker); +REGISTER_OPERATOR(reshape2_grad, ops::Reshape2GradOp); +REGISTER_OP_CPU_KERNEL_FUNCTOR(reshape2, float, ops::ReshapeKernel, double, + ops::ReshapeKernel, int, ops::ReshapeKernel, + int64_t, ops::ReshapeKernel); +REGISTER_OP_CPU_KERNEL_FUNCTOR(reshape2_grad, float, ops::ReshapeGradKernel, + double, ops::ReshapeGradKernel, int, + ops::ReshapeGradKernel, int64_t, + ops::ReshapeGradKernel); + #ifdef PADDLE_WITH_CUDA REGISTER_OP_CUDA_KERNEL_FUNCTOR(reshape, float, ops::ReshapeKernel, double, ops::ReshapeKernel, int, ops::ReshapeKernel, @@ -269,4 +362,11 @@ REGISTER_OP_CUDA_KERNEL_FUNCTOR(reshape_grad, float, ops::ReshapeGradKernel, double, ops::ReshapeGradKernel, int, ops::ReshapeGradKernel, int64_t, ops::ReshapeGradKernel); +REGISTER_OP_CUDA_KERNEL_FUNCTOR(reshape2, float, ops::ReshapeKernel, double, + ops::ReshapeKernel, int, ops::ReshapeKernel, + int64_t, ops::ReshapeKernel); +REGISTER_OP_CUDA_KERNEL_FUNCTOR(reshape2_grad, float, ops::ReshapeGradKernel, + double, ops::ReshapeGradKernel, int, + ops::ReshapeGradKernel, int64_t, + ops::ReshapeGradKernel); #endif diff --git a/paddle/fluid/operators/rmsprop_op.cc b/paddle/fluid/operators/rmsprop_op.cc index 919ebe48ca38040274bd2052b95ef96eccff4db6..2f773f222e50a440801b06a4fd997bf237b34772 100644 --- a/paddle/fluid/operators/rmsprop_op.cc +++ b/paddle/fluid/operators/rmsprop_op.cc @@ -36,9 +36,13 @@ class RmspropOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasOutput("ParamOut"), "Output(param_out) of RmspropOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("MomentOut"), - "Output(Momentum_out) of RmspropOp should not be null."); + "Output(MomentOut) of RmspropOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("MeanSquareOut"), "Output(MeanSquareOut) of RmspropOp should not be null."); + if (ctx->Attrs().Get("centered")) { + PADDLE_ENFORCE(ctx->HasOutput("MeanGradOut"), + "Output(MeanGradOut) of RmspropOp should not be null."); + } auto param_dim = ctx->GetInputDim("Param"); PADDLE_ENFORCE_EQ( @@ -58,6 +62,9 @@ class RmspropOp : public framework::OperatorWithKernel { ctx->SetOutputDim("ParamOut", param_dim); ctx->SetOutputDim("MomentOut", param_dim); ctx->SetOutputDim("MeanSquareOut", param_dim); + if (ctx->Attrs().Get("centered")) { + ctx->SetOutputDim("MeanGradOut", param_dim); + } } }; @@ -70,6 +77,10 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("MeanSquare", "(Tensor, default Tensor)" " The mean square value that gets updated."); + AddInput("MeanGrad", + "(Tensor, default Tensor)" + " The moving average of gradient") + .AsDispensable(); AddInput("LearningRate", "(Tensor, default Tensor) " "The learning rate should be a tensor of size 1."); @@ -82,6 +93,8 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("ParamOut", "(Tensor) Output updated parameter value."); AddOutput("MomentOut", "(Tensor) Output updated moment."); AddOutput("MeanSquareOut", "(Tensor) Output Mean squared updated value."); + AddOutput("MeanGradOut", + "(Tensor) Output moving average of gradient updated value."); AddAttr("epsilon", "(float, default 1e-10) Constant " @@ -93,6 +106,8 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault(0.9f); AddAttr("momentum", "(float, default 0.0) Constant value.") .SetDefault(0.0f); + AddAttr("centered", "(bool, default false) use centered rmsprop.") + .SetDefault(false); AddComment(R"DOC( Rmsprop Optimizer. @@ -103,6 +118,14 @@ MomentOut = momentum * Moment + ParamOut = Param - MomentOut $$ +if centered is true: + +mean_grad = decay * mean_square{t-1} + (1-decay) * gradient +mean_square = decay * mean_square{t-1} + (1-decay) * gradient ** 2 +mom = momentum * mom{t-1} + learning_rate * g_t / + sqrt(mean_square - mean_grad**2 + epsilon) +param -= mom + The original slides that proposed Rmsprop: Slide 29 of http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf) diff --git a/paddle/fluid/operators/rmsprop_op.h b/paddle/fluid/operators/rmsprop_op.h index 12836f43bde47ac87eb0af33dea501593b659a5d..25ed32c5ebb2ff5be962ac1e3e38c970623d705c 100644 --- a/paddle/fluid/operators/rmsprop_op.h +++ b/paddle/fluid/operators/rmsprop_op.h @@ -41,6 +41,7 @@ class RmspropOpKernel : public framework::OpKernel { float epsilon = ctx.Attr("epsilon"); float rho = ctx.Attr("decay"); float momentum = ctx.Attr("momentum"); + bool centered = ctx.Attr("centered"); auto p = EigenVector::Flatten(*ctx.Input("Param")); auto ms = EigenVector::Flatten(*ctx.Input("MeanSquare")); @@ -53,12 +54,24 @@ class RmspropOpKernel : public framework::OpKernel { auto ms_out = EigenVector::Flatten(*mean_square_out); auto& place = *ctx.template device_context().eigen_device(); - Eigen::DSizes grad_dsize(grad->numel()); + Eigen::DSizes grad_dsize(static_cast(grad->numel())); ms_out.device(place) = rho * ms + (1 - rho) * g * g; - mom_out.device(place) = - momentum * mom + - lr.broadcast(grad_dsize) * g / (ms_out + epsilon).sqrt(); + if (centered) { + auto mg = EigenVector::Flatten(*ctx.Input("MeanGrad")); + auto* mean_grad_out = ctx.Output("MeanGradOut"); + mean_grad_out->mutable_data(ctx.GetPlace()); + auto mg_out = EigenVector::Flatten(*mean_grad_out); + + mg_out.device(place) = rho * mg + (1 - rho) * g; + mom_out.device(place) = momentum * mom + + lr.broadcast(grad_dsize) * g / + (ms_out - mg_out.square() + epsilon).sqrt(); + } else { + mom_out.device(place) = + momentum * mom + + lr.broadcast(grad_dsize) * g / (ms_out + epsilon).sqrt(); + } p_out.device(place) = p - mom_out; } }; diff --git a/paddle/fluid/operators/squeeze_op.cc b/paddle/fluid/operators/squeeze_op.cc index 8a683116b8054de12fc4419b5aa5fbc019b675bb..e389c6a65e1e8220685294931c4d08e6fd928b7f 100644 --- a/paddle/fluid/operators/squeeze_op.cc +++ b/paddle/fluid/operators/squeeze_op.cc @@ -126,15 +126,15 @@ class SqueezeOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault({}); AddComment(R"DOC( Squeeze Operator. - - Remove single-dimensional entries from the shape of a tensor. - Takes a parameter axes with a list of axes to squeeze. - If axes is not provided, all the single dimensions will be removed from the shape. + + Remove single-dimensional entries from the shape of a tensor. + Takes a parameter axes with a list of axes to squeeze. + If axes is not provided, all the single dimensions will be removed from the shape. If an axis is selected with shape entry not equal to one, an error is raised. - + Examples: Case 1: - Given + Given X.shape = (1, 3, 1, 5) and axes = [0] @@ -144,7 +144,7 @@ class SqueezeOpMaker : public framework::OpProtoAndCheckerMaker { Case 2: Given X.shape = (1, 3, 1, 5) - and + and axes = [] we get: Out.shape = (3, 5) @@ -181,6 +181,113 @@ class SqueezeGradOp : public framework::OperatorBase { } }; +// FIXME(zcd): squeeze2 adds an intermediate output(XShape) based on squeeze, +// the XShape is used to carry the shape and lod of X which will be used in +// squeeze_grad, in this way, the framework can reuse the memory of X +// immediately the squeeze2_op is finished. +// Considering compatibility issues, we could not fix squeeze2_op +class Squeeze2OpMaker : public SqueezeOpMaker { + public: + void Make() override { + SqueezeOpMaker::Make(); + AddOutput("XShape", + "XShape is just used to store the shape and lod of X, which will " + "be used in SqueezeGradOp.") + .AsIntermediate(); + } +}; + +class Squeeze2OpInferShape : public SqueezeOpInferShape { + public: + void operator()(framework::InferShapeContext *ctx) const override { + SqueezeOpInferShape::operator()(ctx); + PADDLE_ENFORCE(ctx->HasOutput("XShape"), + "Output(XShape) of Squeeze operator should not be null."); + const auto &x_dims = ctx->GetInputDim("X"); + std::vector xshape_dims(x_dims.size() + 1); + xshape_dims[0] = 0; + for (int i = 0; i < x_dims.size(); ++i) { + xshape_dims[i + 1] = x_dims[i]; + } + ctx->SetOutputDim("XShape", framework::make_ddim(xshape_dims)); + ctx->ShareLoD("X", /*->*/ "XShape"); + } +}; + +class Squeeze2Op : public framework::OperatorBase { + public: + using OperatorBase::OperatorBase; + + private: + void RunImpl(const framework::Scope &scope, + const platform::Place &place) const override { + auto &axes = Attr>("axes"); + auto x_dims = scope.FindVar(Input("X"))->Get().dims(); + auto out_dims = Squeeze2OpInferShape::GetOutputShape(axes, x_dims); + + framework::AttributeMap attrs; + attrs["shape"] = framework::vectorize2int(out_dims); + // Invoke Reshape Op + auto reshape_op = framework::OpRegistry::CreateOp( + "reshape2", {{"X", {Input("X")}}, {"Shape", {}}}, + {{"Out", {Output("Out")}}, {"XShape", {Output("XShape")}}}, attrs); + reshape_op->Run(scope, place); + } +}; + +class Squeeze2GradOpMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); + grad_op->SetType("squeeze2_grad"); + grad_op->SetInput("XShape", Output("XShape")); + grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +class Squeeze2GradInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("XShape"), + "Input(XShape) shouldn't be null."); + PADDLE_ENFORCE(context->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) shouldn't be null."); + auto xshape_dims = context->GetInputDim("XShape"); + auto x_dims = framework::slice_ddim(xshape_dims, 1, xshape_dims.size()); + context->SetOutputDim(framework::GradVarName("X"), x_dims); + context->ShareLoD("XShape", framework::GradVarName("X")); + } +}; + +class Squeeze2GradOp : public framework::OperatorBase { + public: + using OperatorBase::OperatorBase; + + private: + void RunImpl(const framework::Scope &scope, + const platform::Place &place) const override { + auto dx_name = Output(framework::GradVarName("X")); + auto dout_name = Input(framework::GradVarName("Out")); + auto xshape_name = Input("XShape"); + auto xshape_dims = + scope.FindVar(xshape_name)->Get().dims(); + auto x_dims = framework::slice_ddim(xshape_dims, 1, xshape_dims.size()); + + framework::AttributeMap attrs; + attrs["shape"] = framework::vectorize2int(x_dims); + + auto reshape_op = framework::OpRegistry::CreateOp( + "reshape2", {{"X", {dout_name}}, {"Shape", {}}}, + {{"Out", {dx_name}}, {"XShape", {xshape_name}}}, attrs); + reshape_op->Run(scope, place); + } +}; + } // namespace operators } // namespace paddle @@ -192,3 +299,8 @@ REGISTER_OPERATOR(squeeze, ops::SqueezeOp, ops::SqueezeOpMaker, ops::SqueezeOpInferShape, paddle::framework::DefaultGradOpDescMaker); REGISTER_OPERATOR(squeeze_grad, ops::SqueezeGradOp, ops::SqueezeGradInferShape); + +REGISTER_OPERATOR(squeeze2, ops::Squeeze2Op, ops::Squeeze2OpMaker, + ops::Squeeze2OpInferShape, ops::Squeeze2GradOpMaker); +REGISTER_OPERATOR(squeeze2_grad, ops::Squeeze2GradOp, + ops::Squeeze2GradInferShape); diff --git a/paddle/fluid/operators/transpose_op.cc b/paddle/fluid/operators/transpose_op.cc index 60556a564c25c08612447ebd47a4b432b8a12d29..6a9fc6611a8f8eaa6749aefac0673ccabaebbcfe 100644 --- a/paddle/fluid/operators/transpose_op.cc +++ b/paddle/fluid/operators/transpose_op.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/transpose_op.h" +#include #include namespace paddle { @@ -24,7 +25,7 @@ class TransposeOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; - void InferShape(framework::InferShapeContext* ctx) const override { + void InferShape(framework::InferShapeContext *ctx) const override { PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should not be null"); auto x_dims = ctx->GetInputDim("X"); @@ -90,7 +91,7 @@ The behavior of this operator is similar to how `numpy.transpose` works. 2 &5 \end{pmatrix}$$ -- Given a input tensor with shape $(N, C, H, W)$ and the `axes` is +- Given a input tensor with shape $(N, C, H, W)$ and the `axes` is $[0, 2, 3, 1]$, then shape of the output tensor will be: $(N, H, W, C)$. )DOC"); @@ -101,7 +102,7 @@ class TransposeOpGrad : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; - void InferShape(framework::InferShapeContext* ctx) const override { + void InferShape(framework::InferShapeContext *ctx) const override { PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), "Input(Out@GRAD) should not be null"); @@ -113,6 +114,93 @@ class TransposeOpGrad : public framework::OperatorWithKernel { } }; +// FIXME(zcd): transpose2 adds an intermediate output(XShape) based on +// transpose, the XShape is used to carry the shape and lod of X which +// will be used in transpose_grad, in this way, the framework can reuse +// the memory of X immediately the transpose2_op is finished. +// Considering compatibility issues, we could not fix transpose2_op +class Transpose2Op : public TransposeOp { + public: + Transpose2Op(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : TransposeOp(type, inputs, outputs, attrs) {} + + void InferShape(framework::InferShapeContext *ctx) const override { + TransposeOp::InferShape(ctx); + PADDLE_ENFORCE(ctx->HasOutput("XShape"), + "Output(XShape) should not be null"); + const auto &in_dims = ctx->GetInputDim("X"); + std::vector x_shape_dim(in_dims.size() + 1); + x_shape_dim[0] = 0; + for (int i = 0; i < in_dims.size(); ++i) { + x_shape_dim[i + 1] = in_dims[i]; + } + ctx->SetOutputDim("XShape", framework::make_ddim(x_shape_dim)); + ctx->ShareLoD("X", /*->*/ "XShape"); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + ctx.device_context()); + } +}; + +class Transpose2OpMaker : public TransposeOpMaker { + public: + void Make() override { + TransposeOpMaker::Make(); + AddOutput("XShape", "(Tensor)The output tensor.").AsIntermediate(); + } +}; + +class Transpose2GradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); + grad_op->SetType("transpose2_grad"); + grad_op->SetInput("XShape", Output("XShape")); + grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +class Transpose2OpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("XShape"), "Input(XShape) should not be null"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + if (ctx->HasOutput(framework::GradVarName("X"))) { + auto xshape_dim = ctx->GetInputDim("XShape"); + auto x_shape_dim = + framework::slice_ddim(xshape_dim, 1, xshape_dim.size()); + ctx->SetOutputDim(framework::GradVarName("X"), x_shape_dim); + ctx->ShareLoD("XShape", framework::GradVarName("X")); + } + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType( + ctx.Input(framework::GradVarName("Out")) + ->type()), + ctx.device_context()); + } +}; + } // namespace operators } // namespace paddle @@ -120,8 +208,20 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(transpose, ops::TransposeOp, ops::TransposeOpMaker, paddle::framework::DefaultGradOpDescMaker); REGISTER_OPERATOR(transpose_grad, ops::TransposeOpGrad); + REGISTER_OP_CPU_KERNEL( transpose, ops::TransposeKernel); REGISTER_OP_CPU_KERNEL( transpose_grad, ops::TransposeGradKernel); + +REGISTER_OPERATOR(transpose2, ops::Transpose2Op, ops::Transpose2OpMaker, + ops::Transpose2GradMaker); +REGISTER_OPERATOR(transpose2_grad, ops::Transpose2OpGrad); + +REGISTER_OP_CPU_KERNEL( + transpose2, + ops::TransposeKernel); +REGISTER_OP_CPU_KERNEL( + transpose2_grad, + ops::TransposeGradKernel); diff --git a/paddle/fluid/operators/transpose_op.cu.cc b/paddle/fluid/operators/transpose_op.cu.cc index bcd1fb631394bc33b6fc162cfa7cbb20d55a654b..c1b5a8b31be243fab3af06a18c8e51986c953700 100644 --- a/paddle/fluid/operators/transpose_op.cu.cc +++ b/paddle/fluid/operators/transpose_op.cu.cc @@ -21,3 +21,10 @@ REGISTER_OP_CUDA_KERNEL( REGISTER_OP_CUDA_KERNEL( transpose_grad, ops::TransposeGradKernel); + +REGISTER_OP_CUDA_KERNEL( + transpose2, + ops::TransposeKernel); +REGISTER_OP_CUDA_KERNEL( + transpose2_grad, + ops::TransposeGradKernel); diff --git a/paddle/fluid/operators/unsqueeze_op.cc b/paddle/fluid/operators/unsqueeze_op.cc index 0fc8d54f6400c9dfb6af1e764ed44e95195bfe6e..405943add238ac2d245df11127bfadb4899e855f 100644 --- a/paddle/fluid/operators/unsqueeze_op.cc +++ b/paddle/fluid/operators/unsqueeze_op.cc @@ -127,13 +127,13 @@ class UnsqueezeOpMaker : public framework::OpProtoAndCheckerMaker { }); AddComment(R"DOC( Unsqueeze Operator. - - Insert single-dimensional entries to the shape of a tensor. - Takes one required argument axes, a list of dimensions that will be inserted. - Dimension indices in axes are as seen in the output tensor. - For example: - Given a tensor such that tensor with shape [3, 4, 5], + Insert single-dimensional entries to the shape of a tensor. + Takes one required argument axes, a list of dimensions that will be inserted. + Dimension indices in axes are as seen in the output tensor. + + For example: + Given a tensor such that tensor with shape [3, 4, 5], then Unsqueeze(tensor, axes=[0, 4]) has shape [1, 3, 4, 5, 1] )DOC"); } @@ -168,6 +168,112 @@ class UnsqueezeGradOp : public framework::OperatorBase { } }; +// FIXME(zcd): unsqueeze2 adds an intermediate output(XShape) based on +// unsqueeze, the XShape is used to carry the shape and lod of X which +// will be used in unsqueeze_grad, in this way, the framework can reuse +// the memory of X immediately the unsqueeze2_op is finished. +// Considering compatibility issues, we could not fix unsqueeze2_op +class Unsqueeze2OpInferShape : public UnsqueezeOpInferShape { + public: + void operator()(framework::InferShapeContext *ctx) const override { + UnsqueezeOpInferShape::operator()(ctx); + PADDLE_ENFORCE(ctx->HasOutput("XShape"), + "Output(XShape) of Unsqueeze operator should not be null."); + const auto &x_dims = ctx->GetInputDim("X"); + std::vector xshape_dims(x_dims.size() + 1); + xshape_dims[0] = 0; + for (int i = 0; i < x_dims.size(); ++i) { + xshape_dims[i + 1] = x_dims[i]; + } + ctx->SetOutputDim("XShape", framework::make_ddim(xshape_dims)); + ctx->ShareLoD("X", /*->*/ "XShape"); + } +}; + +class Unsqueeze2OpMaker : public UnsqueezeOpMaker { + public: + void Make() override { + UnsqueezeOpMaker::Make(); + AddOutput("XShape", + "XShape is just used to store the shape and lod of X, which will " + "be used in UnsqueezeGradOp.") + .AsIntermediate(); + } +}; + +class Unsqueeze2Op : public framework::OperatorBase { + public: + using OperatorBase::OperatorBase; + + private: + void RunImpl(const framework::Scope &scope, + const platform::Place &place) const override { + auto &axes = Attr>("axes"); + auto x_dims = scope.FindVar(Input("X"))->Get().dims(); + auto out_dims = Unsqueeze2OpInferShape::GetOutputShape(axes, x_dims); + + framework::AttributeMap attrs; + attrs["shape"] = framework::vectorize2int(out_dims); + // Invoke Reshape op. + auto reshape_op = framework::OpRegistry::CreateOp( + "reshape2", {{"X", {Input("X")}}, {"Shape", {}}}, + {{"Out", {Output("Out")}}, {"XShape", {Output("XShape")}}}, attrs); + reshape_op->Run(scope, place); + } +}; + +class Unsqueeze2GradOpMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); + grad_op->SetType("unsqueeze2_grad"); + grad_op->SetInput("XShape", Output("XShape")); + grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +class Unsqueeze2GradInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("XShape"), + "Input(XShape) shouldn't be null."); + PADDLE_ENFORCE(context->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) shouldn't be null."); + auto xshape_dims = context->GetInputDim("XShape"); + auto x_dims = framework::slice_ddim(xshape_dims, 1, xshape_dims.size()); + context->SetOutputDim(framework::GradVarName("X"), x_dims); + context->ShareLoD("XShape", framework::GradVarName("X")); + } +}; + +class Unsqueeze2GradOp : public framework::OperatorBase { + public: + using OperatorBase::OperatorBase; + + private: + void RunImpl(const framework::Scope &scope, + const platform::Place &place) const override { + auto dx_name = Output(framework::GradVarName("X")); + auto dout_name = Input(framework::GradVarName("Out")); + auto xshape_name = Input("XShape"); + auto xshape_dims = + scope.FindVar(xshape_name)->Get().dims(); + auto x_dims = framework::slice_ddim(xshape_dims, 1, xshape_dims.size()); + + framework::AttributeMap attrs; + attrs["shape"] = framework::vectorize2int(x_dims); + + auto reshape_op = framework::OpRegistry::CreateOp( + "reshape2", {{"X", {dout_name}}, {"Shape", {}}}, + {{"Out", {dx_name}}, {"XShape", {xshape_name}}}, attrs); + reshape_op->Run(scope, place); + } +}; } // namespace operators } // namespace paddle @@ -180,3 +286,8 @@ REGISTER_OPERATOR(unsqueeze, ops::UnsqueezeOp, ops::UnsqueezeOpMaker, paddle::framework::DefaultGradOpDescMaker); REGISTER_OPERATOR(unsqueeze_grad, ops::UnsqueezeGradOp, ops::UnsqueezeGradInferShape); + +REGISTER_OPERATOR(unsqueeze2, ops::Unsqueeze2Op, ops::Unsqueeze2OpMaker, + ops::Unsqueeze2OpInferShape, ops::Unsqueeze2GradOpMaker); +REGISTER_OPERATOR(unsqueeze2_grad, ops::Unsqueeze2GradOp, + ops::Unsqueeze2GradInferShape); diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc index 4fbfa6354ab45fed4839227a2a4be8fe147e5fd9..6a3ad2151081504fda2a3818c5f99ad47039d91d 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.cc +++ b/paddle/fluid/platform/dynload/dynamic_loader.cc @@ -121,6 +121,12 @@ static inline void* GetDsoHandleFromSearchPath(const std::string& search_root, if (nullptr == dso_handle) { LOG(WARNING) << "Failed to find dynamic library: " << dlPath << " (" << dlerror() << ")"; + if (dlPath.find("nccl") != std::string::npos) { + std::cout + << "You may need to install 'nccl2' from NVIDIA official website: " + << "https://developer.nvidia.com/nccl/nccl-download" + << "before install PaddlePaddle" << std::endl; + } dlPath = dso_name; dso_handle = GetDsoHandleFromDefaultPath(dlPath, dynload_flags); } diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 7199424b4709fbe9fc962cf98aea6223b9f3e51d..9ffde5df9673f192b8970ea832fd0328950969b2 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -115,6 +115,7 @@ function cmake_gen() { -DWITH_FLUID_ONLY=${WITH_FLUID_ONLY:-OFF} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DWITH_CONTRIB=${WITH_CONTRIB:-ON} + -DWITH_INFERENCE=${WITH_INFERENCE:-ON} -DWITH_ANAKIN=${WITH_ANAKIN:-OFF} -DPY_VERSION=${PY_VERSION:-2.7} ======================================== @@ -144,6 +145,7 @@ EOF -DWITH_FLUID_ONLY=${WITH_FLUID_ONLY:-OFF} \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DWITH_CONTRIB=${WITH_CONTRIB:-ON} \ + -DWITH_INFERENCE=${WITH_INFERENCE:-ON} \ -DWITH_ANAKIN=${WITH_ANAKIN:-OFF} \ -DPY_VERSION=${PY_VERSION:-2.7} } @@ -498,7 +500,7 @@ EOF EOF if [[ ${WITH_GPU} == "ON" ]]; then - NCCL_DEPS="apt-get install -y --allow-downgrades libnccl2=2.1.2-1+cuda${CUDA_MAJOR} libnccl-dev=2.1.2-1+cuda${CUDA_MAJOR} &&" + NCCL_DEPS="apt-get install -y --allow-downgrades libnccl2=2.2.13-1+cuda${CUDA_MAJOR} libnccl-dev=2.2.13-1+cuda${CUDA_MAJOR} &&" else NCCL_DEPS="" fi diff --git a/python/paddle/dataset/image.py b/python/paddle/dataset/image.py index 920dbf3b4ebb0bc3d98c9ea986d7d039deed4a4c..19fc229e6fa84792f58aeeb00be09eb2401b19c7 100644 --- a/python/paddle/dataset/image.py +++ b/python/paddle/dataset/image.py @@ -104,7 +104,7 @@ def batch_images_from_tar(data_file, pickle.dump( output, open('%s/batch_%d' % (out_path, file_id), 'wb'), - protocol=pickle.HIGHEST_PROTOCOL) + protocol=2) file_id += 1 data = [] labels = [] @@ -113,9 +113,7 @@ def batch_images_from_tar(data_file, output['label'] = labels output['data'] = data pickle.dump( - output, - open('%s/batch_%d' % (out_path, file_id), 'wb'), - protocol=pickle.HIGHEST_PROTOCOL) + output, open('%s/batch_%d' % (out_path, file_id), 'wb'), protocol=2) with open(meta_file, 'a') as meta: for file in os.listdir(out_path): diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index d8c7cc08b652f91456f557b0296e85b9aebc9dd0..5f49d5bbff53096ece140a185f73722870924677 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -4025,10 +4025,12 @@ def transpose(x, perm, name=None): helper = LayerHelper('transpose', **locals()) out = helper.create_tmp_variable(x.dtype) + x_shape = helper.create_tmp_variable(x.dtype) helper.append_op( - type='transpose', + type='transpose2', inputs={'X': [x]}, - outputs={'Out': [out]}, + outputs={'Out': [out], + 'XShape': [x_shape]}, attrs={'axis': perm}) return out @@ -4520,13 +4522,15 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=True, name=None): "Each dimension size given in shape must not be negtive " "except one unknown dimension.") - helper = LayerHelper("reshape", **locals()) + helper = LayerHelper("reshape2", **locals()) out = helper.create_tmp_variable(dtype=x.dtype) + x_shape = helper.create_tmp_variable(dtype=x.dtype) helper.append_op( - type="reshape", + type="reshape2", inputs=inputs, attrs={"shape": shape}, - outputs={"Out": out}) + outputs={"Out": out, + "XShape": x_shape}) return helper.append_activation(out) @@ -4570,11 +4574,13 @@ def squeeze(input, axes, name=None): """ helper = LayerHelper("squeeze", **locals()) out = helper.create_tmp_variable(dtype=input.dtype) + x_shape = helper.create_tmp_variable(dtype=input.dtype) helper.append_op( - type="squeeze", + type="squeeze2", inputs={"X": input}, attrs={"axes": axes}, - outputs={"Out": out}) + outputs={"Out": out, + "XShape": x_shape}) return out @@ -4605,11 +4611,13 @@ def unsqueeze(input, axes, name=None): """ helper = LayerHelper("unsqueeze", **locals()) out = helper.create_tmp_variable(dtype=input.dtype) + x_shape = helper.create_tmp_variable(dtype=input.dtype) helper.append_op( - type="unsqueeze", + type="unsqueeze2", inputs={"X": input}, attrs={"axes": axes}, - outputs={"Out": out}) + outputs={"Out": out, + "XShape": x_shape}) return out @@ -5811,10 +5819,12 @@ def flatten(x, axis=1, name=None): raise ValueError("The axis should be a int, and in range [0, rank(x)]") out = helper.create_tmp_variable(x.dtype) + x_shape = helper.create_tmp_variable(x.dtype) helper.append_op( - type='flatten', + type='flatten2', inputs={"X": x}, - outputs={'Out': out}, + outputs={'Out': out, + 'XShape': x_shape}, attrs={"axis": axis}) return out diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 33d6311b9717c66f0d6782eb6b3e348cd4c02a69..215f0cf2fc5ab4fbd06719ac4790a01dd00080eb 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -897,7 +897,20 @@ class RMSPropOptimizer(Optimizer): r(w, t) & = \\rho r(w, t-1) + (1 - \\rho)(\\nabla Q_{i}(w))^2 - v(w, t) & = \\beta v(w, t-1) + \\frac{\\eta} {\\sqrt{v(w,t) + + v(w, t) & = \\beta v(w, t-1) + \\frac{\\eta} {\\sqrt{r(w,t) + + \\epsilon}} \\nabla Q_{i}(w) + + w & = w - v(w, t) + + if centered is True: + + .. math:: + + r(w, t) & = \\rho r(w, t-1) + (1 - \\rho)(\\nabla Q_{i}(w))^2 + + g(w, t) & = \\rho g(w, t-1) + (1 - \\rho)\\nabla Q_{i}(w) + + v(w, t) & = \\beta v(w, t-1) + \\frac{\\eta} {\\sqrt{r(w,t) - (g(w, t))^2 + \\epsilon}} \\nabla Q_{i}(w) w & = w - v(w, t) @@ -915,6 +928,10 @@ class RMSPropOptimizer(Optimizer): avoid division by zero, set 1e-6 by default. momentum(float): :math:`\\beta` in equation is the momentum term, set 0.0 by default. + centered(bool): If True, gradients are normalized by the estimated variance of + the gradient; if False, by the uncentered second moment. Setting this to + True may help with training, but is slightly more expensive in terms of + computation and memory. Defaults to False. Raises: ValueError: If learning_rate, rho, epsilon, momentum are None. @@ -928,12 +945,14 @@ class RMSPropOptimizer(Optimizer): _momentum_acc_str = "momentum" _mean_square_acc_str = "mean_square" + _mean_grad_acc_str = "mean_grad" def __init__(self, learning_rate, rho=0.95, epsilon=1.0e-6, momentum=0.0, + centered=False, **kwargs): super(RMSPropOptimizer, self).__init__( learning_rate=learning_rate, **kwargs) @@ -950,6 +969,7 @@ class RMSPropOptimizer(Optimizer): self._rho = rho self._epsilon = epsilon self._momentum = momentum + self._centered = centered def _create_accumulators(self, block, parameters): if not isinstance(block, framework.Block): @@ -958,6 +978,7 @@ class RMSPropOptimizer(Optimizer): for p in parameters: self._add_accumulator(self._momentum_acc_str, p) self._add_accumulator(self._mean_square_acc_str, p) + self._add_accumulator(self._mean_grad_acc_str, p) def _append_optimize_op(self, block, param_and_grad): if not isinstance(block, framework.Block): @@ -967,6 +988,8 @@ class RMSPropOptimizer(Optimizer): param_and_grad[0]) mean_square_acc = self._get_accumulator(self._mean_square_acc_str, param_and_grad[0]) + mean_grad_acc = self._get_accumulator(self._mean_grad_acc_str, + param_and_grad[0]) rmsprop_op = block.append_op( type=self.type, inputs={ @@ -974,17 +997,20 @@ class RMSPropOptimizer(Optimizer): "Grad": param_and_grad[1], "Moment": momentum_acc, "MeanSquare": mean_square_acc, + "MeanGrad": mean_grad_acc, "LearningRate": self._create_param_lr(param_and_grad), }, outputs={ "ParamOut": param_and_grad[0], "MomentOut": momentum_acc, - "MeanSquareOut": mean_square_acc + "MeanSquareOut": mean_square_acc, + "MeanGradOut": mean_grad_acc }, attrs={ "epsilon": self._epsilon, "decay": self._rho, - "momentum": self._momentum + "momentum": self._momentum, + "centered": self._centered }) return rmsprop_op diff --git a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py b/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py index f6017a455df7e8bd197ef2563a759f843b5e7c73..e1368a3392a9cab3e82eff0a73eb225a52aa03bf 100644 --- a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py @@ -47,14 +47,14 @@ def train_program(): loss = fluid.layers.square_error_cost(input=y_predict, label=y) avg_loss = fluid.layers.mean(loss) - return avg_loss + return [avg_loss, y_predict] def optimizer_func(): return fluid.optimizer.SGD(learning_rate=0.001) -def train(use_cuda, train_program, params_dirname): +def train(use_cuda, train_program, params_dirname, inference_model_dirname): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() trainer = fluid.Trainer( @@ -74,6 +74,8 @@ def train(use_cuda, train_program, params_dirname): ''' if params_dirname is not None: trainer.save_params(params_dirname) + trainer.save_inference_model(inference_model_dirname, + ['x'], [1]) trainer.stop() trainer.train( @@ -99,15 +101,55 @@ def infer(use_cuda, inference_program, params_dirname=None): print("infer results: ", results[0]) +def infer_by_saved_model(use_cuda, save_dirname=None): + if save_dirname is None: + return + + place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() + exe = fluid.Executor(place) + + inference_scope = fluid.core.Scope() + with fluid.scope_guard(inference_scope): + # Use fluid.io.load_inference_model to obtain the inference program desc, + # the feed_target_names (the names of variables that will be feeded + # data using feed operators), and the fetch_targets (variables that + # we want to obtain data from using fetch operators). + [inference_program, feed_target_names, + fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + + # The input's dimension should be 2-D and the second dim is 13 + # The input data should be >= 0 + batch_size = 10 + + test_reader = paddle.batch( + paddle.dataset.uci_housing.test(), batch_size=batch_size) + + test_data = next(test_reader()) + test_feat = numpy.array( + [data[0] for data in test_data]).astype("float32") + test_label = numpy.array( + [data[1] for data in test_data]).astype("float32") + + assert feed_target_names[0] == 'x' + results = exe.run(inference_program, + feed={feed_target_names[0]: numpy.array(test_feat)}, + fetch_list=fetch_targets) + print("infer shape: ", results[0].shape) + print("infer results: ", results[0]) + print("ground truth: ", test_label) + + def main(use_cuda): if use_cuda and not fluid.core.is_compiled_with_cuda(): return # Directory for saving the trained model - params_dirname = "fit_a_line.inference.model" + params_dirname = "fit_a_line.model" + inference_model_dirname = "fit_a_line.inference_model" - train(use_cuda, train_program, params_dirname) + train(use_cuda, train_program, params_dirname, inference_model_dirname) infer(use_cuda, inference_program, params_dirname) + infer_by_saved_model(use_cuda, inference_model_dirname) class TestFitALine(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/dist_transformer.py b/python/paddle/fluid/tests/unittests/dist_transformer.py index 7abfa0a4be0dec9fe251704e22dfef1f932e7c5b..e3db316698398ff693157d583ad1410d10dcf81d 100644 --- a/python/paddle/fluid/tests/unittests/dist_transformer.py +++ b/python/paddle/fluid/tests/unittests/dist_transformer.py @@ -36,6 +36,7 @@ import paddle.fluid as fluid import paddle.fluid.layers as layers from paddle.fluid import core from test_dist_base import TestDistRunnerBase, runtime_main +import paddle.compat as cpt from paddle.compat import long_type import hashlib @@ -315,8 +316,9 @@ def pad_batch_data(insts, """ return_list = [] max_len = max(len(inst) for inst in insts) - num_token = reduce(lambda x, y: x + y, - [len(inst) for inst in insts]) if return_num_token else 0 + num_token = six.moves.reduce( + lambda x, y: x + y, + [len(inst) for inst in insts]) if return_num_token else 0 # Any token included in dict can be used to pad, since the paddings' loss # will be masked out by weights and make no effect on parameter gradients. inst_data = np.array( @@ -328,7 +330,7 @@ def pad_batch_data(insts, return_list += [inst_weight.astype("float32").reshape([-1, 1])] else: # position data inst_pos = np.array([ - range(1, len(inst) + 1) + [0] * (max_len - len(inst)) + list(range(1, len(inst) + 1)) + [0] * (max_len - len(inst)) for inst in insts ]) return_list += [inst_pos.astype("int64").reshape([-1, 1])] @@ -385,10 +387,11 @@ def prepare_batch_input(insts, data_input_names, src_pad_idx, trg_pad_idx, return_num_token=True) data_input_dict = dict( - zip(data_input_names, [ - src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, - trg_slf_attn_bias, trg_src_attn_bias, lbl_word, lbl_weight - ])) + list( + zip(data_input_names, [ + src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, + trg_slf_attn_bias, trg_src_attn_bias, lbl_word, lbl_weight + ]))) return data_input_dict, np.asarray([num_token], dtype="float32") @@ -561,7 +564,7 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, np.log(TrainTaskConfig.label_smooth_eps / ( ModelHyperParams.trg_vocab_size - 1) + 1e-20)) init = False - for pass_id in xrange(TrainTaskConfig.pass_num): + for pass_id in six.moves.xrange(TrainTaskConfig.pass_num): pass_start_time = time.time() for batch_id, data in enumerate(train_data()): if batch_id >= 5: @@ -587,11 +590,11 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, ModelHyperParams.eos_idx, ModelHyperParams.n_head, ModelHyperParams.d_model) total_num_token += num_token - feed_kv_pairs = data_input_dict.items() + feed_kv_pairs = list(data_input_dict.items()) if TrainTaskConfig.local: - feed_kv_pairs += { + feed_kv_pairs += list({ lr_scheduler.learning_rate.name: lr_rate - }.items() + }.items()) feed_list.append(dict(feed_kv_pairs)) if not init: @@ -873,6 +876,7 @@ class DataReader(object): f = tarfile.open(fpaths[0], "r") for line in f.extractfile(tar_fname): + line = cpt.to_text(line) fields = line.strip("\n").split(self._field_delimiter) if (not self._only_src and len(fields) == 2) or ( self._only_src and len(fields) == 1): @@ -882,8 +886,9 @@ class DataReader(object): if not os.path.isfile(fpath): raise IOError("Invalid file: %s" % fpath) - with open(fpath, "r") as f: + with open(fpath, "rb") as f: for line in f: + line = cpt.to_text(line) fields = line.strip("\n").split(self._field_delimiter) if (not self._only_src and len(fields) == 2) or ( self._only_src and len(fields) == 1): @@ -892,8 +897,9 @@ class DataReader(object): @staticmethod def load_dict(dict_path, reverse=False): word_dict = {} - with open(dict_path, "r") as fdict: + with open(dict_path, "rb") as fdict: for idx, line in enumerate(fdict): + line = cpt.to_text(line) if reverse: word_dict[idx] = line.strip("\n") else: @@ -1034,7 +1040,7 @@ def multi_head_attention(queries, # size of the input as the output dimension size. return layers.reshape( x=trans_x, - shape=map(int, [0, 0, trans_x.shape[2] * trans_x.shape[3]])) + shape=list(map(int, [0, 0, trans_x.shape[2] * trans_x.shape[3]]))) def scaled_dot_product_attention(q, k, v, attn_bias, d_model, dropout_rate): """ diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index 20f1a37a426e9697048d636bf738c9056213e5f6..56a242b996f67aa4b9c858ab8aaeb1c1cd3bcf60 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -249,7 +249,7 @@ class OpTest(unittest.TestCase): outs, _ = self._calc_output(place) return outs - def _calc_output(self, place, parallel=False): + def _calc_output(self, place, parallel=False, no_check_set=None): program = Program() block = program.global_block() @@ -273,6 +273,8 @@ class OpTest(unittest.TestCase): # if not, fill the fetch_list by the user configured outputs in test. if len(fetch_list) == 0: for var_name, var in six.iteritems(outputs): + if no_check_set is not None and var_name in no_check_set: + continue if isinstance(var, list): for v in var: fetch_list.append(v) @@ -291,11 +293,17 @@ class OpTest(unittest.TestCase): return_numpy=False) return outs, fetch_list - def check_output_with_place(self, place, atol): - outs, fetch_list = self._calc_output(place) + def check_output_with_place(self, + place, + atol, + no_check_set=None, + equal_nan=False): + outs, fetch_list = self._calc_output(place, no_check_set=no_check_set) for out_name, out_dup in Operator.get_op_outputs(self.op_type): if out_name not in self.outputs: continue + if no_check_set is not None and out_name in no_check_set: + continue def find_actual(target_name, fetch_list): found = [ @@ -321,7 +329,7 @@ class OpTest(unittest.TestCase): if isinstance(expect, tuple) else expect self.assertTrue( np.allclose( - actual_t, expect_t, atol=atol), + actual_t, expect_t, atol=atol, equal_nan=equal_nan), "Output (" + sub_out_name + ") has diff at " + str(place)) if isinstance(expect, tuple): @@ -337,7 +345,7 @@ class OpTest(unittest.TestCase): expect_t = expect[0] if isinstance(expect, tuple) else expect self.assertTrue( np.allclose( - actual_t, expect_t, atol=atol), + actual_t, expect_t, atol=atol, equal_nan=equal_nan), "Output (" + out_name + ") has diff at " + str(place) + "\nExpect " + str(expect_t) + "\n" + "But Got" + str(actual_t)) @@ -360,10 +368,10 @@ class OpTest(unittest.TestCase): places.append(core.CUDAPlace(0)) return places - def check_output(self, atol=1e-5): + def check_output(self, atol=1e-5, no_check_set=None, equal_nan=False): places = self._get_places() for place in places: - self.check_output_with_place(place, atol) + self.check_output_with_place(place, atol, no_check_set, equal_nan) def check_output_customized(self, checker): places = self._get_places() diff --git a/python/paddle/fluid/tests/unittests/test_dist_base.py b/python/paddle/fluid/tests/unittests/test_dist_base.py index 58875a1dd19fd91f6f2bed928397ee7f73302dff..c0f5da5a1ae43847dff6348ea5f3e3bfd5e89ab9 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_base.py @@ -55,6 +55,7 @@ class TestDistRunnerBase(object): pserver_prog = t.get_pserver_program(args.current_endpoint) startup_prog = t.get_startup_program(args.current_endpoint, pserver_prog) + place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_prog) @@ -147,6 +148,8 @@ def runtime_main(test_class): import paddle.compat as cpt +import socket +from contextlib import closing class TestDistBase(unittest.TestCase): @@ -156,13 +159,19 @@ class TestDistBase(unittest.TestCase): def setUp(self): self._trainers = 2 self._pservers = 2 - self._ps_endpoints = "127.0.0.1:9123,127.0.0.1:9124" + self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( + self._find_free_port(), self._find_free_port()) self._python_interp = "python" self._sync_mode = True self._mem_opt = False self._use_reduce = False self._setup_config() + def _find_free_port(self): + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(('', 0)) + return s.getsockname()[1] + def start_pserver(self, model_file, check_error_log): ps0_ep, ps1_ep = self._ps_endpoints.split(",") ps_cmd = "%s %s --role pserver --endpoints %s --trainer_id 0 --current_endpoint %s --trainers %d --is_dist" diff --git a/python/paddle/fluid/tests/unittests/test_flatten_op.py b/python/paddle/fluid/tests/unittests/test_flatten_op.py index 17b01e03124e8007c51107b414c628d4bfc49c79..effa2a148eef8b0047b12c676803abb2871e8118 100644 --- a/python/paddle/fluid/tests/unittests/test_flatten_op.py +++ b/python/paddle/fluid/tests/unittests/test_flatten_op.py @@ -22,14 +22,17 @@ from op_test import OpTest class TestFlattenOp(OpTest): def setUp(self): - self.op_type = "flatten" + self.op_type = "flatten2" self.init_test_case() self.inputs = {"X": np.random.random(self.in_shape).astype("float32")} self.init_attrs() - self.outputs = {"Out": self.inputs["X"].reshape(self.new_shape)} + self.outputs = { + "Out": self.inputs["X"].reshape(self.new_shape), + "XShape": np.random.random(self.in_shape).astype("float32") + } def test_check_output(self): - self.check_output() + self.check_output(no_check_set=["XShape"]) def test_check_grad(self): self.check_grad(["X"], "Out") diff --git a/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py b/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py index 4767e9433ea74d5da83867d646f2a63c9a092668..de0c86f96db958eebd7e74346bec244f0c804ed9 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py @@ -53,12 +53,11 @@ class TestFusionLSTMOp(OpTest): self.M = 8 self.D = 16 self.has_initial_state = False + self.use_peepholes = False self.is_reverse = False self.act_gate = 'sigmoid' self.act_cell = 'tanh' self.act_cand = 'tanh' - self.use_peepholes = False - self.use_seq = False self.set_conf() T = sum(self.lod[0]) @@ -108,7 +107,6 @@ class TestFusionLSTMOp(OpTest): } self.attrs = { 'use_peepholes': self.use_peepholes, - 'use_seq': self.use_seq, 'is_reverse': self.is_reverse, 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, @@ -178,50 +176,18 @@ class TestFusionLSTMOpPeepholesReverse(TestFusionLSTMOp): self.is_reverse = True -class TestFusionLSTMOpPoopholesBS1(TestFusionLSTMOp): +class TestFusionLSTMOpPeepholesInitReverse(TestFusionLSTMOp): def set_conf(self): self.use_peepholes = True - self.lod = [[3]] - self.D = 16 - - -class TestFusionLSTMOpSeqInit(TestFusionLSTMOp): - def set_conf(self): - self.use_seq = True - self.has_initial_state = True - - -class TestFusionLSTMOpSeqReverse(TestFusionLSTMOp): - def set_conf(self): - self.use_seq = True - self.is_reverse = True - - -class TestFusionLSTMOpSeqInitReverse(TestFusionLSTMOp): - def set_conf(self): - self.use_seq = True self.has_initial_state = True self.is_reverse = True -class TestFusionLSTMOpSeqPeepholes(TestFusionLSTMOp): +class TestFusionLSTMOpPeepholesBS1(TestFusionLSTMOp): def set_conf(self): - self.use_seq = True self.use_peepholes = True - - -class TestFusionLSTMOpSeqPeepholesInit(TestFusionLSTMOp): - def set_conf(self): - self.use_seq = True - self.use_peepholes = True - self.has_initial_state = True - - -class TestFusionLSTMOpSeqPeepholesReverse(TestFusionLSTMOp): - def set_conf(self): - self.use_seq = True - self.use_peepholes = True - self.is_reverse = True + self.lod = [[2]] + self.D = 8 if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py index 372ef748b2e704fd3858c382e048e51448ed3bd5..a49c5d9b43ae1bffa7cb57764db497f68030b151 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py @@ -85,6 +85,7 @@ class TestFetchOp(unittest.TestCase): assert not math.isnan(np.sum(ret[i])) and \ not math.isinf(np.sum(ret[i])) + @unittest.skip(reason="CI timeout") def test_fetch_op(self): tst_reader = paddle.batch(flowers.test(use_xmap=False), batch_size=16) tst_reader_iter = tst_reader() @@ -139,6 +140,7 @@ class TestFeedParallel(unittest.TestCase): if batch_id == 2: break + @unittest.skip(reason="CI timeout") def test_feed_op(self): os.environ['CPU_NUM'] = str(4) if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_prelu_op.py b/python/paddle/fluid/tests/unittests/test_prelu_op.py index 1e3e40d54a78045c8d8fdd9a3a3715107d1e7a80..48a6b0577b6787d2e1231fdcbe6d2c1bb46414ed 100644 --- a/python/paddle/fluid/tests/unittests/test_prelu_op.py +++ b/python/paddle/fluid/tests/unittests/test_prelu_op.py @@ -16,6 +16,7 @@ from __future__ import print_function import unittest import numpy as np +import six from op_test import OpTest @@ -62,17 +63,20 @@ class PReluTest(OpTest): # TODO(minqiyang): Resume these test cases after fixing Python3 CI job issues -# class TestCase1(PReluTest): -# def initTestCase(self): -# self.attrs = {'mode': "all"} +if six.PY2: -# class TestCase2(PReluTest): -# def initTestCase(self): -# self.attrs = {'mode': "channel"} + class TestCase1(PReluTest): + def initTestCase(self): + self.attrs = {'mode': "all"} + + class TestCase2(PReluTest): + def initTestCase(self): + self.attrs = {'mode': "channel"} + + class TestCase3(PReluTest): + def initTestCase(self): + self.attrs = {'mode': "element"} -# class TestCase3(PReluTest): -# def initTestCase(self): -# self.attrs = {'mode': "element"} if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_reshape_op.py b/python/paddle/fluid/tests/unittests/test_reshape_op.py index 1de35dc35b0176b77eb2d9b25cd6ee4e645e56c3..0557593657e2e480a509902a07f25723b2c710b0 100644 --- a/python/paddle/fluid/tests/unittests/test_reshape_op.py +++ b/python/paddle/fluid/tests/unittests/test_reshape_op.py @@ -22,106 +22,39 @@ from op_test import OpTest class TestReshapeOp(OpTest): def setUp(self): - ori_shape = (2, 25) - new_shape = (5, 10) - - self.op_type = "reshape" - self.inputs = {"X": np.random.random(ori_shape).astype("float32")} - self.attrs = {"shape": new_shape} - self.outputs = {"Out": self.inputs["X"].reshape(new_shape)} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(["X"], "Out") - - -class TestReshapeOpDimInfer1(OpTest): - def setUp(self): - ori_shape = (5, 10) - new_shape = (5, -1, 5) - - self.op_type = "reshape" - self.inputs = {"X": np.random.random(ori_shape).astype("float32")} - self.attrs = {"shape": new_shape} - self.outputs = {"Out": self.inputs["X"].reshape(self.attrs["shape"])} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(["X"], "Out") - - -class TestReshapeOpDimInfer2(OpTest): - def setUp(self): - ori_shape = (2, 2, 6) - new_shape = (2, 0, 3, -1) - infered_shape = (2, 2, 3, -1) - - self.op_type = "reshape" - self.inputs = {"X": np.random.random(ori_shape).astype("float32")} - self.attrs = {"shape": new_shape} - self.outputs = {"Out": self.inputs["X"].reshape(infered_shape)} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(["X"], "Out") - - -class TestReshapeOpInplace(OpTest): - def setUp(self): - ori_shape = (2, 25) - new_shape = (5, 10) - - self.op_type = "reshape" - self.inputs = {"X": np.random.random(ori_shape).astype("float32")} - self.attrs = {"shape": new_shape} - self.outputs = {"Out": self.inputs["X"].reshape(new_shape)} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(["X"], "Out") - - -class TestReshapeOpDimInferInplace1(OpTest): - def setUp(self): - ori_shape = (5, 10) - new_shape = (5, -1, 5) + self.init_data() + self.op_type = "reshape2" + self.inputs = {"X": np.random.random(self.ori_shape).astype("float32")} + self.attrs = {"shape": self.new_shape} + self.outputs = { + "Out": self.inputs["X"].reshape(self.infered_shape), + 'XShape': np.random.random(self.ori_shape).astype("float32") + } - self.op_type = "reshape" - self.inputs = {"X": np.random.random(ori_shape).astype("float32")} - self.attrs = {"shape": new_shape} - self.outputs = {"Out": self.inputs["X"].reshape(new_shape)} + def init_data(self): + self.ori_shape = (2, 25) + self.new_shape = (5, 10) + self.infered_shape = (5, 10) def test_check_output(self): - self.check_output() + self.check_output(no_check_set=['XShape']) def test_check_grad(self): self.check_grad(["X"], "Out") -class TestReshapeOpDimInferInplace2(OpTest): - def setUp(self): - ori_shape = (2, 2, 6) - new_shape = (2, 0, 3, -1) - infered_shape = (2, 2, 3, -1) - - self.op_type = "reshape" - self.inputs = {"X": np.random.random(ori_shape).astype("float32")} - self.attrs = {"shape": new_shape} - self.outputs = {"Out": self.inputs["X"].reshape(infered_shape)} +class TestReshapeOpDimInfer1(TestReshapeOp): + def init_data(self): + self.ori_shape = (5, 10) + self.new_shape = (5, -1, 5) + self.infered_shape = (5, -1, 5) - def test_check_output(self): - self.check_output() - def test_check_grad(self): - self.check_grad(["X"], "Out") +class TestReshapeOpDimInfer2(TestReshapeOp): + def init_data(self): + self.ori_shape = (2, 2, 6) + self.new_shape = (2, 0, 3, -1) + self.infered_shape = (2, 2, 3, -1) class TestReshapeOpWithInputShape(OpTest): @@ -130,20 +63,23 @@ class TestReshapeOpWithInputShape(OpTest): new_shape = (0, -1, 5) actual_shape = (2, 3, 5) - self.op_type = "reshape" + self.op_type = "reshape2" self.inputs = { "X": np.random.random(ori_shape).astype("float32"), "Shape": np.array( actual_shape, dtype="int32") } self.attrs = {"shape": new_shape} - self.outputs = {"Out": self.inputs["X"].reshape(actual_shape)} + self.outputs = { + "Out": self.inputs["X"].reshape(actual_shape), + 'XShape': np.random.random(ori_shape).astype("float32") + } def test_check_output(self): - self.check_output() + self.check_output(no_check_set=['XShape']) def test_check_grad(self): - self.check_grad(["X"], "Out") + self.check_grad(["X"], "Out", sum_outputs=["Out"]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_rmsprop_op.py b/python/paddle/fluid/tests/unittests/test_rmsprop_op.py index 3d4623c74d9a307b12ab6d72ad0b4d2dae938720..70848e4e2239e2be160bb0c1a28a5aecd01a87dc 100644 --- a/python/paddle/fluid/tests/unittests/test_rmsprop_op.py +++ b/python/paddle/fluid/tests/unittests/test_rmsprop_op.py @@ -15,90 +15,164 @@ from __future__ import print_function import unittest + import numpy as np -from op_test import OpTest - - -class TestRmspropOp1(OpTest): - ''' Test RMSProp with explicit inputs - ''' - - def setUp(self): - self.op_type = "rmsprop" - - param = np.random.random((123, 321)).astype("float32") - mean_square = np.random.random((123, 321)).astype("float32") - learning_rate = np.array([0.01]).astype("float32") - grad = np.random.random((123, 321)).astype("float32") - moment = np.zeros((123, 321)).astype("float32") - - epsilon = 1e-6 - decay = 0.9 - momentum = 0.0 - - self.inputs = { - 'Param': param, - 'MeanSquare': mean_square, - 'LearningRate': learning_rate, - 'Grad': grad, - 'Moment': moment, - } - - self.attrs = {'epsilon': epsilon, 'decay': decay, 'momentum': momentum} - - ms_out = decay * mean_square + (1 - decay) * grad * grad - moment_out = momentum * moment + \ - learning_rate * grad / np.sqrt(ms_out + epsilon) - param_out = param - moment_out - - self.outputs = { - 'ParamOut': param_out, - 'MomentOut': moment_out, - 'MeanSquareOut': ms_out - } - - def test_check_output(self): - self.check_output() - - -class TestRmspropOp2(OpTest): - '''Test RMSProp with default values for attributes - ''' - - def setUp(self): - self.op_type = "rmsprop" - - param = np.random.random((123, 321)).astype("float32") - mean_square = np.random.random((123, 321)).astype("float32") - learning_rate = np.array([0.01]).astype("float32") - grad = np.random.random((123, 321)).astype("float32") - moment = np.zeros((123, 321)).astype("float32") - - epsilon = 1.0e-10 - decay = 0.9 - momentum = 0.0 - - self.inputs = { - 'Param': param, - 'MeanSquare': mean_square, - 'LearningRate': learning_rate, - 'Grad': grad, - 'Moment': moment, - } - - ms_out = decay * mean_square + (1 - decay) * grad * grad - moment_out = momentum * moment + \ - learning_rate * grad / np.sqrt(ms_out + epsilon) - param_out = param - moment_out - - self.outputs = { - 'ParamOut': param_out, - 'MomentOut': moment_out, - 'MeanSquareOut': ms_out - } - - def test_check_output(self): - self.check_output() +import paddle.fluid.core as core +from paddle.fluid.op import Operator + + +class TestBase(unittest.TestCase): + def setup(self, centered, epsilon=1e-6): + np.random.seed(5) # fix seed + + self.param_name = "param" + self.param = np.random.random((123, 321)).astype("float32") + + self.mean_square_name = "mean_square" + self.mean_square = np.random.random((123, 321)).astype("float32") + + self.mean_grad_name = "mean_grad" + self.mean_grad = np.random.random((123, 321)).astype("float32") + + self.lr_name = "lr" + self.learning_rate = np.array([0.01]).astype("float32") + + self.grad_name = "grad" + self.grad = np.random.random((123, 321)).astype("float32") + + self.moment_name = "moment" + self.moment = np.zeros((123, 321)).astype("float32") + + self.epsilon = epsilon + self.decay = 0.9 + self.momentum = 0.0 + self.centered = centered + + self.ms_out = self.decay * self.mean_square + (1 - self.decay + ) * self.grad * self.grad + if centered: + self.mg_out = self.decay * self.mean_grad + (1 - self.decay + ) * self.grad + self.moment_out = self.momentum * self.moment + \ + self.learning_rate * self.grad / np.sqrt(self.ms_out - np.square(self.mg_out) + self.epsilon) + else: + self.moment_out = self.momentum * self.moment + \ + self.learning_rate * self.grad / np.sqrt(self.ms_out + self.epsilon) + + self.param_out = self.param - self.moment_out + + def check(self, + actual_t, + expect_t, + place, + out_name, + atol=1e-5, + equal_nan=False): + self.assertTrue( + np.allclose( + actual_t, expect_t, atol=atol, equal_nan=equal_nan), + "Output (" + out_name + ") has diff at " + str(place) + "\nExpect " + + str(expect_t) + "\n" + "But Got" + str(actual_t)) + + +class TestRmspropOp(TestBase): + def check_with_place(self, place, centered, epsilon): + self.setup(centered, epsilon) + scope = core.Scope() + + # create and initialize Param Variable + param = scope.var(self.param_name).get_tensor() + param.set(self.param, place) + + mean_square = scope.var(self.mean_square_name).get_tensor() + mean_square.set(self.mean_square, place) + + lr = scope.var(self.lr_name).get_tensor() + lr.set(self.learning_rate, place) + + grad = scope.var(self.grad_name).get_tensor() + grad.set(self.grad, place) + + moment = scope.var(self.moment_name).get_tensor() + moment.set(self.moment, place) + + # create and run sgd operator + + if self.centered: + mean_grad = scope.var(self.mean_grad_name).get_tensor() + mean_grad.set(self.mean_grad, place) + + rmsprop_op = Operator( + "rmsprop", + Param=self.param_name, + Grad=self.grad_name, + MeanSquare=self.mean_square_name, + MeanGrad=self.mean_grad_name, + Moment=self.moment_name, + LearningRate=self.lr_name, + ParamOut=self.param_name, + MeanSquareOut=self.mean_square_name, + MomentOut=self.moment_name, + MeanGradOut=self.mean_grad_name, + epsilon=self.epsilon, + decay=self.decay, + momentum=self.momentum, + centered=True) + else: + rmsprop_op = Operator( + "rmsprop", + Param=self.param_name, + Grad=self.grad_name, + MeanSquare=self.mean_square_name, + Moment=self.moment_name, + LearningRate=self.lr_name, + ParamOut=self.param_name, + MeanSquareOut=self.mean_square_name, + MomentOut=self.moment_name, + epsilon=self.epsilon, + decay=self.decay, + momentum=self.momentum, + centered=False) + + rmsprop_op.run(scope, place) + + atol = 1e-5 + equal_nan = False + + if self.centered: + atol = 1e-3 + equal_nan = True + + self.check( + np.array(mean_square), self.ms_out, place, self.mean_square_name) + self.check( + np.array(moment), + self.moment_out, + place, + self.moment_name, + atol=atol, + equal_nan=equal_nan) + self.check( + np.array(param), + self.param_out, + place, + self.param_name, + atol=atol, + equal_nan=equal_nan) + + if self.centered: + self.check( + np.array(mean_grad), self.mg_out, place, self.mean_grad_name) + + def test_rmsprop(self): + places = [core.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(core.CUDAPlace(0)) + for place in places: + self.check_with_place(place, False, 1e-6) + self.check_with_place(place, False, 1e-10) + self.check_with_place(place, True, 1e-6) + self.check_with_place(place, True, 1e-10) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_squeeze_op.py b/python/paddle/fluid/tests/unittests/test_squeeze_op.py index 2be8e24a0fae6945351eb767ac924d7ca70848ab..204a4bb40196bd1fc2f5861aa31cf9560ea4d349 100644 --- a/python/paddle/fluid/tests/unittests/test_squeeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_squeeze_op.py @@ -23,14 +23,17 @@ from op_test import OpTest # Correct: General. class TestSqueezeOp(OpTest): def setUp(self): - self.op_type = "squeeze" + self.op_type = "squeeze2" self.init_test_case() self.inputs = {"X": np.random.random(self.ori_shape).astype("float32")} self.init_attrs() - self.outputs = {"Out": self.inputs["X"].reshape(self.new_shape)} + self.outputs = { + "Out": self.inputs["X"].reshape(self.new_shape), + "XShape": np.random.random(self.ori_shape).astype("float32") + } def test_check_output(self): - self.check_output() + self.check_output(no_check_set=['XShape']) def test_check_grad(self): self.check_grad(["X"], "Out") diff --git a/python/paddle/fluid/tests/unittests/test_transpose_op.py b/python/paddle/fluid/tests/unittests/test_transpose_op.py index 0853f80b82030679d140f7fabdd42557c2374599..c30da2389d50d3b6bdf1f911aaed6ed71f274153 100644 --- a/python/paddle/fluid/tests/unittests/test_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_transpose_op.py @@ -22,16 +22,19 @@ from op_test import OpTest class TestTransposeOp(OpTest): def setUp(self): self.initTestCase() - self.op_type = "transpose" + self.op_type = "transpose2" self.inputs = {'X': np.random.random(self.shape).astype("float32")} self.attrs = {'axis': list(self.axis)} - self.outputs = {'Out': self.inputs['X'].transpose(self.axis)} + self.outputs = { + 'XShape': np.random.random(self.shape).astype("float32"), + 'Out': self.inputs['X'].transpose(self.axis) + } def test_check_output(self): - self.check_output() + self.check_output(no_check_set=['XShape']) def test_check_grad(self): - self.check_grad(['X'], 'Out') + self.check_grad(['X'], 'Out', sum_outputs=['Out']) def initTestCase(self): self.shape = (3, 4) diff --git a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py index a324438ba5a3c3b57fd956bd11189ef7d50267e2..14dd2bb06f9a18d0b15a4aee4e9e6bfdf8c41206 100644 --- a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py @@ -24,13 +24,16 @@ from op_test import OpTest class TestUnsqueezeOp(OpTest): def setUp(self): self.init_test_case() - self.op_type = "unsqueeze" + self.op_type = "unsqueeze2" self.inputs = {"X": np.random.random(self.ori_shape).astype("float32")} self.init_attrs() - self.outputs = {"Out": self.inputs["X"].reshape(self.new_shape)} + self.outputs = { + "Out": self.inputs["X"].reshape(self.new_shape), + "XShape": np.random.random(self.ori_shape).astype("float32") + } def test_check_output(self): - self.check_output() + self.check_output(no_check_set=["XShape"]) def test_check_grad(self): self.check_grad(["X"], "Out") diff --git a/python/paddle/fluid/trainer.py b/python/paddle/fluid/trainer.py index d094647afe1900809fc32cae93f777765f72c675..30cdfe4ad2c9892184862b70ff49417ce5a08516 100644 --- a/python/paddle/fluid/trainer.py +++ b/python/paddle/fluid/trainer.py @@ -431,6 +431,28 @@ class Trainer(object): exe = executor.Executor(self.place) io.save_persistables(exe, dirname=param_path) + def save_inference_model(self, param_path, feeded_var_names, + target_var_indexes): + """ + Save model for cpp inference into :code:`param_path`. + + Args: + param_path(str): The path to save parameters. + feeded_var_names(list(str)): The name of the vars that you + need to feed in before run program. + target_var_indexes(list(int)): the index of target var that + you need to return in trainer.train_func. + Returns: + None + """ + with self._prog_and_scope_guard(): + exe = executor.Executor(self.place) + target_vars = [ + self.train_func_outputs[index] for index in target_var_indexes + ] + io.save_inference_model(param_path, feeded_var_names, target_vars, + exe) + @contextlib.contextmanager def _prog_and_scope_guard(self): with framework.program_guard( diff --git a/python/paddle/fluid/transpiler/details/program_utils.py b/python/paddle/fluid/transpiler/details/program_utils.py index f0fafaa84a73d641ff6ceb74def6addaea759516..a83aa0f11eed9bfc1674d8d75dcfacc297f056b0 100644 --- a/python/paddle/fluid/transpiler/details/program_utils.py +++ b/python/paddle/fluid/transpiler/details/program_utils.py @@ -153,7 +153,7 @@ def block_to_code(block, block_idx): indent += 1 # sort all vars - all_vars = sorted(block.vars.iteritems(), key=lambda x: x[0]) + all_vars = sorted(six.iteritems(block.vars), key=lambda x: x[0]) for var in all_vars: print("{}{}".format(get_indent_space(indent), variable_to_code(var[1]))) diff --git a/python/paddle/fluid/transpiler/distribute_transpiler.py b/python/paddle/fluid/transpiler/distribute_transpiler.py index 8a330e0dee7eda02d0858446778363f2235a3d73..d4d218d547a394a56c040ade2a9ba703b691b86b 100644 --- a/python/paddle/fluid/transpiler/distribute_transpiler.py +++ b/python/paddle/fluid/transpiler/distribute_transpiler.py @@ -300,7 +300,7 @@ class DistributeTranspiler(object): input_deps = grad_name_to_send_dummy_out.values() program.global_block().append_op( type="send_barrier", - inputs={"X": input_deps}, + inputs={"X": list(input_deps)}, outputs={"Out": send_barrier_out}, attrs={ "endpoints": pserver_endpoints, @@ -401,7 +401,7 @@ class DistributeTranspiler(object): Args: recv_vars (list): Variable list to recv for current trainer_id - eplist (list): A list of strings indicating + eplist (list): A list of strings indicating Returns: Program: trainer side startup program. @@ -455,7 +455,7 @@ class DistributeTranspiler(object): if len(splited_var) <= 1: continue # NOTE: if enable memory optimization, origin vars maybe removed. - if startup_program.global_block().vars.has_key(varname): + if varname in startup_program.global_block().vars: orig_param = startup_program.global_block().vars[varname] else: origin_param_var = self.origin_program.global_block().vars[ @@ -690,7 +690,7 @@ class DistributeTranspiler(object): Args: endpoint (str): current pserver endpoint. - + Returns: tuple: (main_program, startup_program), of type "Program" """ @@ -713,7 +713,7 @@ class DistributeTranspiler(object): endpoint (str): current pserver endpoint. pserver_program (Program): deprecated, call get_pserver_program first. startup_program (Program): deprecated, should pass startup_program - when initalizing + when initalizing Returns: Program: parameter server side startup program.