From a62de41a34dc75676b45cd8ad2267e2b2d404f84 Mon Sep 17 00:00:00 2001 From: Zhenghai Zhang <65210872+ccsuzzh@users.noreply.github.com> Date: Fri, 7 Apr 2023 14:17:26 +0800 Subject: [PATCH] add autogen code support for warpctc op (#52610) --- paddle/fluid/operators/CMakeLists.txt | 12 +- paddle/fluid/operators/warpctc_op.cc | 170 ----------------------- paddle/phi/api/yaml/backward.yaml | 13 ++ paddle/phi/api/yaml/legacy_backward.yaml | 12 -- paddle/phi/api/yaml/legacy_ops.yaml | 12 -- paddle/phi/api/yaml/op_compat.yaml | 7 + paddle/phi/api/yaml/ops.yaml | 12 ++ paddle/phi/ops/compat/warpctc_sig.cc | 37 ----- 8 files changed, 33 insertions(+), 242 deletions(-) delete mode 100644 paddle/fluid/operators/warpctc_op.cc delete mode 100644 paddle/phi/ops/compat/warpctc_sig.cc diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt index 29f9bcbb310..4070d680394 100644 --- a/paddle/fluid/operators/CMakeLists.txt +++ b/paddle/fluid/operators/CMakeLists.txt @@ -92,7 +92,7 @@ endif() set(OP_HEADER_DEPS ${OP_HEADER_DEPS} phi phi_utils backward_infermeta sparse_backward_infermeta static_prim_api get_expected_kernel_func) -register_operators(EXCLUDES py_func_op warpctc_op dgc_op generated_op1 generated_op2 generated_op3 generated_op4 load_combine_op lstm_op run_program_op quantize_linear_op +register_operators(EXCLUDES py_func_op dgc_op generated_op1 generated_op2 generated_op3 generated_op4 load_combine_op lstm_op run_program_op quantize_linear_op recurrent_op save_combine_op sparse_attention_op sync_batch_norm_op activation_op ${OP_MKL_DEPS} DEPS ${OP_HEADER_DEPS}) op_library(generated_op UNITY SRCS generated_op1.cc generated_op2.cc generated_op3.cc generated_op4.cc DEPS ${OP_HEADER_DEPS}) @@ -111,20 +111,10 @@ else() endif() if (WITH_GPU OR WITH_ROCM) - if(WITH_ROCM) - op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale SRCS warpctc_op.cc) - # warpctc_op needs cudnn 7 above - elseif(${CUDNN_MAJOR_VERSION} VERSION_LESS 7) - op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale SRCS warpctc_op.cc) - else() - op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale) - endif() op_library(sync_batch_norm_op) if ((NOT WIN32) AND (NOT WITH_ROCM) AND (NOT PADDLE_WITH_ARM) AND (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_LESS 11.3) ) op_library(sparse_attention_op) endif() -else() - op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale) endif() if (WITH_ASCEND_CL) diff --git a/paddle/fluid/operators/warpctc_op.cc b/paddle/fluid/operators/warpctc_op.cc deleted file mode 100644 index 93811358c7e..00000000000 --- a/paddle/fluid/operators/warpctc_op.cc +++ /dev/null @@ -1,170 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include - -#include "paddle/fluid/framework/infershape_utils.h" -#include "paddle/fluid/framework/op_registry.h" -#include "paddle/fluid/platform/device/gpu/gpu_dnn.h" -#include "paddle/phi/core/infermeta_utils.h" -#include "paddle/phi/infermeta/multiary.h" - -namespace paddle { -namespace operators { - -class WarpCTCOp : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - phi::KernelKey GetExpectedKernelType( - const framework::ExecutionContext& ctx) const override { - return phi::KernelKey( - OperatorWithKernel::IndicateVarDataType(ctx, "Logits"), ctx.GetPlace()); - } -}; - -class WarpCTCOpMaker : public framework::OpProtoAndCheckerMaker { - public: - void Make() override { - AddInput("Logits", - "(2-D phi::DenseTensor) or (3-D phi::DenseTensor), " - "the unscaled probabilities of variable-length sequences." - "When is a 2-D Tensor with LoD information, " - "it's shape is [Lp, num_classes + 1], " - "where Lp is the sum of all input sequences' length " - "and num_classes is the true number of classes " - "(not including the blank label)." - "When it is 3-D Tensor, it's shape is " - "[max_logit_length, batch_size, num_classes + 1], " - "where max_logit_length is the length of the longest " - "logit sequence."); - AddInput("Label", - "(2-D phi::DenseTensor), the " - "ground truth of variable-length sequence. " - "When it is a 2-D Tensor with LoD information, " - "it is of the shape [Lg, 1], where Lg is th sum of " - "all labels' length." - "When it is a 2-D Tensor, it's shape is also [Lg, 1]."); - AddInput("LogitsLength", - "1-D Tensor. " - "Input sequence length for Logits when Logits is a 3-D tensor.") - .AsDispensable(); - AddInput("LabelLength", - "1-D Tensor. " - "Target sequence length for Label when Label is a 2-D tensor.") - .AsDispensable(); - AddOutput("WarpCTCGrad", - "(Tensor), a temporary " - "output Tensor to store the gradients of warp-ctc, which is " - "computed with loss together in one call. It is a 3-D Tensor of " - "the shape [max_sequence_length, batch_size, num_classes + 1].") - .AsIntermediate(); - AddOutput("Loss", - "(Tensor), the Connectionist " - "Temporal Classification (CTC) loss, which is a 2-D Tensor of " - "the shape [batch_size, 1]"); - AddAttr("blank", - "(int, default: 0), the blank label of Connectionist " - "Temporal Classification (CTC) loss, which is in the " - "half-opened interval [0, num_classes + 1).") - .SetDefault(0); - AddAttr("norm_by_times", - "(bool, default: false), whether to " - "normalize the gradients by the number of time-step, " - "which is also the sequence's length.") - .SetDefault(false); - AddComment(R"DOC( -An operator integrating the open-source -[warp-ctc](https://github.com/baidu-research/warp-ctc) library, which is used in -[Deep Speech 2: End-toEnd Speech Recognition in English and Mandarin]( -https://arxiv.org/pdf/1512.02595v1.pdf), -to compute Connectionist Temporal Classification (CTC) loss. -It can be aliased as softmax with ctc, since a native softmax activation is -interated to the warp-ctc library, to normalize values for each row of the -input tensor. - -More detail of CTC loss can be found by referring to -[Connectionist Temporal Classification: Labelling Unsegmented Sequence Data with -Recurrent Neural Networks]( -http://machinelearning.wustl.edu/mlpapers/paper_files/icml2006_GravesFGS06.pdf). -)DOC"); - } -}; - -template -class WarpCTCGradOpMaker : public framework::SingleGradOpMaker { - public: - using framework::SingleGradOpMaker::SingleGradOpMaker; - - protected: - void Apply(GradOpPtr op) const override { - op->SetType("warpctc_grad"); - - op->SetInput("WarpCTCGrad", this->Output("WarpCTCGrad")); - op->SetInput("Logits", this->Input("Logits")); - op->SetInput(framework::GradVarName("Loss"), this->OutputGrad("Loss")); - - op->SetInput("LogitsLength", this->Input("LogitsLength")); - - op->SetOutput(framework::GradVarName("Logits"), this->InputGrad("Logits")); - - op->SetAttrMap(this->Attrs()); - } -}; - -class WarpCTCGradOp : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - void InferShape(framework::InferShapeContext* ctx) const override { - OP_INOUT_CHECK( - ctx->HasInput("WarpCTCGrad"), "Input", "WarpCTCGrad", "WarpCTCGrad"); - OP_INOUT_CHECK(ctx->HasOutput(framework::GradVarName("Logits")), - "Output", - framework::GradVarName("Logits"), - "WarpCTCGrad"); - ctx->SetOutputDim(framework::GradVarName("Logits"), - ctx->GetInputDim("Logits")); - ctx->ShareLoD("Logits", /*->*/ framework::GradVarName("Logits")); - } - - protected: - phi::KernelKey GetExpectedKernelType( - const framework::ExecutionContext& ctx) const override { - return phi::KernelKey(OperatorWithKernel::IndicateVarDataType( - ctx, framework::GradVarName("Loss")), - ctx.GetPlace()); - } -}; - -DECLARE_NO_NEED_BUFFER_VARS_INFERER(WarpCTCGradOpNoNeedBufferVarInferer, - "Logits"); - -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -DECLARE_INFER_SHAPE_FUNCTOR(warpctc, - WarpctcInferShapeFunctor, - PD_INFER_META(phi::WarpctcInferMeta)); -REGISTER_OPERATOR(warpctc, - ops::WarpCTCOp, - ops::WarpCTCOpMaker, - ops::WarpCTCGradOpMaker, - ops::WarpCTCGradOpMaker, - WarpctcInferShapeFunctor); -REGISTER_OPERATOR(warpctc_grad, - ops::WarpCTCGradOp, - ops::WarpCTCGradOpNoNeedBufferVarInferer); diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml index 3ebb72f4fdb..b8244915369 100644 --- a/paddle/phi/api/yaml/backward.yaml +++ b/paddle/phi/api/yaml/backward.yaml @@ -1869,6 +1869,19 @@ kernel : func : unstack_grad +- backward_op : warpctc_grad + forward : warpctc (Tensor logits, Tensor label, Tensor logits_length, Tensor labels_length, int blank = 0, bool norm_by_times = false) -> Tensor(loss), Tensor(warpctcgrad) + args : (Tensor logits, Tensor logits_length, Tensor warpctcgrad, Tensor loss_grad, int blank, bool norm_by_times) + output : Tensor(logits_grad) + infer_meta : + func : UnchangedInferMeta + param : [logits] + kernel : + func : warpctc_grad + data_type : loss_grad + optional : logits_length + no_need_buffer : logits + - backward_op : warprnnt_grad forward : warprnnt (Tensor input, Tensor label, Tensor input_lengths, Tensor label_lengths, int blank = 0, float fastemit_lambda = 0.0) -> Tensor(loss), Tensor(warprnntgrad) args : (Tensor input, Tensor input_lengths, Tensor warprnntgrad, Tensor loss_grad, int blank = 0, float fastemit_lambda = 0.0) diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml index 94c34f775b0..d60f2b70597 100755 --- a/paddle/phi/api/yaml/legacy_backward.yaml +++ b/paddle/phi/api/yaml/legacy_backward.yaml @@ -1224,18 +1224,6 @@ func : uniform_inplace_grad inplace : (out_grad -> x_grad) -- backward_op : warpctc_grad - forward : warpctc (Tensor logits, Tensor label, Tensor logits_length, Tensor labels_length, int blank, bool norm_by_times) -> Tensor(loss), Tensor(warpctcgrad) - args : (Tensor logits, Tensor logits_length, Tensor warpctcgrad, Tensor loss_grad, int blank, bool norm_by_times) - output : Tensor(logits_grad) - infer_meta : - func : UnchangedInferMeta - param : [logits] - kernel : - func : warpctc_grad - optional : logits_length - no_need_buffer : logits - - backward_op : yolo_loss_grad forward : yolo_loss(Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, int[] anchors, int[] anchor_mask, int class_num, float ignore_thresh, int downsample_ratio, bool use_label_smooth=true, float scale_x_y=1.0) -> Tensor(loss), Tensor(objectness_mask), Tensor(gt_match_mask) args : (Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, Tensor objectness_mask, Tensor gt_match_mask, Tensor loss_grad, int[] anchors, int[] anchor_mask, int class_num, float ignore_thresh, int downsample_ratio, bool use_label_smooth=true, float scale_x_y=1.0) diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml index 8e97c11ae7b..770062f2817 100755 --- a/paddle/phi/api/yaml/legacy_ops.yaml +++ b/paddle/phi/api/yaml/legacy_ops.yaml @@ -1547,18 +1547,6 @@ data_type: x backward: unpool3d_grad -- op : warpctc - args : (Tensor logits, Tensor label, Tensor logits_length, Tensor labels_length, int blank, bool norm_by_times) - output : Tensor(loss), Tensor(warpctcgrad) - infer_meta : - func : WarpctcInferMeta - kernel : - func : warpctc - data_type: logits - optional: logits_length, labels_length - intermediate: warpctcgrad - backward : warpctc_grad - - op : yolo_box args : (Tensor x, Tensor img_size, int[] anchors, int class_num, float conf_thresh, int downsample_ratio, bool clip_bbox, float scale_x_y=1.0, bool iou_aware=false, float iou_aware_factor=0.5) output : Tensor(boxes), Tensor(scores) diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index 549029bcd22..6dab0c0f4c3 100644 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -2193,6 +2193,13 @@ outputs : {scores : Scores, path : Path} +- op : warpctc + backward : warpctc_grad + inputs : + {logits : Logits, label : Label, logits_length : LogitsLength, labels_length : LabelLength} + outputs : + {warpctcgrad : WarpCTCGrad, loss : Loss} + - op : where backward : where_grad inputs : diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index cb55a85b45c..60408958521 100644 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -1827,6 +1827,18 @@ func : viterbi_decode data_type : potentials +- op : warpctc + args : (Tensor logits, Tensor label, Tensor logits_length, Tensor labels_length, int blank = 0, bool norm_by_times = false) + output : Tensor(loss), Tensor(warpctcgrad) + infer_meta : + func : WarpctcInferMeta + kernel : + func : warpctc + data_type: logits + optional: logits_length, labels_length + intermediate: warpctcgrad + backward : warpctc_grad + - op : warprnnt args : (Tensor input, Tensor label, Tensor input_lengths, Tensor label_lengths, int blank = 0, float fastemit_lambda = 0.0) output : Tensor(loss), Tensor(warprnntgrad) diff --git a/paddle/phi/ops/compat/warpctc_sig.cc b/paddle/phi/ops/compat/warpctc_sig.cc deleted file mode 100644 index 44878aabff2..00000000000 --- a/paddle/phi/ops/compat/warpctc_sig.cc +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/phi/core/compat/op_utils.h" - -namespace phi { - -KernelSignature WarpctcOpArgumentMapping(const ArgumentMappingContext& ctx) { - return KernelSignature("warpctc", - {"Logits", "Label", "LogitsLength", "LabelLength"}, - {"blank", "norm_by_times"}, - {"Loss", "WarpCTCGrad"}); -} - -KernelSignature WarpctcGradOpArgumentMapping( - const ArgumentMappingContext& ctx) { - return KernelSignature("warpctc_grad", - {"Logits", "LogitsLength", "WarpCTCGrad", "Loss@GRAD"}, - {"blank", "norm_by_times"}, - {"Logits@GRAD"}); -} - -} // namespace phi - -PD_REGISTER_ARG_MAPPING_FN(warpctc, phi::WarpctcOpArgumentMapping); -PD_REGISTER_ARG_MAPPING_FN(warpctc_grad, phi::WarpctcGradOpArgumentMapping); -- GitLab