add autogen code support for warpctc op (#52610)

paddle/fluid/operators/CMakeLists.txt

@@ -92,7 +92,7 @@ endif()
 
 set(OP_HEADER_DEPS ${OP_HEADER_DEPS} phi phi_utils backward_infermeta sparse_backward_infermeta static_prim_api get_expected_kernel_func)
 
-register_operators(EXCLUDES py_func_op warpctc_op dgc_op generated_op1 generated_op2 generated_op3 generated_op4 load_combine_op lstm_op run_program_op quantize_linear_op
+register_operators(EXCLUDES py_func_op dgc_op generated_op1 generated_op2 generated_op3 generated_op4 load_combine_op lstm_op run_program_op quantize_linear_op
         recurrent_op save_combine_op sparse_attention_op sync_batch_norm_op activation_op ${OP_MKL_DEPS} DEPS ${OP_HEADER_DEPS})
 
 op_library(generated_op UNITY SRCS generated_op1.cc generated_op2.cc generated_op3.cc generated_op4.cc DEPS ${OP_HEADER_DEPS})
@@ -111,20 +111,10 @@ else()
 endif()
 
 if (WITH_GPU OR WITH_ROCM)
-    if(WITH_ROCM)
-        op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale SRCS warpctc_op.cc)
-    # warpctc_op needs cudnn 7 above
-    elseif(${CUDNN_MAJOR_VERSION} VERSION_LESS 7)
-        op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale SRCS warpctc_op.cc)
-    else()
-        op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale)
-    endif()
     op_library(sync_batch_norm_op)
     if ((NOT WIN32) AND (NOT WITH_ROCM) AND (NOT PADDLE_WITH_ARM) AND (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_LESS 11.3) )
         op_library(sparse_attention_op)
     endif()
-else()
-    op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale)
 endif()
 
 if (WITH_ASCEND_CL)

paddle/fluid/operators/warpctc_op.cc

-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include <memory>
-
-#include "paddle/fluid/framework/infershape_utils.h"
-#include "paddle/fluid/framework/op_registry.h"
-#include "paddle/fluid/platform/device/gpu/gpu_dnn.h"
-#include "paddle/phi/core/infermeta_utils.h"
-#include "paddle/phi/infermeta/multiary.h"
-
-namespace paddle {
-namespace operators {
-
-class WarpCTCOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    return phi::KernelKey(
-        OperatorWithKernel::IndicateVarDataType(ctx, "Logits"), ctx.GetPlace());
-  }
-};
-
-class WarpCTCOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("Logits",
-             "(2-D phi::DenseTensor<float>) or (3-D phi::DenseTensor<float>), "
-             "the unscaled probabilities of variable-length sequences."
-             "When is a 2-D Tensor with LoD information, "
-             "it's shape is [Lp, num_classes + 1], "
-             "where Lp is the sum of all input sequences' length "
-             "and num_classes is the true number of classes "
-             "(not including the blank label)."
-             "When it is 3-D Tensor, it's shape is "
-             "[max_logit_length, batch_size, num_classes + 1], "
-             "where max_logit_length is the length of the longest "
-             "logit sequence.");
-    AddInput("Label",
-             "(2-D phi::DenseTensor<int>), the "
-             "ground truth of variable-length sequence. "
-             "When it is a 2-D Tensor with LoD information, "
-             "it is of the shape [Lg, 1], where Lg is th sum of "
-             "all labels' length."
-             "When it is a 2-D Tensor<int>, it's shape is also [Lg, 1].");
-    AddInput("LogitsLength",
-             "1-D Tensor<int64_t>. "
-             "Input sequence length for Logits when Logits is a 3-D tensor.")
-        .AsDispensable();
-    AddInput("LabelLength",
-             "1-D Tensor<int64_t>. "
-             "Target sequence length for Label when Label is a 2-D tensor.")
-        .AsDispensable();
-    AddOutput("WarpCTCGrad",
-              "(Tensor), a temporary "
-              "output Tensor to store the gradients of warp-ctc, which is "
-              "computed with loss together in one call. It is a 3-D Tensor of "
-              "the shape [max_sequence_length, batch_size, num_classes + 1].")
-        .AsIntermediate();
-    AddOutput("Loss",
-              "(Tensor), the Connectionist "
-              "Temporal Classification (CTC) loss, which is a 2-D Tensor of "
-              "the shape [batch_size, 1]");
-    AddAttr<int>("blank",
-                 "(int, default: 0), the blank label of Connectionist "
-                 "Temporal Classification (CTC) loss, which is in the "
-                 "half-opened interval [0, num_classes + 1).")
-        .SetDefault(0);
-    AddAttr<bool>("norm_by_times",
-                  "(bool, default: false), whether to "
-                  "normalize the gradients by the number of time-step, "
-                  "which is also the sequence's length.")
-        .SetDefault(false);
-    AddComment(R"DOC(
-An operator integrating the open-source
-[warp-ctc](https://github.com/baidu-research/warp-ctc) library, which is used in
-[Deep Speech 2: End-toEnd Speech Recognition in English and Mandarin](
-https://arxiv.org/pdf/1512.02595v1.pdf),
-to compute Connectionist Temporal Classification (CTC) loss.
-It can be aliased as softmax with ctc, since a native softmax activation is
-interated to the warp-ctc library, to normalize values for each row of the
-input tensor.
-
-More detail of CTC loss can be found by referring to
-[Connectionist Temporal Classification: Labelling Unsegmented Sequence Data with
-Recurrent Neural Networks](
-http://machinelearning.wustl.edu/mlpapers/paper_files/icml2006_GravesFGS06.pdf).
-)DOC");
-  }
-};
-
-template <typename T>
-class WarpCTCGradOpMaker : public framework::SingleGradOpMaker<T> {
- public:
-  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
-
- protected:
-  void Apply(GradOpPtr<T> op) const override {
-    op->SetType("warpctc_grad");
-
-    op->SetInput("WarpCTCGrad", this->Output("WarpCTCGrad"));
-    op->SetInput("Logits", this->Input("Logits"));
-    op->SetInput(framework::GradVarName("Loss"), this->OutputGrad("Loss"));
-
-    op->SetInput("LogitsLength", this->Input("LogitsLength"));
-
-    op->SetOutput(framework::GradVarName("Logits"), this->InputGrad("Logits"));
-
-    op->SetAttrMap(this->Attrs());
-  }
-};
-
-class WarpCTCGradOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-
-  void InferShape(framework::InferShapeContext* ctx) const override {
-    OP_INOUT_CHECK(
-        ctx->HasInput("WarpCTCGrad"), "Input", "WarpCTCGrad", "WarpCTCGrad");
-    OP_INOUT_CHECK(ctx->HasOutput(framework::GradVarName("Logits")),
-                   "Output",
-                   framework::GradVarName("Logits"),
-                   "WarpCTCGrad");
-    ctx->SetOutputDim(framework::GradVarName("Logits"),
-                      ctx->GetInputDim("Logits"));
-    ctx->ShareLoD("Logits", /*->*/ framework::GradVarName("Logits"));
-  }
-
- protected:
-  phi::KernelKey GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    return phi::KernelKey(OperatorWithKernel::IndicateVarDataType(
-                              ctx, framework::GradVarName("Loss")),
-                          ctx.GetPlace());
-  }
-};
-
-DECLARE_NO_NEED_BUFFER_VARS_INFERER(WarpCTCGradOpNoNeedBufferVarInferer,
-                                    "Logits");
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-DECLARE_INFER_SHAPE_FUNCTOR(warpctc,
-                            WarpctcInferShapeFunctor,
-                            PD_INFER_META(phi::WarpctcInferMeta));
-REGISTER_OPERATOR(warpctc,
-                  ops::WarpCTCOp,
-                  ops::WarpCTCOpMaker,
-                  ops::WarpCTCGradOpMaker<paddle::framework::OpDesc>,
-                  ops::WarpCTCGradOpMaker<paddle::imperative::OpBase>,
-                  WarpctcInferShapeFunctor);
-REGISTER_OPERATOR(warpctc_grad,
-                  ops::WarpCTCGradOp,
-                  ops::WarpCTCGradOpNoNeedBufferVarInferer);

paddle/phi/api/yaml/backward.yaml

@@ -1869,6 +1869,19 @@
   kernel :
     func : unstack_grad
 
+- backward_op : warpctc_grad
+  forward : warpctc (Tensor logits, Tensor label, Tensor logits_length, Tensor labels_length, int blank = 0, bool norm_by_times = false) -> Tensor(loss), Tensor(warpctcgrad)
+  args : (Tensor logits, Tensor logits_length, Tensor warpctcgrad, Tensor loss_grad, int blank, bool norm_by_times)
+  output : Tensor(logits_grad)
+  infer_meta :
+    func : UnchangedInferMeta
+    param : [logits]
+  kernel :
+    func : warpctc_grad
+    data_type : loss_grad
+  optional : logits_length
+  no_need_buffer : logits
+
 - backward_op : warprnnt_grad
   forward : warprnnt (Tensor input, Tensor label, Tensor input_lengths, Tensor label_lengths, int blank = 0, float fastemit_lambda = 0.0) -> Tensor(loss), Tensor(warprnntgrad)
   args : (Tensor input, Tensor input_lengths, Tensor warprnntgrad, Tensor loss_grad, int blank = 0, float fastemit_lambda = 0.0)

paddle/phi/api/yaml/legacy_backward.yaml

@@ -1224,18 +1224,6 @@
     func : uniform_inplace_grad
   inplace : (out_grad -> x_grad)
 
-- backward_op : warpctc_grad
-  forward : warpctc (Tensor logits, Tensor label, Tensor logits_length, Tensor labels_length, int blank, bool norm_by_times) -> Tensor(loss), Tensor(warpctcgrad)
-  args : (Tensor logits, Tensor logits_length, Tensor warpctcgrad, Tensor loss_grad, int blank, bool norm_by_times)
-  output : Tensor(logits_grad)
-  infer_meta :
-    func : UnchangedInferMeta
-    param : [logits]
-  kernel :
-    func : warpctc_grad
-  optional : logits_length
-  no_need_buffer : logits
-
 - backward_op : yolo_loss_grad
   forward : yolo_loss(Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, int[] anchors, int[] anchor_mask, int class_num, float ignore_thresh, int downsample_ratio, bool use_label_smooth=true, float scale_x_y=1.0) -> Tensor(loss), Tensor(objectness_mask), Tensor(gt_match_mask)
   args : (Tensor x, Tensor gt_box, Tensor gt_label, Tensor gt_score, Tensor objectness_mask, Tensor gt_match_mask, Tensor loss_grad, int[] anchors, int[] anchor_mask, int class_num, float ignore_thresh, int downsample_ratio, bool use_label_smooth=true, float scale_x_y=1.0)

paddle/phi/api/yaml/legacy_ops.yaml

@@ -1547,18 +1547,6 @@
     data_type: x
   backward: unpool3d_grad
 
-- op : warpctc
-  args : (Tensor logits, Tensor label, Tensor logits_length, Tensor labels_length, int blank, bool norm_by_times)
-  output :  Tensor(loss), Tensor(warpctcgrad)
-  infer_meta :
-    func : WarpctcInferMeta
-  kernel :
-    func : warpctc
-    data_type: logits
-  optional: logits_length, labels_length
-  intermediate: warpctcgrad
-  backward : warpctc_grad
-
 - op : yolo_box
   args : (Tensor x, Tensor img_size, int[] anchors, int class_num, float conf_thresh, int downsample_ratio, bool clip_bbox, float scale_x_y=1.0, bool iou_aware=false, float iou_aware_factor=0.5)
   output : Tensor(boxes), Tensor(scores)

paddle/phi/api/yaml/op_compat.yaml

@@ -2193,6 +2193,13 @@
   outputs :
     {scores : Scores, path : Path}
 
+- op : warpctc
+  backward : warpctc_grad
+  inputs :
+    {logits : Logits, label : Label, logits_length : LogitsLength, labels_length : LabelLength}
+  outputs :
+    {warpctcgrad : WarpCTCGrad, loss : Loss}
+
 - op : where
   backward : where_grad
   inputs :

paddle/phi/api/yaml/ops.yaml

@@ -1827,6 +1827,18 @@
     func : viterbi_decode
     data_type : potentials
 
+- op : warpctc
+  args : (Tensor logits, Tensor label, Tensor logits_length, Tensor labels_length, int blank = 0, bool norm_by_times = false)
+  output :  Tensor(loss), Tensor(warpctcgrad)
+  infer_meta :
+    func : WarpctcInferMeta
+  kernel :
+    func : warpctc
+    data_type: logits
+  optional: logits_length, labels_length
+  intermediate: warpctcgrad
+  backward : warpctc_grad
+
 - op : warprnnt
   args : (Tensor input, Tensor label, Tensor input_lengths, Tensor label_lengths, int blank = 0, float fastemit_lambda = 0.0)
   output :  Tensor(loss), Tensor(warprnntgrad)

paddle/phi/ops/compat/warpctc_sig.cc

-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/phi/core/compat/op_utils.h"
-
-namespace phi {
-
-KernelSignature WarpctcOpArgumentMapping(const ArgumentMappingContext& ctx) {
-  return KernelSignature("warpctc",
-                         {"Logits", "Label", "LogitsLength", "LabelLength"},
-                         {"blank", "norm_by_times"},
-                         {"Loss", "WarpCTCGrad"});
-}
-
-KernelSignature WarpctcGradOpArgumentMapping(
-    const ArgumentMappingContext& ctx) {
-  return KernelSignature("warpctc_grad",
-                         {"Logits", "LogitsLength", "WarpCTCGrad", "Loss@GRAD"},
-                         {"blank", "norm_by_times"},
-                         {"Logits@GRAD"});
-}
-
-}  // namespace phi
-
-PD_REGISTER_ARG_MAPPING_FN(warpctc, phi::WarpctcOpArgumentMapping);
-PD_REGISTER_ARG_MAPPING_FN(warpctc_grad, phi::WarpctcGradOpArgumentMapping);