From b32c13dc20a7d8751120f8b2c6554385dc124f29 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Fri, 16 Nov 2018 12:20:19 +0800 Subject: [PATCH] Add cudnn ctc loss (#12366) * add cudnn ctc loss * wip add test test=develop * wip * wip * done test=develop * move include cudnn test=develop * test test=develop * fix build test=develop * fix build test=develop * fix build on cudnn5 test=develop * fix cudnn5 build test=develop * fix cudnn5 build test=develop * merge develop softmax functor change test=develop --- paddle/fluid/API.spec | 2 +- paddle/fluid/operators/CMakeLists.txt | 9 +- paddle/fluid/operators/warpctc_cudnn_op.cu.cc | 195 ++++++++++++++++++ paddle/fluid/operators/warpctc_op.cc | 17 +- paddle/fluid/platform/cudnn_helper.h | 23 +++ paddle/fluid/platform/dynload/cudnn.h | 8 +- python/paddle/fluid/layers/nn.py | 10 +- .../fluid/tests/unittests/test_warpctc_op.py | 23 ++- 8 files changed, 279 insertions(+), 8 deletions(-) create mode 100644 paddle/fluid/operators/warpctc_cudnn_op.cu.cc diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index da835b3305..a23deebb25 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -93,7 +93,7 @@ paddle.fluid.layers.edit_distance ArgSpec(args=['input', 'label', 'normalized', paddle.fluid.layers.l2_normalize ArgSpec(args=['x', 'axis', 'epsilon', 'name'], varargs=None, keywords=None, defaults=(1e-12, None)) paddle.fluid.layers.matmul ArgSpec(args=['x', 'y', 'transpose_x', 'transpose_y', 'alpha', 'name'], varargs=None, keywords=None, defaults=(False, False, 1.0, None)) paddle.fluid.layers.topk ArgSpec(args=['input', 'k', 'name'], varargs=None, keywords=None, defaults=(None,)) -paddle.fluid.layers.warpctc ArgSpec(args=['input', 'label', 'blank', 'norm_by_times'], varargs=None, keywords=None, defaults=(0, False)) +paddle.fluid.layers.warpctc ArgSpec(args=['input', 'label', 'blank', 'norm_by_times', 'use_cudnn'], varargs=None, keywords=None, defaults=(0, False, False)) paddle.fluid.layers.sequence_reshape ArgSpec(args=['input', 'new_dim'], varargs=None, keywords=None, defaults=None) paddle.fluid.layers.transpose ArgSpec(args=['x', 'perm', 'name'], varargs=None, keywords=None, defaults=(None,)) paddle.fluid.layers.im2sequence ArgSpec(args=['input', 'filter_size', 'stride', 'padding', 'input_image_size', 'out_stride', 'name'], varargs=None, keywords=None, defaults=(1, 1, 0, None, 1, None)) diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt index f06ef199d1..2dc83c391b 100644 --- a/paddle/fluid/operators/CMakeLists.txt +++ b/paddle/fluid/operators/CMakeLists.txt @@ -300,7 +300,6 @@ if (NOT WIN32) op_library(gru_op DEPS sequence2batch gru_compute) endif(NOT WIN32) op_library(recurrent_op DEPS executor) -op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale) op_library(cos_sim_op DEPS cos_sim_functor) op_library(parallel_do_op DEPS executor) op_library(unsqueeze_op DEPS reshape_op) @@ -331,6 +330,14 @@ op_library(load_combine_op DEPS lod_tensor) op_library(concat_op DEPS concat_and_split) op_library(tensor_array_to_tensor_op DEPS concat_op) +set(DEPS_OPS ${DEPS_OPS} warpctc_op) +if (WITH_GPU) + if (${CUDNN_MAJOR_VERSION} VERSION_LESS 7) + op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale SRCS warpctc_op.cc warpctc_op.cu.cc) + endif() +endif() +op_library(warpctc_op DEPS dynload_warpctc sequence_padding sequence_scale) + list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS}) foreach(src ${GENERAL_OPS}) diff --git a/paddle/fluid/operators/warpctc_cudnn_op.cu.cc b/paddle/fluid/operators/warpctc_cudnn_op.cu.cc new file mode 100644 index 0000000000..a764d59410 --- /dev/null +++ b/paddle/fluid/operators/warpctc_cudnn_op.cu.cc @@ -0,0 +1,195 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/framework/mixed_vector.h" +#include "paddle/fluid/operators/math/softmax.h" +#include "paddle/fluid/operators/warpctc_op.h" +#include "paddle/fluid/platform/cudnn_helper.h" + +namespace paddle { +namespace operators { + +#if CUDNN_VERSION >= 7001 +using ScopedTensorDescriptor = platform::ScopedTensorDescriptor; +using ScopedCTCLossDescriptor = platform::ScopedCTCLossDescriptor; +using DataLayout = platform::DataLayout; + +template +class CudnnCTCKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + // =====================Copied code from warpctc=========================== + auto* logits = ctx.Input("Logits"); + auto* label = ctx.Input("Label"); + auto* warpctc_grad = ctx.Output("WarpCTCGrad"); + auto* loss = ctx.Output("Loss"); + + const size_t level = 0; + + auto logits_lod = framework::ToAbsOffset(logits->lod()); + auto logits_dims = logits->dims(); + PADDLE_ENFORCE_EQ(logits_dims[0], + static_cast(logits_lod[level].back()), + "The first dimension of Input(Logits) should be equal to " + "the sum of all sequences' lengths."); + + auto label_lod = framework::ToAbsOffset(label->lod()); + auto label_dims = label->dims(); + PADDLE_ENFORCE_EQ( + label_dims[0], label->numel(), + "The width of each timestep in Input(Label) should be 1."); + + const size_t num_sequences = logits_lod[level].size() - 1; + PADDLE_ENFORCE_EQ(num_sequences, label_lod[level].size() - 1, + "The number of sequences of Input(Logits) should be " + "equal to that of Input(Label)."); + PADDLE_ENFORCE_LE(num_sequences, 256, + "The labelLengths must less than 256 for cudnn call."); + + const size_t sequence_width = logits->numel() / logits_dims[0]; + auto loss_dims = + framework::make_ddim({static_cast(num_sequences), 1}); + + // NOTE: cudnn takes softmax input, calculate softmax first, then do padding + auto& dev_ctx = ctx.template device_context(); + LoDTensor softmax_logits; + softmax_logits.mutable_data(logits->dims(), ctx.GetPlace()); + softmax_logits.set_lod(logits_lod); + int rank = logits->dims().size(); + Tensor in_2d = framework::ReshapeToMatrix(*logits, rank - 1); + Tensor out_2d = framework::ReshapeToMatrix(softmax_logits, rank - 1); + math::SoftmaxFunctor()(dev_ctx, &in_2d, &out_2d); + + // ctc needs sequences data stored in transposed padding format + // logits and grad using padding data of layout 'TNC' + // T: max_sequence_length + // N: batch_size (num_sequences) + // C: width + LoDTensor warpctc_logits; + const size_t max_sequence_length = + math::MaximumSequenceLength(logits_lod[level]); + auto warpctc_logits_dims = + framework::make_ddim({static_cast(max_sequence_length), + static_cast(num_sequences), + static_cast(sequence_width)}); + warpctc_logits.mutable_data(warpctc_logits_dims, ctx.GetPlace()); + + LoDTensor cpu_pad_value; + T* pad_value_data = + cpu_pad_value.mutable_data({1}, platform::CPUPlace()); + *pad_value_data = static_cast(0); + LoDTensor pad_value; + if (platform::is_cpu_place(ctx.GetPlace())) { + pad_value = cpu_pad_value; + } else { + TensorCopySync(cpu_pad_value, ctx.GetPlace(), &pad_value); + } + + math::PaddingLoDTensorFunctor()( + ctx.template device_context(), softmax_logits, + &warpctc_logits, pad_value, -1, 0, false /* norm_by_times */, + math::kLengthBatchWidth); + const T* warpctc_logits_data = warpctc_logits.data(); + + std::vector warpctc_label_lengths(num_sequences); + std::vector warpctc_logits_lengths(num_sequences); + + for (size_t i = 0; i < num_sequences; ++i) { + warpctc_label_lengths[i] = label_lod[level][i + 1] - label_lod[level][i]; + warpctc_logits_lengths[i] = + logits_lod[level][i + 1] - logits_lod[level][i]; + } + + T* warpctc_grad_data = + warpctc_grad->mutable_data(warpctc_logits.dims(), ctx.GetPlace()); + + math::SetConstant()( + ctx.template device_context(), warpctc_grad, + static_cast(0)); + + Tensor warpctc_label; + TensorCopySync(*label, platform::CPUPlace(), &warpctc_label); + const int* warpctc_label_data = warpctc_label.data(); + // ======================================================================== + + ScopedTensorDescriptor logits_desc; + ScopedTensorDescriptor grad_desc; + ScopedCTCLossDescriptor ctcloss_desc; + // layout here doesn't have effect. + DataLayout layout = DataLayout::kNCHW; + + auto cu_logits_desc = logits_desc.descriptor( + layout, framework::vectorize2int(warpctc_logits.dims())); + auto cu_grad_desc = grad_desc.descriptor( + layout, framework::vectorize2int(warpctc_grad->dims())); + auto cu_ctcloss_desc = ctcloss_desc.descriptor(); + + auto handle = dev_ctx.cudnn_handle(); + size_t workspace_size; + + CUDNN_ENFORCE(platform::dynload::cudnnGetCTCLossWorkspaceSize( + handle, cu_logits_desc, cu_grad_desc, warpctc_label_data, + warpctc_label_lengths.data(), warpctc_logits_lengths.data(), + CUDNN_CTC_LOSS_ALGO_DETERMINISTIC, cu_ctcloss_desc, &workspace_size)); + + T* loss_data = loss->mutable_data(loss_dims, ctx.GetPlace()); + + auto workspace_handle = dev_ctx.cudnn_workspace_handle(); + auto cudnn_func = [&](void* cudnn_workspace) { + CUDNN_ENFORCE(platform::dynload::cudnnCTCLoss( + handle, cu_logits_desc, warpctc_logits_data, warpctc_label_data, + warpctc_label_lengths.data(), warpctc_logits_lengths.data(), + loss_data, cu_grad_desc, warpctc_grad_data, + CUDNN_CTC_LOSS_ALGO_DETERMINISTIC, cu_ctcloss_desc, cudnn_workspace, + workspace_size)); + }; + workspace_handle.RunFunc(cudnn_func, workspace_size); + } +}; + +template +class CudnnCTCGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* warpctc_grad = ctx.Input("WarpCTCGrad"); + auto* logits_grad = ctx.Output(framework::GradVarName("Logits")); + const Tensor* loss_grad = ctx.Input(framework::GradVarName("Loss")); + + logits_grad->mutable_data(ctx.GetPlace()); + bool norm_by_times = ctx.Attr("norm_by_times"); + math::UnpaddingLoDTensorFunctor()( + ctx.template device_context(), *warpctc_grad, + logits_grad, -1, 0, norm_by_times, math::kLengthBatchWidth); + + const T* loss_grad_data = loss_grad->data(); + math::ScaleLoDTensorFunctor()( + ctx.template device_context(), loss_grad_data, + logits_grad); + } +}; + +#endif +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +namespace plat = paddle::platform; +#if CUDNN_VERSION >= 7001 +REGISTER_OP_KERNEL( + warpctc, CUDNN, plat::CUDAPlace, + ops::CudnnCTCKernel); +REGISTER_OP_KERNEL( + warpctc_grad, CUDNN, plat::CUDAPlace, + ops::CudnnCTCGradKernel); +#endif diff --git a/paddle/fluid/operators/warpctc_op.cc b/paddle/fluid/operators/warpctc_op.cc index e06c8c962f..6a257cebf5 100644 --- a/paddle/fluid/operators/warpctc_op.cc +++ b/paddle/fluid/operators/warpctc_op.cc @@ -14,6 +14,10 @@ limitations under the License. */ #include "paddle/fluid/operators/warpctc_op.h" +#ifdef PADDLE_WITH_CUDA +#include "paddle/fluid/platform/cudnn_helper.h" +#endif + namespace paddle { namespace operators { @@ -45,9 +49,16 @@ class WarpCTCOp : public framework::OperatorWithKernel { protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { + framework::LibraryType library_{framework::LibraryType::kPlain}; +#ifdef PADDLE_WITH_CUDA + if (platform::CanCUDNNBeUsed(ctx)) { + library_ = framework::LibraryType::kCUDNN; + } +#endif + framework::DataLayout layout_ = framework::DataLayout::kAnyLayout; return framework::OpKernelType( framework::ToDataType(ctx.Input("Logits")->type()), - ctx.device_context()); + ctx.device_context(), layout_, library_); } }; @@ -86,6 +97,10 @@ class WarpCTCOpMaker : public framework::OpProtoAndCheckerMaker { "normalize the gradients by the number of time-step, " "which is also the sequence's length.") .SetDefault(false); + AddAttr("use_cudnn", + "(bool, default: false), whether to " + "use cudnn kernel.") + .SetDefault(false); AddComment(R"DOC( An operator integrating the open-source [warp-ctc](https://github.com/baidu-research/warp-ctc) library, which is used in diff --git a/paddle/fluid/platform/cudnn_helper.h b/paddle/fluid/platform/cudnn_helper.h index 07bb02be19..f174a7bc48 100644 --- a/paddle/fluid/platform/cudnn_helper.h +++ b/paddle/fluid/platform/cudnn_helper.h @@ -380,5 +380,28 @@ inline bool CanCUDNNBeUsed(const framework::ExecutionContext& ctx) { return use_cudnn; } +#if CUDNN_VERSION >= 7001 +class ScopedCTCLossDescriptor { + public: + ScopedCTCLossDescriptor() { + PADDLE_ENFORCE(dynload::cudnnCreateCTCLossDescriptor(&desc_)); + } + ~ScopedCTCLossDescriptor() { + PADDLE_ENFORCE(dynload::cudnnDestroyCTCLossDescriptor(desc_)); + } + + template + inline cudnnCTCLossDescriptor_t descriptor() { + PADDLE_ENFORCE( + dynload::cudnnSetCTCLossDescriptor(desc_, CudnnDataType::type)); + return desc_; + } + + private: + cudnnCTCLossDescriptor_t desc_; + DISABLE_COPY_AND_ASSIGN(ScopedCTCLossDescriptor); +}; +#endif + } // namespace platform } // namespace paddle diff --git a/paddle/fluid/platform/dynload/cudnn.h b/paddle/fluid/platform/dynload/cudnn.h index c26143d2f2..db2e28bc91 100644 --- a/paddle/fluid/platform/dynload/cudnn.h +++ b/paddle/fluid/platform/dynload/cudnn.h @@ -154,7 +154,13 @@ CUDNN_DNN_ROUTINE_EACH_R5(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP) #if CUDNN_VERSION >= 7001 #define CUDNN_DNN_ROUTINE_EACH_R7(__macro) \ __macro(cudnnSetConvolutionGroupCount); \ - __macro(cudnnSetConvolutionMathType); + __macro(cudnnSetConvolutionMathType); \ + __macro(cudnnCreateCTCLossDescriptor); \ + __macro(cudnnDestroyCTCLossDescriptor); \ + __macro(cudnnGetCTCLossDescriptor); \ + __macro(cudnnSetCTCLossDescriptor); \ + __macro(cudnnGetCTCLossWorkspaceSize); \ + __macro(cudnnCTCLoss); CUDNN_DNN_ROUTINE_EACH_R7(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP) #endif diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index f60f373163..002d0f006b 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -4187,7 +4187,7 @@ def ctc_greedy_decoder(input, blank, name=None): return ctc_out -def warpctc(input, label, blank=0, norm_by_times=False): +def warpctc(input, label, blank=0, norm_by_times=False, use_cudnn=False): """ An operator integrating the open source Warp-CTC library (https://github.com/baidu-research/warp-ctc) @@ -4212,6 +4212,7 @@ def warpctc(input, label, blank=0, norm_by_times=False): by the number of time-step, which is also the sequence's length. There is no need to normalize the gradients if warpctc layer was follewed by a mean_op. + use_cudnn (bool, default false): Whether to use cudnn. Returns: Variable: The Connectionist Temporal Classification (CTC) loss, @@ -4235,8 +4236,11 @@ def warpctc(input, label, blank=0, norm_by_times=False): 'Label': [label]}, outputs={'WarpCTCGrad': [grad_out], 'Loss': [loss_out]}, - attrs={'blank': blank, - 'norm_by_times': norm_by_times}) + attrs={ + 'blank': blank, + 'norm_by_times': norm_by_times, + 'use_cudnn': use_cudnn + }) return loss_out diff --git a/python/paddle/fluid/tests/unittests/test_warpctc_op.py b/python/paddle/fluid/tests/unittests/test_warpctc_op.py index 5e3aa13546..ec0592baa2 100644 --- a/python/paddle/fluid/tests/unittests/test_warpctc_op.py +++ b/python/paddle/fluid/tests/unittests/test_warpctc_op.py @@ -183,6 +183,7 @@ class TestWarpCTCOp(OpTest): self.labels_lod = [[3, 1, 4, 4]] self.blank = self.num_classes - 1 self.norm_by_times = False + self.use_cudnn = False def setUp(self): self.op_type = "warpctc" @@ -215,7 +216,11 @@ class TestWarpCTCOp(OpTest): "Label": (labels, self.labels_lod) } self.outputs = {"Loss": loss} - self.attrs = {"blank": self.blank, "norm_by_times": self.norm_by_times} + self.attrs = { + "blank": self.blank, + "norm_by_times": self.norm_by_times, + "use_cudnn": self.use_cudnn + } def test_check_output(self): self.check_output() @@ -233,6 +238,22 @@ class TestWarpCTCOpCase1(TestWarpCTCOp): self.labels_lod = [[3, 1, 4, 4]] self.blank = 0 self.norm_by_times = False + self.use_cudnn = False + + +class TestCudnnCTCOp(TestWarpCTCOp): + def config(self): + self.batch_size = 4 + self.num_classes = 8 + self.logits_lod = [[4, 1, 3, 3]] + self.labels_lod = [[3, 1, 4, 4]] + self.blank = 0 + self.norm_by_times = False + self.use_cudnn = True + + def test_check_grad(self): + self.outputs['WarpCTCGrad'] = self.gradient + self.check_grad(["Logits"], "Loss", max_relative_error=0.01) if __name__ == "__main__": -- GitLab