/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. Indicesou may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #include "paddle/framework/op_registry.h" #include "paddle/framework/tensor.h" #include "paddle/operators/math/detection_util.h" #include "paddle/operators/math/math_function.h" #include "paddle/operators/math/softmax.h" namespace paddle { namespace operators { template class Detection_output_Kernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { const framework::Tensor* in_loc = context.Input("Loc"); const framework::Tensor* in_conf = context.Input("Conf"); const framework::Tensor* in_priorbox = context.Input("PriorBox"); auto* out = context.Output("Out"); int num_classes = context.template Attr("num_classes"); int top_k = context.template Attr("top_k"); int nms_top_k = context.template Attr("nms_top_k"); int background_label_id = context.template Attr("background_label_id"); float nms_threshold = context.template Attr("nms_threshold"); float confidence_threshold = context.template Attr("confidence_threshold"); int input_num = in_loc->dims()[0]; int batch_size = in_loc->dims()[1]; int channels = in_loc->dims()[2]; int height = in_loc->dims()[3]; int weight = in_loc->dims()[4]; int loc_sum_size = in_loc->numel(); int conf_sum_size = in_conf->numel(); std::vector loc_shape_vec({1, loc_sum_size}); std::vector conf_shape_vec( {conf_sum_size / num_classes, num_classes}); framework::DDim loc_shape(framework::make_ddim(loc_shape_vec)); framework::DDim conf_shape(framework::make_ddim(conf_shape_vec)); framework::Tensor loc_tensor; framework::Tensor conf_tensor; loc_tensor.Resize(loc_shape); conf_tensor.Resize(conf_shape); loc_tensor.mutable_data(loc_shape, context.GetPlace()); conf_tensor.mutable_data(conf_shape, context.GetPlace()); framework::Tensor loc_cpu; framework::Tensor conf_cpu; framework::Tensor priorbox_cpu; const T* in_loc_data = in_loc->data(); const T* in_conf_data = in_conf->data(); T* loc_data; T* conf_data; const T* priorbox_data = in_priorbox->data(); if (platform::is_gpu_place(context.GetPlace())) { loc_cpu.mutable_data(in_loc->dims(), platform::CPUPlace()); framework::CopyFrom(*in_loc, platform::CPUPlace(), context.device_context(), &loc_cpu); in_loc_data = loc_cpu.data(); conf_cpu.mutable_data(in_conf->dims(), platform::CPUPlace()); framework::CopyFrom(*in_conf, platform::CPUPlace(), context.device_context(), &conf_cpu); in_conf_data = conf_cpu.data(); priorbox_cpu.mutable_data(in_priorbox->dims(), platform::CPUPlace()); framework::CopyFrom(*in_priorbox, platform::CPUPlace(), context.device_context(), &priorbox_cpu); priorbox_data = priorbox_cpu.data(); loc_tensor.mutable_data(loc_shape, platform::CPUPlace()); conf_tensor.mutable_data(conf_shape, platform::CPUPlace()); } T* loc_tensor_data = loc_tensor.data(); T* conf_tensor_data = conf_tensor.data(); for (int i = 0; i < input_num; ++i) { math::appendWithPermute(in_loc_data, input_num, batch_size, channels, height, weight, loc_tensor_data); math::appendWithPermute(in_conf_data, input_num, batch_size, channels, height, weight, conf_tensor_data); } loc_data = loc_tensor.data(); if (platform::is_gpu_place(context.GetPlace())) { framework::Tensor conf_gpu; conf_gpu.Resize(conf_shape); conf_gpu.mutable_data(conf_shape, context.GetPlace()); framework::CopyFrom(conf_tensor, platform::GPUPlace(), context.device_context(), &conf_gpu); // softmax math::SoftmaxFunctor()(context.device_context(), &conf_gpu, &conf_gpu); conf_tensor.mutable_data(conf_gpu.dims(), platform::CPUPlace()); framework::CopyFrom(conf_gpu, platform::CPUPlace(), context.device_context(), &conf_tensor); } else { // softmax math::SoftmaxFunctor()(context.device_context(), &conf_tensor, &conf_tensor); } conf_data = conf_tensor.data(); // get decode bboxes size_t num_priors = in_priorbox->numel() / 8; std::vector>> all_decoded_bboxes; for (size_t n = 0; n < batch_size; ++n) { std::vector> decoded_bboxes; for (size_t i = 0; i < num_priors; ++i) { size_t prior_offset = i * 8; size_t loc_pred_offset = n * num_priors * 4 + i * 4; std::vector> prior_bbox_vec; math::getBBoxFromPriorData(priorbox_data + prior_offset, 1, prior_bbox_vec); std::vector> prior_bbox_var; math::getBBoxVarFromPriorData(priorbox_data + prior_offset, 1, prior_bbox_var); std::vector loc_pred_data; for (size_t j = 0; j < 4; ++j) loc_pred_data.push_back(*(loc_data + loc_pred_offset + j)); math::BBox bbox = math::decodeBBoxWithVar( prior_bbox_vec[0], prior_bbox_var[0], loc_pred_data); decoded_bboxes.push_back(bbox); } all_decoded_bboxes.push_back(decoded_bboxes); } std::vector>> all_indices; int num_kept = math::getDetectionIndices( conf_data, num_priors, num_classes, background_label_id, batch_size, confidence_threshold, nms_top_k, nms_threshold, top_k, all_decoded_bboxes, &all_indices); if (num_kept <= 0) { std::vector out_shape_vec({0, 0}); framework::DDim out_shape(framework::make_ddim(out_shape_vec)); out->Resize(out_shape); return; } std::vector out_shape_vec({num_kept, 7}); framework::DDim out_shape(framework::make_ddim(out_shape_vec)); out->mutable_data(out_shape, context.GetPlace()); framework::Tensor out_cpu; T* out_data = out->data(); if (platform::is_gpu_place(context.GetPlace())) { out_cpu.mutable_data(out->dims(), platform::CPUPlace()); out_data = out_cpu.data(); } math::getDetectionOutput(conf_data, num_kept, num_priors, num_classes, batch_size, all_indices, all_decoded_bboxes, out_data); if (platform::is_gpu_place(context.GetPlace())) { framework::CopyFrom(out_cpu, platform::GPUPlace(), context.device_context(), out); } } }; } // namespace operators } // namespace paddle