yolo_box_op.cc 11.1 KB
Newer Older
D
dengkaipeng 已提交
1
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
D
dengkaipeng 已提交
2 3 4 5 6 7 8 9 10 11 12 13
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
   http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License. */

#include "paddle/fluid/operators/detection/yolo_box_op.h"
#include "paddle/fluid/framework/op_registry.h"
14
#include "paddle/fluid/framework/op_version_registry.h"
D
dengkaipeng 已提交
15 16 17 18 19 20 21 22 23 24

namespace paddle {
namespace operators {

using framework::Tensor;

class YoloBoxOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;
  void InferShape(framework::InferShapeContext* ctx) const override {
X
xiaoting 已提交
25 26 27 28
    OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "YoloBoxOp");
    OP_INOUT_CHECK(ctx->HasInput("ImgSize"), "Input", "ImgSize", "YoloBoxOp");
    OP_INOUT_CHECK(ctx->HasOutput("Boxes"), "Output", "Boxes", "YoloBoxOp");
    OP_INOUT_CHECK(ctx->HasOutput("Scores"), "Output", "Scores", "YoloBoxOp");
D
dengkaipeng 已提交
29 30

    auto dim_x = ctx->GetInputDim("X");
31
    auto dim_imgsize = ctx->GetInputDim("ImgSize");
D
dengkaipeng 已提交
32 33 34
    auto anchors = ctx->Attrs().Get<std::vector<int>>("anchors");
    int anchor_num = anchors.size() / 2;
    auto class_num = ctx->Attrs().Get<int>("class_num");
35 36
    auto iou_aware = ctx->Attrs().Get<bool>("iou_aware");
    auto iou_aware_factor = ctx->Attrs().Get<float>("iou_aware_factor");
D
dengkaipeng 已提交
37

X
xiaoting 已提交
38 39 40 41
    PADDLE_ENFORCE_EQ(dim_x.size(), 4, platform::errors::InvalidArgument(
                                           "Input(X) should be a 4-D tensor."
                                           "But received X dimension(%s)",
                                           dim_x.size()));
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
    if (iou_aware) {
      PADDLE_ENFORCE_EQ(
          dim_x[1], anchor_num * (6 + class_num),
          platform::errors::InvalidArgument(
              "Input(X) dim[1] should be equal to (anchor_mask_number * (6 "
              "+ class_num)) while iou_aware is true."
              "But received dim[1](%s) != (anchor_mask_number * "
              "(6+class_num)(%s).",
              dim_x[1], anchor_num * (6 + class_num)));
      PADDLE_ENFORCE_GE(
          iou_aware_factor, 0,
          platform::errors::InvalidArgument(
              "Attr(iou_aware_factor) should greater than or equal to 0."
              "But received iou_aware_factor (%s)",
              iou_aware_factor));
      PADDLE_ENFORCE_LE(
          iou_aware_factor, 1,
          platform::errors::InvalidArgument(
              "Attr(iou_aware_factor) should less than or equal to 1."
              "But received iou_aware_factor (%s)",
              iou_aware_factor));
    } else {
      PADDLE_ENFORCE_EQ(
          dim_x[1], anchor_num * (5 + class_num),
          platform::errors::InvalidArgument(
              "Input(X) dim[1] should be equal to (anchor_mask_number * (5 "
              "+ class_num))."
              "But received dim[1](%s) != (anchor_mask_number * "
              "(5+class_num)(%s).",
              dim_x[1], anchor_num * (5 + class_num)));
    }
73
    PADDLE_ENFORCE_EQ(dim_imgsize.size(), 2,
X
xiaoting 已提交
74 75 76 77
                      platform::errors::InvalidArgument(
                          "Input(ImgSize) should be a 2-D tensor."
                          "But received Imgsize size(%s)",
                          dim_imgsize.size()));
78 79 80 81 82 83
    if ((dim_imgsize[0] > 0 && dim_x[0] > 0) || ctx->IsRuntime()) {
      PADDLE_ENFORCE_EQ(
          dim_imgsize[0], dim_x[0],
          platform::errors::InvalidArgument(
              "Input(ImgSize) dim[0] and Input(X) dim[0] should be same."));
    }
X
xiaoting 已提交
84 85 86 87 88
    PADDLE_ENFORCE_EQ(
        dim_imgsize[1], 2,
        platform::errors::InvalidArgument("Input(ImgSize) dim[1] should be 2."
                                          "But received imgsize dim[1](%s).",
                                          dim_imgsize[1]));
D
dengkaipeng 已提交
89
    PADDLE_ENFORCE_GT(anchors.size(), 0,
X
xiaoting 已提交
90 91 92 93
                      platform::errors::InvalidArgument(
                          "Attr(anchors) length should be greater than 0."
                          "But received anchors length(%s).",
                          anchors.size()));
D
dengkaipeng 已提交
94
    PADDLE_ENFORCE_EQ(anchors.size() % 2, 0,
X
xiaoting 已提交
95 96 97 98
                      platform::errors::InvalidArgument(
                          "Attr(anchors) length should be even integer."
                          "But received anchors length (%s)",
                          anchors.size()));
D
dengkaipeng 已提交
99
    PADDLE_ENFORCE_GT(class_num, 0,
X
xiaoting 已提交
100 101 102 103
                      platform::errors::InvalidArgument(
                          "Attr(class_num) should be an integer greater than 0."
                          "But received class_num (%s)",
                          class_num));
D
dengkaipeng 已提交
104

105 106 107 108 109 110
    int box_num;
    if ((dim_x[2] > 0 && dim_x[3] > 0) || ctx->IsRuntime()) {
      box_num = dim_x[2] * dim_x[3] * anchor_num;
    } else {
      box_num = -1;
    }
D
dengkaipeng 已提交
111
    std::vector<int64_t> dim_boxes({dim_x[0], box_num, 4});
112
    ctx->SetOutputDim("Boxes", phi::make_ddim(dim_boxes));
D
dengkaipeng 已提交
113 114

    std::vector<int64_t> dim_scores({dim_x[0], box_num, class_num});
115
    ctx->SetOutputDim("Scores", phi::make_ddim(dim_scores));
D
dengkaipeng 已提交
116 117 118 119 120
  }

 protected:
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
121 122
    return framework::OpKernelType(
        OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace());
D
dengkaipeng 已提交
123 124 125 126 127 128 129
  }
};

class YoloBoxOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  void Make() override {
    AddInput("X",
D
dengkaipeng 已提交
130 131
             "The input tensor of YoloBox operator is a 4-D tensor with "
             "shape of [N, C, H, W]. The second dimension(C) stores "
D
dengkaipeng 已提交
132 133
             "box locations, confidence score and classification one-hot "
             "keys of each anchor box. Generally, X should be the output "
D
dengkaipeng 已提交
134
             "of YOLOv3 network.");
135 136
    AddInput("ImgSize",
             "The image size tensor of YoloBox operator, "
D
dengkaipeng 已提交
137
             "This is a 2-D tensor with shape of [N, 2]. This tensor holds "
D
dengkaipeng 已提交
138
             "height and width of each input image used for resizing output "
139
             "box in input image scale.");
D
dengkaipeng 已提交
140 141
    AddOutput("Boxes",
              "The output tensor of detection boxes of YoloBox operator, "
D
dengkaipeng 已提交
142 143
              "This is a 3-D tensor with shape of [N, M, 4], N is the "
              "batch num, M is output box number, and the 3rd dimension "
D
dengkaipeng 已提交
144 145
              "stores [xmin, ymin, xmax, ymax] coordinates of boxes.");
    AddOutput("Scores",
D
dengkaipeng 已提交
146 147 148 149
              "The output tensor of detection boxes scores of YoloBox "
              "operator, This is a 3-D tensor with shape of "
              "[N, M, :attr:`class_num`], N is the batch num, M is "
              "output box number.");
D
dengkaipeng 已提交
150 151 152 153 154 155 156 157 158 159 160 161

    AddAttr<int>("class_num", "The number of classes to predict.");
    AddAttr<std::vector<int>>("anchors",
                              "The anchor width and height, "
                              "it will be parsed pair by pair.")
        .SetDefault(std::vector<int>{});
    AddAttr<int>("downsample_ratio",
                 "The downsample ratio from network input to YoloBox operator "
                 "input, so 32, 16, 8 should be set for the first, second, "
                 "and thrid YoloBox operators.")
        .SetDefault(32);
    AddAttr<float>("conf_thresh",
D
dengkaipeng 已提交
162 163
                   "The confidence scores threshold of detection boxes. "
                   "Boxes with confidence scores under threshold should "
D
dengkaipeng 已提交
164 165
                   "be ignored.")
        .SetDefault(0.01);
166 167 168 169
    AddAttr<bool>("clip_bbox",
                  "Whether clip output bonding box in Input(ImgSize) "
                  "boundary. Default true.")
        .SetDefault(true);
170 171 172 173
    AddAttr<float>("scale_x_y",
                   "Scale the center point of decoded bounding "
                   "box. Default 1.0")
        .SetDefault(1.);
174 175 176 177
    AddAttr<bool>("iou_aware", "Whether use iou aware. Default false.")
        .SetDefault(false);
    AddAttr<float>("iou_aware_factor", "iou aware factor. Default 0.5.")
        .SetDefault(0.5);
D
dengkaipeng 已提交
178
    AddComment(R"DOC(
D
dengkaipeng 已提交
179
         This operator generates YOLO detection boxes from output of YOLOv3 network.
D
dengkaipeng 已提交
180 181
         
         The output of previous network is in shape [N, C, H, W], while H and W
D
dengkaipeng 已提交
182 183
         should be the same, H and W specify the grid size, each grid point predict 
         given number boxes, this given number, which following will be represented as S,
D
dengkaipeng 已提交
184
         is specified by the number of anchors. In the second dimension(the channel
185 186
         dimension), C should be equal to S * (5 + class_num) if :attr:`iou_aware` is false,
         otherwise C should be equal to S * (6 + class_num). class_num is the object
D
dengkaipeng 已提交
187
         category number of source dataset(such as 80 in coco dataset), so the 
D
dengkaipeng 已提交
188 189 190 191 192 193
         second(channel) dimension, apart from 4 box location coordinates x, y, w, h, 
         also includes confidence score of the box and class one-hot key of each anchor 
         box.

         Assume the 4 location coordinates are :math:`t_x, t_y, t_w, t_h`, the box 
         predictions should be as follows:
D
dengkaipeng 已提交
194 195

         $$
D
dengkaipeng 已提交
196
         b_x = \\sigma(t_x) + c_x
D
dengkaipeng 已提交
197 198
         $$
         $$
D
dengkaipeng 已提交
199
         b_y = \\sigma(t_y) + c_y
D
dengkaipeng 已提交
200 201
         $$
         $$
D
dengkaipeng 已提交
202
         b_w = p_w e^{t_w}
D
dengkaipeng 已提交
203 204
         $$
         $$
D
dengkaipeng 已提交
205 206 207
         b_h = p_h e^{t_h}
         $$

D
dengkaipeng 已提交
208 209
         in the equation above, :math:`c_x, c_y` is the left top corner of current grid
         and :math:`p_w, p_h` is specified by anchors.
D
dengkaipeng 已提交
210

D
dengkaipeng 已提交
211 212
         The logistic regression value of the 5th channel of each anchor prediction boxes
         represents the confidence score of each prediction box, and the logistic
D
dengkaipeng 已提交
213
         regression value of the last :attr:`class_num` channels of each anchor prediction 
D
dengkaipeng 已提交
214
         boxes represents the classifcation scores. Boxes with confidence scores less than
D
dengkaipeng 已提交
215
         :attr:`conf_thresh` should be ignored, and box final scores is the product of 
D
dengkaipeng 已提交
216
         confidence scores and classification scores.
D
dengkaipeng 已提交
217

D
dengkaipeng 已提交
218 219 220 221
         $$
         score_{pred} = score_{conf} * score_{class}
         $$

222 223 224 225 226 227 228 229 230
         where the confidence scores follow the formula bellow

         .. math::

            score_{conf} = \begin{case}
                             obj, \text{if } iou_aware == flase \\
                             obj^{1 - iou_aware_factor} * iou^{iou_aware_factor}, \text{otherwise}
                           \end{case}

D
dengkaipeng 已提交
231 232 233 234 235 236 237 238
         )DOC");
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
H
hong 已提交
239 240 241 242
REGISTER_OPERATOR(
    yolo_box, ops::YoloBoxOp, ops::YoloBoxOpMaker,
    paddle::framework::EmptyGradOpMaker<paddle::framework::OpDesc>,
    paddle::framework::EmptyGradOpMaker<paddle::imperative::OpBase>);
D
dengkaipeng 已提交
243 244
REGISTER_OP_CPU_KERNEL(yolo_box, ops::YoloBoxKernel<float>,
                       ops::YoloBoxKernel<double>);
245 246 247 248 249 250 251 252 253

REGISTER_OP_VERSION(yolo_box)
    .AddCheckpoint(
        R"ROC(
      Upgrade yolo box to add new attribute [iou_aware, iou_aware_factor].
    )ROC",
        paddle::framework::compatible::OpVersionDesc()
            .NewAttr("iou_aware", "Whether use iou aware", false)
            .NewAttr("iou_aware_factor", "iou aware factor", 0.5f));