/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/detection/yolov3_loss_op.h" #include #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/imperative/type_defs.h" namespace paddle { namespace operators { using framework::Tensor; class Yolov3LossOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; void InferShape(framework::InferShapeContext* ctx) const override { OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "Yolov3LossOp"); OP_INOUT_CHECK(ctx->HasInput("GTBox"), "Input", "GTBox", "Yolov3LossOp"); OP_INOUT_CHECK(ctx->HasInput("GTLabel"), "Input", "GTLabel", "Yolov3LossOp"); OP_INOUT_CHECK(ctx->HasOutput("Loss"), "Output", "Loss", "Yolov3LossOp"); OP_INOUT_CHECK(ctx->HasOutput("ObjectnessMask"), "Output", "ObjectnessMask", "Yolov3LossOp"); OP_INOUT_CHECK(ctx->HasOutput("GTMatchMask"), "Output", "GTMatchMask", "Yolov3LossOp"); auto dim_x = ctx->GetInputDim("X"); auto dim_gtbox = ctx->GetInputDim("GTBox"); auto dim_gtlabel = ctx->GetInputDim("GTLabel"); auto anchors = ctx->Attrs().Get>("anchors"); int anchor_num = anchors.size() / 2; auto anchor_mask = ctx->Attrs().Get>("anchor_mask"); int mask_num = anchor_mask.size(); auto class_num = ctx->Attrs().Get("class_num"); PADDLE_ENFORCE_EQ(dim_x.size(), 4, platform::errors::InvalidArgument( "Input(X) should be a 4-D tensor. But received " "X dimension size(%s)", dim_x.size())); PADDLE_ENFORCE_EQ(dim_x[2], dim_x[3], platform::errors::InvalidArgument( "Input(X) dim[3] and dim[4] should be euqal." "But received dim[3](%s) != dim[4](%s)", dim_x[2], dim_x[3])); PADDLE_ENFORCE_EQ( dim_x[1], mask_num * (5 + class_num), platform::errors::InvalidArgument( "Input(X) dim[1] should be equal to (anchor_mask_number * (5 " "+ class_num))." "But received dim[1](%s) != (anchor_mask_number * " "(5+class_num)(%s).", dim_x[1], mask_num * (5 + class_num))); PADDLE_ENFORCE_EQ(dim_gtbox.size(), 3, platform::errors::InvalidArgument( "Input(GTBox) should be a 3-D tensor, but " "received gtbox dimension size(%s)", dim_gtbox.size())); PADDLE_ENFORCE_EQ(dim_gtbox[2], 4, platform::errors::InvalidArgument( "Input(GTBox) dim[2] should be 4", "But receive dim[2](%s) != 5. ", dim_gtbox[2])); PADDLE_ENFORCE_EQ( dim_gtlabel.size(), 2, platform::errors::InvalidArgument( "Input(GTLabel) should be a 2-D tensor," "But received Input(GTLabel) dimension size(%s) != 2.", dim_gtlabel.size())); PADDLE_ENFORCE_EQ( dim_gtlabel[0], dim_gtbox[0], platform::errors::InvalidArgument( "Input(GTBox) dim[0] and Input(GTLabel) dim[0] should be same," "But received Input(GTLabel) dim[0](%s) != " "Input(GTBox) dim[0](%s)", dim_gtlabel[0], dim_gtbox[0])); PADDLE_ENFORCE_EQ( dim_gtlabel[1], dim_gtbox[1], platform::errors::InvalidArgument( "Input(GTBox) and Input(GTLabel) dim[1] should be same," "But received Input(GTBox) dim[1](%s) != Input(GTLabel) " "dim[1](%s)", dim_gtbox[1], dim_gtlabel[1])); PADDLE_ENFORCE_GT(anchors.size(), 0, platform::errors::InvalidArgument( "Attr(anchors) length should be greater then 0." "But received anchors length(%s)", anchors.size())); PADDLE_ENFORCE_EQ(anchors.size() % 2, 0, platform::errors::InvalidArgument( "Attr(anchors) length should be even integer." "But received anchors length(%s)", anchors.size())); for (size_t i = 0; i < anchor_mask.size(); i++) { PADDLE_ENFORCE_LT( anchor_mask[i], anchor_num, platform::errors::InvalidArgument( "Attr(anchor_mask) should not crossover Attr(anchors)." "But received anchor_mask[i](%s) > anchor_num(%s)", anchor_mask[i], anchor_num)); } PADDLE_ENFORCE_GT(class_num, 0, platform::errors::InvalidArgument( "Attr(class_num) should be an integer greater then 0." "But received class_num(%s) < 0", class_num)); if (ctx->HasInput("GTScore")) { auto dim_gtscore = ctx->GetInputDim("GTScore"); PADDLE_ENFORCE_EQ(dim_gtscore.size(), 2, platform::errors::InvalidArgument( "Input(GTScore) should be a 2-D tensor" "But received GTScore dimension(%s)", dim_gtbox.size())); PADDLE_ENFORCE_EQ( dim_gtscore[0], dim_gtbox[0], platform::errors::InvalidArgument( "Input(GTBox) and Input(GTScore) dim[0] should be same" "But received GTBox dim[0](%s) != GTScore dim[0](%s)", dim_gtbox[0], dim_gtscore[0])); PADDLE_ENFORCE_EQ( dim_gtscore[1], dim_gtbox[1], platform::errors::InvalidArgument( "Input(GTBox) and Input(GTScore) dim[1] should be same" "But received GTBox dim[1](%s) != GTScore dim[1](%s)", dim_gtscore[1], dim_gtbox[1])); } std::vector dim_out({dim_x[0]}); ctx->SetOutputDim("Loss", framework::make_ddim(dim_out)); std::vector dim_obj_mask({dim_x[0], mask_num, dim_x[2], dim_x[3]}); ctx->SetOutputDim("ObjectnessMask", framework::make_ddim(dim_obj_mask)); std::vector dim_gt_match_mask({dim_gtbox[0], dim_gtbox[1]}); ctx->SetOutputDim("GTMatchMask", framework::make_ddim(dim_gt_match_mask)); } protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { return framework::OpKernelType( OperatorWithKernel::IndicateVarDataType(ctx, "X"), platform::CPUPlace()); } }; class Yolov3LossOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { AddInput("X", "The input tensor of YOLOv3 loss operator, " "This is a 4-D tensor with shape of [N, C, H, W]." "H and W should be same, and the second dimension(C) stores" "box locations, confidence score and classification one-hot" "keys of each anchor box"); AddInput("GTBox", "The input tensor of ground truth boxes, " "This is a 3-D tensor with shape of [N, max_box_num, 5], " "max_box_num is the max number of boxes in each image, " "In the third dimension, stores x, y, w, h coordinates, " "x, y is the center coordinate of boxes and w, h is the " "width and height and x, y, w, h should be divided by " "input image height to scale to [0, 1]."); AddInput("GTLabel", "The input tensor of ground truth label, " "This is a 2-D tensor with shape of [N, max_box_num], " "and each element should be an integer to indicate the " "box class id."); AddInput("GTScore", "The score of GTLabel, This is a 2-D tensor in same shape " "GTLabel, and score values should in range (0, 1). This " "input is for GTLabel score can be not 1.0 in image mixup " "augmentation.") .AsDispensable(); AddOutput("Loss", "The output yolov3 loss tensor, " "This is a 1-D tensor with shape of [N]"); AddOutput("ObjectnessMask", "This is an intermediate tensor with shape of [N, M, H, W], " "M is the number of anchor masks. This parameter caches the " "mask for calculate objectness loss in gradient kernel.") .AsIntermediate(); AddOutput("GTMatchMask", "This is an intermediate tensor with shape of [N, B], " "B is the max box number of GT boxes. This parameter caches " "matched mask index of each GT boxes for gradient calculate.") .AsIntermediate(); AddAttr("class_num", "The number of classes to predict."); AddAttr>("anchors", "The anchor width and height, " "it will be parsed pair by pair.") .SetDefault(std::vector{}); AddAttr>("anchor_mask", "The mask index of anchors used in " "current YOLOv3 loss calculation.") .SetDefault(std::vector{}); AddAttr("downsample_ratio", "The downsample ratio from network input to YOLOv3 loss " "input, so 32, 16, 8 should be set for the first, second, " "and thrid YOLOv3 loss operators.") .SetDefault(32); AddAttr("ignore_thresh", "The ignore threshold to ignore confidence loss.") .SetDefault(0.7); AddAttr("use_label_smooth", "Whether to use label smooth. Default True.") .SetDefault(true); AddAttr("scale_x_y", "Scale the center point of decoded bounding " "box. Default 1.0") .SetDefault(1.); AddComment(R"DOC( This operator generates yolov3 loss based on given predict result and ground truth boxes. The output of previous network is in shape [N, C, H, W], while H and W should be the same, H and W specify the grid size, each grid point predict given number bounding boxes, this given number, which following will be represented as S, is specified by the number of anchor clusters in each scale. In the second dimension(the channel dimension), C should be equal to S * (class_num + 5), class_num is the object category number of source dataset(such as 80 in coco dataset), so in the second(channel) dimension, apart from 4 box location coordinates x, y, w, h, also includes confidence score of the box and class one-hot key of each anchor box. Assume the 4 location coordinates are :math:`t_x, t_y, t_w, t_h`, the box predictions should be as follows: $$ b_x = \\sigma(t_x) + c_x $$ $$ b_y = \\sigma(t_y) + c_y $$ $$ b_w = p_w e^{t_w} $$ $$ b_h = p_h e^{t_h} $$ In the equation above, :math:`c_x, c_y` is the left top corner of current grid and :math:`p_w, p_h` is specified by anchors. As for confidence score, it is the logistic regression value of IoU between anchor boxes and ground truth boxes, the score of the anchor box which has the max IoU should be 1, and if the anchor box has IoU bigger than ignore thresh, the confidence score loss of this anchor box will be ignored. Therefore, the yolov3 loss consists of three major parts: box location loss, objectness loss and classification loss. The L1 loss is used for box coordinates (w, h), sigmoid cross entropy loss is used for box coordinates (x, y), objectness loss and classification loss. Each groud truth box finds a best matching anchor box in all anchors. Prediction of this anchor box will incur all three parts of losses, and prediction of anchor boxes with no GT box matched will only incur objectness loss. In order to trade off box coordinate losses between big boxes and small boxes, box coordinate losses will be mutiplied by scale weight, which is calculated as follows. $$ weight_{box} = 2.0 - t_w * t_h $$ Final loss will be represented as follows. $$ loss = (loss_{xy} + loss_{wh}) * weight_{box} + loss_{conf} + loss_{class} $$ While :attr:`use_label_smooth` is set to be :attr:`True`, the classification target will be smoothed when calculating classification loss, target of positive samples will be smoothed to :math:`1.0 - 1.0 / class\_num` and target of negetive samples will be smoothed to :math:`1.0 / class\_num`. While :attr:`GTScore` is given, which means the mixup score of ground truth boxes, all losses incured by a ground truth box will be multiplied by its mixup score. )DOC"); } }; class Yolov3LossOpGrad : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE_EQ( ctx->HasInput("X"), true, platform::errors::NotFound("Input(X) should not be null")); PADDLE_ENFORCE_EQ( ctx->HasInput(framework::GradVarName("Loss")), true, platform::errors::NotFound("Input(Loss@GRAD) should not be null")); auto dim_x = ctx->GetInputDim("X"); if (ctx->HasOutput(framework::GradVarName("X"))) { ctx->SetOutputDim(framework::GradVarName("X"), dim_x); } } protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { return framework::OpKernelType( OperatorWithKernel::IndicateVarDataType(ctx, "X"), platform::CPUPlace()); } }; template class Yolov3LossGradMaker : public framework::SingleGradOpMaker { public: using framework::SingleGradOpMaker::SingleGradOpMaker; protected: void Apply(GradOpPtr op) const override { op->SetType("yolov3_loss_grad"); op->SetInput("X", this->Input("X")); op->SetInput("GTBox", this->Input("GTBox")); op->SetInput("GTLabel", this->Input("GTLabel")); op->SetInput("GTScore", this->Input("GTScore")); op->SetInput(framework::GradVarName("Loss"), this->OutputGrad("Loss")); op->SetInput("ObjectnessMask", this->Output("ObjectnessMask")); op->SetInput("GTMatchMask", this->Output("GTMatchMask")); op->SetAttrMap(this->Attrs()); op->SetOutput(framework::GradVarName("X"), this->InputGrad("X")); op->SetOutput(framework::GradVarName("GTBox"), this->EmptyInputGrad()); op->SetOutput(framework::GradVarName("GTLabel"), this->EmptyInputGrad()); op->SetOutput(framework::GradVarName("GTScore"), this->EmptyInputGrad()); } }; } // namespace operators } // namespace paddle namespace ops = paddle::operators; REGISTER_OPERATOR(yolov3_loss, ops::Yolov3LossOp, ops::Yolov3LossOpMaker, ops::Yolov3LossGradMaker, ops::Yolov3LossGradMaker); REGISTER_OPERATOR(yolov3_loss_grad, ops::Yolov3LossOpGrad); REGISTER_OP_CPU_KERNEL(yolov3_loss, ops::Yolov3LossKernel, ops::Yolov3LossKernel); REGISTER_OP_CPU_KERNEL(yolov3_loss_grad, ops::Yolov3LossGradKernel, ops::Yolov3LossGradKernel);