diff --git a/paddle/operators/mine_hard_examples_op.cc b/paddle/operators/mine_hard_examples_op.cc index 75098d0bcd8fe2989a823600efc360ecdc055981..603368f93ca370677239e925a3192d132973754b 100644 --- a/paddle/operators/mine_hard_examples_op.cc +++ b/paddle/operators/mine_hard_examples_op.cc @@ -12,41 +12,178 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/mine_hard_examples_op.h" +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" namespace paddle { namespace operators { +enum MiningType { kNone = 0, kMaxNegative, kHardExample }; + +template +bool SortScoreDescend(const std::pair& pair1, + const std::pair& pair2) { + return pair1.first > pair2.first; +} + +inline bool IsEligibleMining(const MiningType mining_type, const int match_idx, + const float match_dist, + const float neg_dist_threshold) { + if (mining_type == MiningType::kMaxNegative) { + return match_idx == -1 && match_dist < neg_dist_threshold; + } else if (mining_type == MiningType::kHardExample) { + return true; + } else { + return false; + } +} + +MiningType GetMiningType(std::string str) { + if (str == "max_negative") { + return MiningType::kMaxNegative; + } else if (str == "hard_example") { + return MiningType::kHardExample; + } else { + return MiningType::kNone; + } +} + +template +class MineHardExamplesKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* in_cls_loss = ctx.Input("ClsLoss"); + auto* in_loc_loss = ctx.Input("LocLoss"); + auto* in_matched_indices = ctx.Input("MatchIndices"); + auto* in_match_dist = ctx.Input("MatchDist"); + float neg_pos_ratio = ctx.Attr("neg_pos_ratio"); + T neg_dist_threshold = + static_cast(ctx.Attr("neg_dist_threshold")); + int sample_size = ctx.Attr("sample_size"); + MiningType mining_type = + GetMiningType(ctx.Attr("mining_type")); + + auto out_neg_indices = ctx.Output("NegIndices"); + auto out_match_indices = + ctx.Output("UpdatedMatchIndices"); + + framework::Copy(*in_matched_indices, ctx.GetPlace(), out_match_indices); + + int batch_size = in_matched_indices->dims()[0]; + int prior_num = in_matched_indices->dims()[1]; + + auto match_indices = framework::EigenMatrix::From(*in_matched_indices); + + auto match_indices_et = + framework::EigenMatrix::From(*out_match_indices); + + auto match_dist = framework::EigenMatrix::From(*in_match_dist); + + const T* cls_loss = in_cls_loss->data(); + const T* loc_loss = nullptr; + if (in_loc_loss) { + loc_loss = in_loc_loss->data(); + } + + std::vector> all_neg_indices; + std::vector batch_starts = {0}; + for (int n = 0; n < batch_size; ++n) { + std::vector> loss_idx; + int neg_sel = 0; + for (int m = 0; m < prior_num; ++m) { + if (IsEligibleMining(mining_type, match_indices(n, m), match_dist(n, m), + neg_dist_threshold)) { + T loss = cls_loss[n * prior_num + m]; + if (mining_type == MiningType::kHardExample && loc_loss != nullptr) { + loss = cls_loss[n * prior_num + m] + loc_loss[n * prior_num + m]; + } + loss_idx.push_back(std::make_pair(loss, m)); + ++neg_sel; + } + } + + if (mining_type == MiningType::kMaxNegative) { + int num_pos = 0; + for (int m = 0; m < prior_num; ++m) { + if (match_indices(n, m) != -1) ++num_pos; + } + neg_sel = std::min(static_cast(num_pos * neg_pos_ratio), neg_sel); + } else if (mining_type == MiningType::kHardExample) { + neg_sel = std::min(sample_size, neg_sel); + } + + std::sort(loss_idx.begin(), loss_idx.end(), SortScoreDescend); + std::set sel_indices; + std::vector neg_indices; + std::transform(loss_idx.begin(), loss_idx.begin() + neg_sel, + std::inserter(sel_indices, sel_indices.begin()), + [](std::pair l) -> int { + return static_cast(l.second); + }); + + for (int m = 0; m < prior_num; ++m) { + if (match_indices(n, m) > -1) { + if (mining_type == MiningType::kHardExample && + sel_indices.find(m) == sel_indices.end()) { + match_indices_et(n, m) = -1; + } + } else { + if (sel_indices.find(m) != sel_indices.end()) { + neg_indices.push_back(m); + } + } + } + all_neg_indices.push_back(neg_indices); + batch_starts.push_back(batch_starts.back() + neg_indices.size()); + } + + framework::LoD out_neg_indices_lod; + out_neg_indices_lod.emplace_back(batch_starts); + int neg_offset = 0; + auto neg_data = out_neg_indices->mutable_data( + framework::make_ddim({static_cast(batch_starts.back()), 1}), + ctx.GetPlace()); + + for (auto neg_indices : all_neg_indices) { + std::copy(neg_indices.begin(), neg_indices.end(), neg_data + neg_offset); + neg_offset += neg_indices.size(); + } + out_neg_indices->set_lod(out_neg_indices_lod); + return; + } +}; + class MineHardExamplesOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(framework::InferShapeContext *ctx) const override { + void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasInput("ClsLoss"), "Input(ClsLoss) of MineHardExamplesOp should not be null."); PADDLE_ENFORCE( - ctx->HasInput("MatchIndics"), - "Input(MatchIndics) of MineHardExamplesOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("MatchDis"), - "Input(MatchDis) of MineHardExamplesOp should not be null."); + ctx->HasInput("MatchIndices"), + "Input(MatchIndices) of MineHardExamplesOp should not be null."); PADDLE_ENFORCE( - ctx->HasOutput("NegIndics"), - "Output(NegIndics) of MineHardExamplesOp should not be null."); + ctx->HasInput("MatchDist"), + "Input(MatchDist) of MineHardExamplesOp should not be null."); PADDLE_ENFORCE( - ctx->HasOutput("UpdatedMatchIndics"), - "Output(UpdatedMatchIndics) of MineHardExamplesOp should not be null."); + ctx->HasOutput("NegIndices"), + "Output(NegIndices) of MineHardExamplesOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("UpdatedMatchIndices"), + "Output(UpdatedMatchIndices) of MineHardExamplesOp should " + "not be null."); auto cls_loss_dims = ctx->GetInputDim("ClsLoss"); - auto idx_dims = ctx->GetInputDim("MatchIndics"); - auto dis_dims = ctx->GetInputDim("MatchDis"); + auto idx_dims = ctx->GetInputDim("MatchIndices"); + auto dis_dims = ctx->GetInputDim("MatchDist"); PADDLE_ENFORCE_EQ(cls_loss_dims.size(), 2UL, "The shape of ClsLoss is [N, Np]."); PADDLE_ENFORCE_EQ(idx_dims.size(), 2UL, - "The shape of MatchIndics is [N, Np]."); + "The shape of MatchIndices is [N, Np]."); PADDLE_ENFORCE_EQ(dis_dims.size(), 2UL, - "The shape of MatchDis is [N, Np]."); + "The shape of MatchDist is [N, Np]."); if (ctx->HasInput("LocLoss")) { auto loc_loss_dims = ctx->GetInputDim("LocLoss"); @@ -61,16 +198,16 @@ class MineHardExamplesOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ( cls_loss_dims[0], idx_dims[0], - "Batch size of ClsLoss and MatchIndics must be the same."); + "Batch size of ClsLoss and MatchIndices must be the same."); PADDLE_ENFORCE_EQ( cls_loss_dims[1], idx_dims[1], - "Prior box number of ClsLoss and MatchIndics must be the same."); + "Prior box number of ClsLoss and MatchIndices must be the same."); PADDLE_ENFORCE_EQ(cls_loss_dims[0], dis_dims[0], - "Batch size of ClsLoss and MatchDis must be the same."); + "Batch size of ClsLoss and MatchDist must be the same."); PADDLE_ENFORCE_EQ( cls_loss_dims[1], idx_dims[1], - "Prior box number of ClsLoss and MatchDis must be the same."); + "Prior box number of ClsLoss and MatchDist must be the same."); auto mining_type = GetMiningType(ctx->Attrs().Get("mining_type")); @@ -80,13 +217,13 @@ class MineHardExamplesOp : public framework::OperatorWithKernel { if (mining_type == MiningType::kMaxNegative) { auto neg_pos_ratio = ctx->Attrs().Get("neg_pos_ratio"); - auto neg_dis_threshold = ctx->Attrs().Get("neg_dis_threshold"); + auto neg_dist_threshold = ctx->Attrs().Get("neg_dist_threshold"); PADDLE_ENFORCE_GT( neg_pos_ratio, 0.0f, "neg_pos_ratio must greater than zero in max_negative mode"); PADDLE_ENFORCE_GT( - neg_dis_threshold, 0.0f, - "neg_dis_threshold must greater than zero in max_negative mode"); + neg_dist_threshold, 0.0f, + "neg_dist_threshold must greater than zero in max_negative mode"); } else if (mining_type == MiningType::kHardExample) { auto sample_size = ctx->Attrs().Get("sample_size"); PADDLE_ENFORCE_GT( @@ -94,12 +231,12 @@ class MineHardExamplesOp : public framework::OperatorWithKernel { "sample_size must greater than zero in hard_example mode"); } - ctx->SetOutputDim("UpdatedMatchIndics", idx_dims); + ctx->SetOutputDim("UpdatedMatchIndices", idx_dims); } protected: framework::OpKernelType GetExpectedKernelType( - const framework::ExecutionContext &ctx) const override { + const framework::ExecutionContext& ctx) const override { return framework::OpKernelType( framework::ToDataType(ctx.Input("ClsLoss")->type()), ctx.device_context()); @@ -108,30 +245,31 @@ class MineHardExamplesOp : public framework::OperatorWithKernel { class MineHardExamplesOpMaker : public framework::OpProtoAndCheckerMaker { public: - MineHardExamplesOpMaker(OpProto *proto, OpAttrChecker *op_checker) + MineHardExamplesOpMaker(OpProto* proto, OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput( "ClsLoss", - "(Tensor, default Tensor), The classification loss wit shape " + "(Tensor, default Tensor), The classification loss with shape " "[N, Np], N is the batch size and Np is the number of prior box."); AddInput("LocLoss", "(Tensor, optional, default Tensor), The localization loss " "wit shape [N, Np], N is the batch size and Np is the number of " "prior box.") .AsDispensable(); - AddInput("MatchIndics", + AddInput("MatchIndices", "(Tensor, Tensor), Matched indices with shape [N, Np], N is " "the batch size and Np is the number of prior box. " - "MatchIndics[i][j] equal -1 means box[j] does not match any " - "entity, otherwise means Box[j] is matched to row."); - AddInput("MatchDis", + "MatchIndices[i][j] equal -1 means the j-th prior box in i-th " + "instance does not match any entity, otherwise means it is " + "matched to row."); + AddInput("MatchDist", "(Tensor, default Tensor) Matched indices with shape [N, " "Np], N is the batch size and Np is the number of prior box."); AddAttr("neg_pos_ratio", "(float) The ratio of the negative box to the positive " "box. Use only when mining_type is equal to max_negative.") .SetDefault(1.0); - AddAttr("neg_dis_threshold", + AddAttr("neg_dist_threshold", "(float) The negative box dis value threshold. " "Use only when mining_type is equal to max_negative.") .SetDefault(0.5); @@ -145,29 +283,31 @@ class MineHardExamplesOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault("max_negative") .InEnum({"hard_example", "max_negative"}); - AddOutput("NegIndics", - "(LoDTensor) The output of negative example indics.a lod tensor " - "with shape [Neg, 1]. The size of lod[0] is batch size, " - "and each element is the box index. " - "For example, the batch size is 2, the lod is [[0, 1, 2]], " - "the sample 0's box 1(MatchIndics[0][1]) is selected, " - "and sample 1's box 0 is selected. The output NegIndics is " - "[[1], [0]]."); - - AddOutput("UpdatedMatchIndics", - "(Tensor) The output of updated MatchIndics, a tensor with " - "shape [N, M]. Only update when mining_type is equal to " - "hard_example. The input MatchIndics elements will be update to " - "-1 when it not in the highest loss list"); + AddOutput( + "NegIndices", + "(LoDTensor) The output of negative example indices. a LoDTensor " + "with shape [Neg, 1]. The size of lod[0] minus 1 is batch size, " + "and each element is the prior box index. " + "For example, the batch size is 2, the lod is [[0, 1, 2]], " + "the sample 0's box 1(MatchIndices[0][1]) is selected, " + "and sample 1's box 0 is selected. The output NegIndices is " + "[[1], [0]]."); + + AddOutput("UpdatedMatchIndices", + "(Tensor) The output of updated MatchIndices, a tensor with " + "shape [N, Np]. Only update when mining_type is equal to " + "hard_example. The input MatchIndices elements will be update to " + "-1 when it is not in the candidate high loss list of negative " + "examples."); AddComment(R"DOC( Mine hard examples Operator. -This operator implements hard example mining to select a subset of negative box indics. +This operator implements hard example mining to select a subset of negative box indices. For each image, selects the box with highest losses. subject to the condition that the box cannot have -an MatchDis > neg_dis_threshold when mining_type is equals max_negative. The selected number is +an Matcht > neg_dist_threshold when mining_type is equals max_negative. The selected number is min(sample_size, max_negative_box_number) when mining_type is equals hard_example, or min(neg_pos_ratio * positive_box_number, max_negative_box_number) when mining_type is -equals max_negative, where the max_negative_box_number is the count of MatchIndics elements with value -1. +equals max_negative, where the max_negative_box_number is the count of MatchIndices elements with value -1. )DOC"); } }; diff --git a/paddle/operators/mine_hard_examples_op.h b/paddle/operators/mine_hard_examples_op.h deleted file mode 100755 index 0a652a60c51073ebc9ae9f8ba4ef6a2973085224..0000000000000000000000000000000000000000 --- a/paddle/operators/mine_hard_examples_op.h +++ /dev/null @@ -1,148 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { - -enum MiningType { kNone = 0, kMaxNegative, kHardExample }; - -template -bool SortScoreDescend(const std::pair& pair1, - const std::pair& pair2) { - return pair1.first > pair2.first; -} - -inline bool IsEligibleMining(const MiningType mining_type, const int match_idx, - const float match_dis, - const float neg_dis_threshold) { - if (mining_type == MiningType::kMaxNegative) { - return match_idx == -1 && match_dis < neg_dis_threshold; - } else if (mining_type == MiningType::kHardExample) { - return true; - } else { - return false; - } -} - -MiningType GetMiningType(std::string str) { - if (str == "max_negative") { - return MiningType::kMaxNegative; - } else if (str == "hard_example") { - return MiningType::kHardExample; - } else { - return MiningType::kNone; - } -} - -template -class MineHardExamplesKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& ctx) const override { - auto* in_cls_loss = ctx.Input("ClsLoss"); - auto* in_loc_loss = ctx.Input("LocLoss"); - auto* in_matched_indics = ctx.Input("MatchIndics"); - auto* in_match_dis = ctx.Input("MatchDis"); - float neg_pos_ratio = ctx.Attr("neg_pos_ratio"); - T neg_dis_threshold = static_cast(ctx.Attr("neg_dis_threshold")); - int sample_size = ctx.Attr("sample_size"); - MiningType mining_type = - GetMiningType(ctx.Attr("mining_type")); - - auto out_neg_indics = ctx.Output("NegIndics"); - auto out_match_indics = ctx.Output("UpdatedMatchIndics"); - - framework::Copy(*in_matched_indics, ctx.GetPlace(), out_match_indics); - - int batch_size = in_matched_indics->dims()[0]; - int prior_num = in_matched_indics->dims()[1]; - - auto match_indices = framework::EigenMatrix::From(*in_matched_indics); - - auto match_indices_et = - framework::EigenMatrix::From(*out_match_indics); - - auto match_dis = framework::EigenMatrix::From(*in_match_dis); - auto cls_loss = framework::EigenMatrix::From(*in_cls_loss); - auto loc_loss = framework::EigenMatrix::From(*in_loc_loss); - - std::vector> all_neg_indices; - int all_neg_num = 0; - for (int n = 0; n < batch_size; ++n) { - std::vector> loss_idx; - int neg_sel = 0; - for (int m = 0; m < prior_num; ++m) { - if (IsEligibleMining(mining_type, match_indices(n, m), match_dis(n, m), - neg_dis_threshold)) { - T loss = cls_loss(n, m); - if (mining_type == MiningType::kHardExample) { - loss = cls_loss(n, m) + loc_loss(n, m); - } - loss_idx.push_back(std::make_pair(loss, m)); - ++neg_sel; - } - } - if (mining_type == MiningType::kMaxNegative) { - int num_pos = 0; - for (int m = 0; m < prior_num; ++m) { - if (match_indices(n, m) != -1) ++num_pos; - } - neg_sel = std::min(static_cast(num_pos * neg_pos_ratio), neg_sel); - } else if (mining_type == MiningType::kHardExample) { - neg_sel = std::min(sample_size, neg_sel); - } - std::sort(loss_idx.begin(), loss_idx.end(), SortScoreDescend); - std::set sel_indices; - std::vector neg_indices; - for (int n = 0; n < neg_sel; ++n) { - sel_indices.insert(loss_idx[n].second); - } - - for (int m = 0; m < prior_num; ++m) { - if (match_indices(n, m) > -1) { - if (mining_type == MiningType::kHardExample && - sel_indices.find(m) == sel_indices.end()) { - match_indices_et(n, m) = -1; - } - } else { - if (sel_indices.find(m) != sel_indices.end()) { - neg_indices.push_back(m); - } - } - } - all_neg_indices.push_back(neg_indices); - all_neg_num += neg_indices.size(); - } - - framework::LoD out_neg_indics_lod; - out_neg_indics_lod.resize(1); - int neg_offset = 0; - auto neg_data = out_neg_indics->mutable_data( - framework::make_ddim({all_neg_num, 1}), ctx.GetPlace()); - out_neg_indics_lod[0].push_back(neg_offset); - for (auto neg_indices : all_neg_indices) { - for (auto neg_idx : neg_indices) { - neg_data[neg_offset++] = neg_idx; - } - out_neg_indics_lod[0].push_back(neg_offset); - } - out_neg_indics->set_lod(out_neg_indics_lod); - return; - } -}; -} // namespace operators - -} // namespace paddle diff --git a/python/paddle/v2/fluid/tests/test_mine_hard_examples_op.py b/python/paddle/v2/fluid/tests/test_mine_hard_examples_op.py index e7dd04740a68523c80298102f8663a8cbde21bed..c27573c3d69037bc48e0b6a90636b3f027f15a41 100755 --- a/python/paddle/v2/fluid/tests/test_mine_hard_examples_op.py +++ b/python/paddle/v2/fluid/tests/test_mine_hard_examples_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import sys @@ -24,8 +25,8 @@ class TestMineHardExamplesOp(OpTest): self.inputs = { 'ClsLoss': self.cls_loss, 'LocLoss': self.loc_loss, - 'MatchIndics': self.match_indices, - 'MatchDis': self.match_dis + 'MatchIndices': self.match_indices, + 'MatchDist': self.match_dis } self.attrs = { @@ -36,8 +37,8 @@ class TestMineHardExamplesOp(OpTest): } self.outputs = { - 'NegIndics': (self.neg_indices, self.neg_indices_lod), - 'UpdatedMatchIndics': self.updated_match_indices + 'NegIndices': (self.neg_indices, self.neg_indices_lod), + 'UpdatedMatchIndices': self.updated_match_indices } def test_check_output(self):