DetectionMAPEvaluator.cpp

/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "Evaluator.h"
#include "paddle/gserver/layers/DetectionUtil.h"

using std::map;
using std::vector;
using std::pair;
using std::make_pair;

namespace paddle {

/**
 * @brief detection map Evaluator
 *
 * The config file api is detection_map_evaluator.
 */
class DetectionMAPEvaluator : public Evaluator {
public:
  DetectionMAPEvaluator()
      : evaluateDifficult_(false), cpuOutput_(nullptr), cpuLabel_(nullptr) {}

  virtual void start() {
    Evaluator::start();
    allTruePos_.clear();
    allFalsePos_.clear();
    numPos_.clear();
  }

  virtual real evalImp(std::vector<Argument>& arguments) {
    overlapThreshold_ = config_.overlap_threshold();
    backgroundId_ = config_.background_id();
    evaluateDifficult_ = config_.evaluate_difficult();
    apType_ = config_.ap_type();

    MatrixPtr detectTmpValue = arguments[0].value;
    Matrix::resizeOrCreate(cpuOutput_,
                           detectTmpValue->getHeight(),
                           detectTmpValue->getWidth(),
                           false,
                           false);

    MatrixPtr labelTmpValue = arguments[1].value;
    Matrix::resizeOrCreate(cpuLabel_,
                           labelTmpValue->getHeight(),
                           labelTmpValue->getWidth(),
                           false,
                           false);

    cpuOutput_->copyFrom(*detectTmpValue);
    cpuLabel_->copyFrom(*labelTmpValue);

    Argument label = arguments[1];
    const int* labelIndex = label.sequenceStartPositions->getData(false);
    size_t batchSize = label.getNumSequences();

    vector<map<size_t, vector<NormalizedBBox>>> allGTBBoxes;
    vector<map<size_t, vector<pair<real, NormalizedBBox>>>> allDetectBBoxes;

    for (size_t n = 0; n < batchSize; ++n) {
      map<size_t, vector<NormalizedBBox>> bboxes;
      for (int i = labelIndex[n]; i < labelIndex[n + 1]; ++i) {
        vector<NormalizedBBox> bbox;
        getBBoxFromLabelData(cpuLabel_->getData() + i * 6, 1, bbox);
        int c = cpuLabel_->getData()[i * 6];
        bboxes[c].push_back(bbox[0]);
      }
      allGTBBoxes.push_back(bboxes);
    }

    size_t imgId = 0;
    for (size_t n = 0; n < cpuOutput_->getHeight();) {
      map<size_t, vector<pair<real, NormalizedBBox>>> bboxes;
      while (cpuOutput_->getData()[n * 7] == imgId &&
             n < cpuOutput_->getHeight()) {
        vector<real> label;
        vector<real> score;
        vector<NormalizedBBox> bbox;
        getBBoxFromDetectData(
            cpuOutput_->getData() + n * 7, 1, label, score, bbox);
        bboxes[label[0]].push_back(make_pair(score[0], bbox[0]));
        ++n;
      }
      ++imgId;
      if (imgId > batchSize) break;
      allDetectBBoxes.push_back(bboxes);
    }

    for (size_t n = 0; n < batchSize; ++n) {
      for (map<size_t, vector<NormalizedBBox>>::iterator it =
               allGTBBoxes[n].begin();
           it != allGTBBoxes[n].end();
           ++it) {
        size_t count = 0;
        if (evaluateDifficult_) {
          count = it->second.size();
        } else {
          for (size_t i = 0; i < it->second.size(); ++i)
            if (!(it->second[i].isDifficult)) ++count;
        }
        if (numPos_.find(it->first) == numPos_.end() && count != 0) {
          numPos_[it->first] = count;
        } else {
          numPos_[it->first] += count;
        }
      }
    }

    // calcTFPos
    calcTFPos(
        batchSize, allGTBBoxes, allDetectBBoxes, &allTruePos_, &allFalsePos_);

    return 0;
  }

  virtual void printStats(std::ostream& os) const {
    real mAP = calcMAP();
    os << "Detection mAP=" << mAP * 100;
  }

  virtual void distributeEval(ParameterClient2* client) {
    LOG(FATAL) << "Distribute detection evaluation not implemented.";
  }

protected:
  void calcTFPos(const size_t batchSize,
                 const vector<map<size_t, vector<NormalizedBBox>>>& allGTBBoxes,
                 const vector<map<size_t, vector<pair<real, NormalizedBBox>>>>&
                     allDetectBBoxes,
                 map<size_t, vector<pair<real, size_t>>>* allTruePos,
                 map<size_t, vector<pair<real, size_t>>>* allFalsePos) {
    for (size_t n = 0; n < allDetectBBoxes.size(); ++n) {
      if (allGTBBoxes[n].size() == 0) {
        for (map<size_t, vector<pair<real, NormalizedBBox>>>::const_iterator
                 it = allDetectBBoxes[n].begin();
             it != allDetectBBoxes[n].end();
             ++it) {
          size_t label = it->first;
          for (size_t i = 0; i < it->second.size(); ++i) {
            (*allTruePos)[label].push_back(make_pair(it->second[i].first, 0));
            (*allFalsePos)[label].push_back(make_pair(it->second[i].first, 1));
          }
        }
      } else {
        for (map<size_t, vector<pair<real, NormalizedBBox>>>::const_iterator
                 it = allDetectBBoxes[n].begin();
             it != allDetectBBoxes[n].end();
             ++it) {
          size_t label = it->first;
          vector<pair<real, NormalizedBBox>> predBBoxes = it->second;
          if (allGTBBoxes[n].find(label) == allGTBBoxes[n].end()) {
            for (size_t i = 0; i < predBBoxes.size(); ++i) {
              (*allTruePos)[label].push_back(make_pair(predBBoxes[i].first, 0));
              (*allFalsePos)[label].push_back(
                  make_pair(predBBoxes[i].first, 1));
            }
          } else {
            vector<NormalizedBBox> gtBBoxes =
                allGTBBoxes[n].find(label)->second;
            vector<bool> visited(gtBBoxes.size(), false);
            // Sort detections in descend order based on scores
            std::sort(predBBoxes.begin(),
                      predBBoxes.end(),
                      sortScorePairDescend<NormalizedBBox>);
            for (size_t i = 0; i < predBBoxes.size(); ++i) {
              real maxOverlap = -1.0;
              size_t maxIdx = 0;
              for (size_t j = 0; j < gtBBoxes.size(); ++j) {
                real overlap =
                    jaccardOverlap(predBBoxes[i].second, gtBBoxes[j]);
                if (overlap > maxOverlap) {
                  maxOverlap = overlap;
                  maxIdx = j;
                }
              }
              if (maxOverlap > overlapThreshold_) {
                if (evaluateDifficult_ ||
                    (!evaluateDifficult_ && !gtBBoxes[maxIdx].isDifficult)) {
                  if (!visited[maxIdx]) {
                    (*allTruePos)[label].push_back(
                        make_pair(predBBoxes[i].first, 1));
                    (*allFalsePos)[label].push_back(
                        make_pair(predBBoxes[i].first, 0));
                    visited[maxIdx] = true;
                  } else {
                    (*allTruePos)[label].push_back(
                        make_pair(predBBoxes[i].first, 0));
                    (*allFalsePos)[label].push_back(
                        make_pair(predBBoxes[i].first, 1));
                  }
                }
              } else {
                (*allTruePos)[label].push_back(
                    make_pair(predBBoxes[i].first, 0));
                (*allFalsePos)[label].push_back(
                    make_pair(predBBoxes[i].first, 1));
              }
            }
          }
        }
      }
    }
  }

  real calcMAP() const {
    real mAP = 0.0;
    size_t count = 0;
    for (map<size_t, size_t>::const_iterator it = numPos_.begin();
         it != numPos_.end();
         ++it) {
      size_t label = it->first;
      size_t labelNumPos = it->second;
      if (labelNumPos == 0 || allTruePos_.find(label) == allTruePos_.end())
        continue;
      vector<pair<real, size_t>> labelTruePos = allTruePos_.find(label)->second;
      vector<pair<real, size_t>> labelFalsePos =
          allFalsePos_.find(label)->second;
      // Compute average precision.
      vector<size_t> tpCumSum;
      getAccumulation(labelTruePos, &tpCumSum);
      vector<size_t> fpCumSum;
      getAccumulation(labelFalsePos, &fpCumSum);
      std::vector<real> precision, recall;
      size_t num = tpCumSum.size();
      // Compute Precision.
      for (size_t i = 0; i < num; ++i) {
        CHECK_LE(tpCumSum[i], labelNumPos);
        precision.push_back(static_cast<real>(tpCumSum[i]) /
                            static_cast<real>(tpCumSum[i] + fpCumSum[i]));
        recall.push_back(static_cast<real>(tpCumSum[i]) / labelNumPos);
      }
      // VOC2007 style
      if (apType_ == "11point") {
        vector<real> maxPrecisions(11, 0.0);
        int startIdx = num - 1;
        for (int j = 10; j >= 0; --j)
          for (int i = startIdx; i >= 0; --i) {
            if (recall[i] < j / 10.) {
              startIdx = i;
              if (j > 0) maxPrecisions[j - 1] = maxPrecisions[j];
              break;
            } else {
              if (maxPrecisions[j] < precision[i])
                maxPrecisions[j] = precision[i];
            }
          }
        for (int j = 10; j >= 0; --j) mAP += maxPrecisions[j] / 11;
        ++count;
      } else if (apType_ == "Integral") {
        // Nature integral
        real averagePrecisions = 0.;
        real prevRecall = 0.;
        for (size_t i = 0; i < num; ++i) {
          if (fabs(recall[i] - prevRecall) > 1e-6)
            averagePrecisions += precision[i] * fabs(recall[i] - prevRecall);
          prevRecall = recall[i];
        }
        mAP += averagePrecisions;
        ++count;
      } else {
        LOG(FATAL) << "Unkown ap version: " << apType_;
      }
    }
    if (count != 0) mAP /= count;
    return mAP;
  }

  void getAccumulation(vector<pair<real, size_t>> inPairs,
                       vector<size_t>* accuVec) const {
    std::stable_sort(
        inPairs.begin(), inPairs.end(), sortScorePairDescend<size_t>);
    accuVec->clear();
    size_t sum = 0;
    for (size_t i = 0; i < inPairs.size(); ++i) {
      sum += inPairs[i].second;
      accuVec->push_back(sum);
    }
  }

  std::string getTypeImpl() const { return "detection_map"; }

  real getValueImpl() const { return calcMAP() * 100; }

private:
  real overlapThreshold_;
  bool evaluateDifficult_;
  size_t backgroundId_;
  std::string apType_;

  MatrixPtr cpuOutput_;
  MatrixPtr cpuLabel_;

  map<size_t, size_t> numPos_;
  map<size_t, vector<pair<real, size_t>>> allTruePos_;
  map<size_t, vector<pair<real, size_t>>> allFalsePos_;
};

REGISTER_EVALUATOR(detection_map, DetectionMAPEvaluator);

}  // namespace paddle