Evaluators¶

Defines

REGISTER_EVALUATOR(__type_name, __class_name)¶

namespace paddle¶

class AucEvaluator¶

Inherits from paddle::Evaluator

Public Functions

AucEvaluator(int32_t colIdx)¶

evaluate AUC using colIdx-th column as prediction.

colIdx = 0: the 0-th column. colIdx > 0: the colIdx-th column. colIdx < 0: the last colIdx-th column.

virtual void start()¶: start to evaluate some data

virtual real evalImp(std::vector<Argument> &arguments)¶: Process a batch of data. return the score for the batch if it make sense to sum the score across batches. Otherwise evaluator should return 0 and override finish() and printStats() to do the right calculation.

virtual void printStats(std::ostream &os)¶

virtual void distributeEval(ParameterClient2 *client)¶

Private Functions

AucEvaluator()¶

double calcAuc()¶

Private Members

double statPos_[kBinNum_+1]¶

double statNeg_[kBinNum_+1]¶

int32_t colIdx_¶

uint32_t realColumnIdx_¶

MatrixPtr cpuOutput_¶

IVectorPtr cpuLabel_¶

MatrixPtr cpuWeight_¶

Private Static Functions

static double trapezoidArea(double X1, double X2, double Y1, double Y2)¶

Private Static Attributes

const uint32_t kBinNum_¶

const int kNegativeLabel_¶

class DummyEvaluator¶

Inherits from paddle::Evaluator

Public Functions

DummyEvaluator()¶

virtual void init(const EvaluatorConfig&)¶

virtual void start()¶: start to evaluate some data

virtual void eval(const NeuralNetwork &nn)¶: Process a batch of data.

virtual real evalImp(std::vector<Argument> &arguments)¶: Process a batch of data. return the score for the batch if it make sense to sum the score across batches. Otherwise evaluator should return 0 and override finish() and printStats() to do the right calculation.

virtual void finish()¶: finish the evaluation.

virtual void printStats(std::ostream&)¶

class Evaluator¶

Subclassed by paddle::AucEvaluator, paddle::ChunkEvaluator, paddle::ClassificationErrorEvaluator, paddle::ColumnSumEvaluator, paddle::CombinedEvaluator, paddle::CTCErrorEvaluator, paddle::DummyEvaluator, paddle::GradientPrinter, paddle::MaxFramePrinter, paddle::MaxIdPrinter, paddle::MultiCombinedEvaluator, paddle::PnpairEvaluator, paddle::PrecisionRecallEvaluator, paddle::RankAucEvaluator, paddle::SequenceTextPrinter, paddle::SumEvaluator, paddle::ValuePrinter

Public Functions

Evaluator()¶

virtual ~Evaluator()¶

virtual void init(const EvaluatorConfig &config)¶

virtual void start()¶: start to evaluate some data

virtual void eval(const NeuralNetwork &nn)¶: Process a batch of data.

virtual real evalImp(std::vector<Argument> &arguments) = 0¶: Process a batch of data. return the score for the batch if it make sense to sum the score across batches. Otherwise evaluator should return 0 and override finish() and printStats() to do the right calculation.

virtual void updateSamplesNum(const std::vector<Argument> &arguments)¶: Update the number of processed samples

virtual void distributeEval(ParameterClient2 *client)¶

void mergeResultsOfAllClients(ParameterClient2 *client)¶

virtual void finish()¶: finish the evaluation.

virtual void printStats(std::ostream &os)¶

Public Static Functions

Evaluator *create(const EvaluatorConfig &config)¶

Public Static Attributes

ClassRegistrar<Evaluator> registrar_¶

Protected Attributes

EvaluatorConfig config_¶

double numSamples_¶

double totalScore_¶

Friends

std::ostream &operator<<(std::ostream &os, Evaluator &evaluator)¶

std::ostream &&operator<<(std::ostream &&os, Evaluator &evaluator)¶

class PnpairEvaluator¶

#include <Evaluator.h>

Positive-negative pair rate Evaluator

Inherits from paddle::Evaluator

Public Functions

PnpairEvaluator()¶

virtual void start()¶: start to evaluate some data

virtual real evalImp(std::vector<Argument> &arguments)¶: Process a batch of data. return the score for the batch if it make sense to sum the score across batches. Otherwise evaluator should return 0 and override finish() and printStats() to do the right calculation.

void printPredictResults()¶

void stat(size_t start, size_t end, PredictionResult *answers, double &pos, double &neg, double &spe)¶

void calc(std::vector<PredictionResult> &predictArray)¶

virtual void finish()¶: finish the evaluation.

virtual void printStats(std::ostream &os)¶

virtual void distributeEval(ParameterClient2 *client)¶

Public Members

std::vector<PredictionResult> predictArray_¶

Private Members

double pairArray_[kPairArrayNum_]¶

MatrixPtr cpuOutput_¶

IVectorPtr cpuLabel_¶

IVectorPtr cpuInfo_¶

MatrixPtr cpuWeight_¶

Private Static Attributes

const uint32_t kPairArrayNum_¶

struct PredictionResult¶

Public Functions

PredictionResult(real __out, int __label, int __queryid, real __weight)¶

Public Members

real out¶

int label¶

int queryid¶

real weight¶

class PrecisionRecallEvaluator¶

Inherits from paddle::Evaluator

Public Functions

PrecisionRecallEvaluator()¶

virtual void start()¶: start to evaluate some data

virtual real evalImp(std::vector<Argument> &arguments)¶: Process a batch of data. return the score for the batch if it make sense to sum the score across batches. Otherwise evaluator should return 0 and override finish() and printStats() to do the right calculation.

virtual void printStats(std::ostream &os)¶

virtual void distributeEval(ParameterClient2 *client)¶

Private Functions

void calcStatsInfo(const MatrixPtr &output, const IVectorPtr &label, const MatrixPtr &weight)¶

void calcStatsInfoMulti(const MatrixPtr &output, const MatrixPtr &label, const MatrixPtr &weight)¶

Private Members

bool isMultiBinaryLabel_¶

std::vector<StatsInfo> statsInfo_¶

MatrixPtr cpuOutput_¶

IVectorPtr cpuLabel_¶

MatrixPtr cpuWeight_¶

Private Static Functions

static double calcPrecision(double TP, double FP)¶

static double calcRecall(double TP, double FN)¶

static double calcF1Score(double precision, double recall)¶

struct StatsInfo¶

Public Functions

StatsInfo()¶

Public Members

double TP¶

double TN¶

double FP¶

double FN¶

class RankAucEvaluator¶

#include <Evaluator.h>

RankAucEvaluator calculates the AUC of each list (i.e., titles under the same query), and averages them.

Each list should be organized as a sequence. The inputs of this evaluator is [output, click, pv]. If pv is not provided, it will be set to 1. The types of click and pv are dense value.

Inherits from paddle::Evaluator

Public Functions

virtual void start()¶: start to evaluate some data

virtual void updateSamplesNum(const std::vector<Argument> &arguments)¶: Update the number of processed samples

virtual real evalImp(std::vector<Argument> &arguments)¶: Process a batch of data. return the score for the batch if it make sense to sum the score across batches. Otherwise evaluator should return 0 and override finish() and printStats() to do the right calculation.

virtual void distributeEval(ParameterClient2 *client)¶

Private Functions

double calcRankAuc(real *outputData, real *clickData, real *pvData, size_t size)¶

Private Members

MatrixPtr output_¶

MatrixPtr click_¶

MatrixPtr pv_¶

std::vector<std::pair<real, int>> outputPair_¶

namespace paddle¶

Functions

REGISTER_EVALUATOR(chunk, ChunkEvaluator)¶

class ChunkEvaluator¶

Chunk evaluator is used to evaluate segment labelling accuracy for a sequence. It calculates the chunk detection F1 score.

A chunk is correctly detected if its beginning, end and type are correct. Other chunk type is ignored. For each label in the label sequence, we have

tagType = label % numTagType
chunkType = label / numTagType
otherChunkType = numChunkTypes

The total number of different labels is numTagType*numChunkTypes+1 We support 4 labelling scheme The tag type for each of the scheme is shown as follows:

Scheme Begin Inside End   Single
 plain  0     -      -     -
 IOB    0     1      -     -
 IOE    -     0      1     -
 IOBES  0     1      2     3

‘plain’ means the whole chunk must contain exactly the same chunk label.

Inherits from paddle::Evaluator

Public Functions

virtual void init(const EvaluatorConfig &config)¶

virtual void start()¶: start to evaluate some data

virtual void printStats(std::ostream &os)¶

virtual void distributeEval(ParameterClient2 *client)¶

virtual real evalImp(std::vector<Argument> &arguments)¶: Process a batch of data. return the score for the batch if it make sense to sum the score across batches. Otherwise evaluator should return 0 and override finish() and printStats() to do the right calculation.

void eval1(int *output, int *label, int length)¶

void getSegments(int *label, int length, std::vector<Segment> &segments)¶

bool isChunkEnd(int prevTag, int prevType, int tag, int type)¶

bool isChunkBegin(int prevTag, int prevType, int tag, int type)¶

Private Members

int otherChunkType_¶

int numChunkTypes_¶

int numTagTypes_¶

int tagBegin_¶

int tagInside_¶

int tagEnd_¶

int tagSingle_¶

int64_t numLabelSegments_¶

int64_t numOutputSegments_¶

int64_t numCorrect_¶

std::vector<Segment> labelSegments_¶

std::vector<Segment> outputSegments_¶

struct Segment¶

Public Functions

bool operator==(const Segment &y) const¶

Public Members

int begin¶

int end¶

int type¶

namespace paddle¶

Functions

REGISTER_EVALUATOR(ctc_edit_distance, CTCErrorEvaluator)¶

class CTCErrorEvaluator¶

calculate sequence-to-sequence edit distance

Inherits from paddle::Evaluator

Public Functions

CTCErrorEvaluator()¶

virtual real evalImp(std::vector<Argument> &arguments)¶: Process a batch of data. return the score for the batch if it make sense to sum the score across batches. Otherwise evaluator should return 0 and override finish() and printStats() to do the right calculation.

virtual void eval(const NeuralNetwork &nn)¶: Process a batch of data.

virtual void start()¶: start to evaluate some data

virtual void printStats(std::ostream &os)¶

virtual void distributeEval(ParameterClient2 *client)¶

Private Functions

std::vector<int> path2String(const std::vector<int> &path)¶

std::vector<int> bestLabelSeq()¶

real stringAlignment(std::vector<int> &gtStr, std::vector<int> &recogStr, bool backtrace = true, real sp = 1.0, real dp = 1.0, real ip = 1.0)¶

real editDistance(real *output, int numTimes, int numClasses, int *labels, int labelsLen)¶

Private Members

MatrixPtr outActivations_¶

int numTimes_¶

int numClasses_¶

int numSequences_¶

int blank_¶

real deletions_¶

real insertions_¶

real substitutions_¶

int seqClassficationError_¶

Evaluators¶

Previous topic

Next topic

This Page