From 796eaf345d177e579414fd194c902ee1c365441f Mon Sep 17 00:00:00 2001 From: Dong Zhihong Date: Thu, 2 Nov 2017 20:20:16 -0700 Subject: [PATCH] "add accuracy " --- doc/design/evaluator.md | 40 ++++++-------- python/paddle/v2/framework/evaluator.py | 69 ++++++++++++++++--------- python/paddle/v2/framework/framework.py | 2 +- 3 files changed, 62 insertions(+), 49 deletions(-) diff --git a/doc/design/evaluator.md b/doc/design/evaluator.md index ccec3068e..771cb4d5f 100644 --- a/doc/design/evaluator.md +++ b/doc/design/evaluator.md @@ -7,9 +7,9 @@ During training or serving, we provide the evaluation function to measure the mo ### Evaluator Design Currently, every operation is expressed in the graph. we divide the evaluator process into three steps. -1. Initialize the metric state necessary and add it into the block. +1. Initialize the metric state and add it into the block. -2. Calculate the statistic of the metric state in every mini-batch. The single operator is only responsible for calculating necessary statistics for one mini-batch. For example, accuracy operator only calculate a minibatch data if run once.\ +2. Calculate the statistic of the metric state in every mini-batch. The single operator is only responsible for calculating necessary statistics for one mini-batch. For example, accuracy operator only calculate a minibatch data if run once. 3. Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. When it comes to distributed training/Multi-GPU training, aggregate the value from different devices. @@ -20,38 +20,30 @@ This design is shown in python API. There would be an abstract python interface ```python class Evaluator(object): """ - Evalutor Base class. + Evaluator Base class. """ def __init__(self): """ - create metric states and append to block + Different evaluator may has different metric states. E.g, Accuracy need two variables, total and right sample counts. + Auc need four variables, `true_positives`, + `true_negatives`, `false_positives` and `false_negatives`. So every evaluator should create its needed variables and append the related mini-batch operator to main_program + + The initialization of Evaluator should be responsible for: + create metric states and append to the main_program + add mini-batch evaluator caculate operators to the main_program + add increment operator to accumulate the metric states """ pass - def _clear_state(self): + def clear(self): """ - clear metric states at the begin of each pass + clear metric states at the begin of each pass/user specified batch """ - pass - - def _append_evalutor_op(self): - """ - add mini-batch caculate operators to block - add increment operator to accumulate the metric state - """ - pass - - def _merge(self): - """ - Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. - """ - pass + return init_program def evaluate(self): """ - only one exported interface - user calculate the result + Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. """ - pass - + return eval_program ``` diff --git a/python/paddle/v2/framework/evaluator.py b/python/paddle/v2/framework/evaluator.py index 90a7601c6..47bcca0b7 100644 --- a/python/paddle/v2/framework/evaluator.py +++ b/python/paddle/v2/framework/evaluator.py @@ -1,41 +1,62 @@ -import paddle.v2.framework.op as op -import numpy as np +from paddle.v2.framework.framework import Program, g_program, g_init_program import paddle.v2.framework.core as core class Evaluator(object): """ Evalutor Base class. + + create metric states + add mini-batch evaluator caculate operator + add increment operator to accumulate the metric states """ - def __init__(self): - """ - create metric states and append to block - """ - pass + def __init__(self, input=None, **kwargs): + if "program" in kwargs: + self._program = kwargs.get("program") + else: + self._program = input.program + self._states = [] - def _clear_state(self): - """ - clear metric states at the begin of each pass - """ - pass + def _create_tmp_variable(self, name, dtype): + return self.program.current_block().create_var( + name=unique_name(".".join([self.name, 'tmp'])), + dtype=dtype, + persistable=False) - def _append_evalutor_op(self): + @staticmethod + def clear(self): """ - add mini-batch caculate operators to block - add increment operator to accumulate the metric state + clear metric states at the begin of each pass/user specified batch + return a clear """ - pass + raise NotImplementedError() - def _merge(self): + def evaluate(self): """ Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. """ - pass + raise NotImplementedError() - def evaluate(self): - """ - only one exported interface - user calculate the result - """ - pass + +class Accuracy(Evaluator): + def __init__(self, input, label, k=1, **kwargs): + super(Accuracy, self).__init__(input=input, **kwargs) + topk_out = helper.create_tmp_variable(dtype=input.data_type) + topk_indices = helper.create_tmp_variable(dtype="int64") + helper.append_op( + type="top_k", + inputs={"X": [input]}, + outputs={"Out": [topk_out], + "Indices": [topk_indices]}, + attrs={"k": k}) + acc_out_dtype = kwargs.get("out_dtype", "float32") + acc_out = helper.create_tmp_variable(dtype=acc_out_dtype) + helper.append_op( + type="accuracy", + inputs={ + "Out": [topk_out], + "Indices": [topk_indices], + "Label": [label] + }, + outputs={"Accuracy": [acc_out]}) diff --git a/python/paddle/v2/framework/framework.py b/python/paddle/v2/framework/framework.py index 7da6f8135..548f04aa4 100644 --- a/python/paddle/v2/framework/framework.py +++ b/python/paddle/v2/framework/framework.py @@ -530,7 +530,7 @@ class Parameter(Variable): raise ValueError("Parameter shape should not be related with " "batch-size") - Variable.__init__( + super(Parameter, self).__init__( self, block, persistable=True, shape=shape, dtype=dtype, **kwargs) self.trainable = kwargs.get('trainable', True) -- GitLab