diff --git a/doc/design/evaluator.md b/doc/design/evaluator.md index ccec3068e65f6339600b073bf403bfae6e71d2fa..771cb4d5f73b7234d810a38d97e0758d5a54a0dc 100644 --- a/doc/design/evaluator.md +++ b/doc/design/evaluator.md @@ -7,9 +7,9 @@ During training or serving, we provide the evaluation function to measure the mo ### Evaluator Design Currently, every operation is expressed in the graph. we divide the evaluator process into three steps. -1. Initialize the metric state necessary and add it into the block. +1. Initialize the metric state and add it into the block. -2. Calculate the statistic of the metric state in every mini-batch. The single operator is only responsible for calculating necessary statistics for one mini-batch. For example, accuracy operator only calculate a minibatch data if run once.\ +2. Calculate the statistic of the metric state in every mini-batch. The single operator is only responsible for calculating necessary statistics for one mini-batch. For example, accuracy operator only calculate a minibatch data if run once. 3. Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. When it comes to distributed training/Multi-GPU training, aggregate the value from different devices. @@ -20,38 +20,30 @@ This design is shown in python API. There would be an abstract python interface ```python class Evaluator(object): """ - Evalutor Base class. + Evaluator Base class. """ def __init__(self): """ - create metric states and append to block + Different evaluator may has different metric states. E.g, Accuracy need two variables, total and right sample counts. + Auc need four variables, `true_positives`, + `true_negatives`, `false_positives` and `false_negatives`. So every evaluator should create its needed variables and append the related mini-batch operator to main_program + + The initialization of Evaluator should be responsible for: + create metric states and append to the main_program + add mini-batch evaluator caculate operators to the main_program + add increment operator to accumulate the metric states """ pass - def _clear_state(self): + def clear(self): """ - clear metric states at the begin of each pass + clear metric states at the begin of each pass/user specified batch """ - pass - - def _append_evalutor_op(self): - """ - add mini-batch caculate operators to block - add increment operator to accumulate the metric state - """ - pass - - def _merge(self): - """ - Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. - """ - pass + return init_program def evaluate(self): """ - only one exported interface - user calculate the result + Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. """ - pass - + return eval_program ``` diff --git a/python/paddle/v2/framework/evaluator.py b/python/paddle/v2/framework/evaluator.py index 90a7601c66bce04771c486eceb40d44945166b49..47bcca0b79dbff02322a840f02abe1c5022503e3 100644 --- a/python/paddle/v2/framework/evaluator.py +++ b/python/paddle/v2/framework/evaluator.py @@ -1,41 +1,62 @@ -import paddle.v2.framework.op as op -import numpy as np +from paddle.v2.framework.framework import Program, g_program, g_init_program import paddle.v2.framework.core as core class Evaluator(object): """ Evalutor Base class. + + create metric states + add mini-batch evaluator caculate operator + add increment operator to accumulate the metric states """ - def __init__(self): - """ - create metric states and append to block - """ - pass + def __init__(self, input=None, **kwargs): + if "program" in kwargs: + self._program = kwargs.get("program") + else: + self._program = input.program + self._states = [] - def _clear_state(self): - """ - clear metric states at the begin of each pass - """ - pass + def _create_tmp_variable(self, name, dtype): + return self.program.current_block().create_var( + name=unique_name(".".join([self.name, 'tmp'])), + dtype=dtype, + persistable=False) - def _append_evalutor_op(self): + @staticmethod + def clear(self): """ - add mini-batch caculate operators to block - add increment operator to accumulate the metric state + clear metric states at the begin of each pass/user specified batch + return a clear """ - pass + raise NotImplementedError() - def _merge(self): + def evaluate(self): """ Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. """ - pass + raise NotImplementedError() - def evaluate(self): - """ - only one exported interface - user calculate the result - """ - pass + +class Accuracy(Evaluator): + def __init__(self, input, label, k=1, **kwargs): + super(Accuracy, self).__init__(input=input, **kwargs) + topk_out = helper.create_tmp_variable(dtype=input.data_type) + topk_indices = helper.create_tmp_variable(dtype="int64") + helper.append_op( + type="top_k", + inputs={"X": [input]}, + outputs={"Out": [topk_out], + "Indices": [topk_indices]}, + attrs={"k": k}) + acc_out_dtype = kwargs.get("out_dtype", "float32") + acc_out = helper.create_tmp_variable(dtype=acc_out_dtype) + helper.append_op( + type="accuracy", + inputs={ + "Out": [topk_out], + "Indices": [topk_indices], + "Label": [label] + }, + outputs={"Accuracy": [acc_out]}) diff --git a/python/paddle/v2/framework/framework.py b/python/paddle/v2/framework/framework.py index 7da6f81359b07e0b73736124d6eec7a0f3fd3036..548f04aa4499f1d40aee4130da4706b148075c0d 100644 --- a/python/paddle/v2/framework/framework.py +++ b/python/paddle/v2/framework/framework.py @@ -530,7 +530,7 @@ class Parameter(Variable): raise ValueError("Parameter shape should not be related with " "batch-size") - Variable.__init__( + super(Parameter, self).__init__( self, block, persistable=True, shape=shape, dtype=dtype, **kwargs) self.trainable = kwargs.get('trainable', True)