diff --git a/doc/design/evaluator.md b/doc/design/evaluator.md index 771cb4d5f73b7234d810a38d97e0758d5a54a0dc..f43bad18390641f872f31c8a891b000270d7e156 100644 --- a/doc/design/evaluator.md +++ b/doc/design/evaluator.md @@ -15,35 +15,44 @@ Currently, every operation is expressed in the graph. we divide the evaluator pr 3. Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. When it comes to distributed training/Multi-GPU training, aggregate the value from different devices. ### Implementation -This design is shown in python API. There would be an abstract python interface and multiple inheritances for each evaluation method. +This design is shown in python API. +Each metric operator need to caculate the metric statistic and return the batch aware states, Python side responsible for accumulate the states for each pass. + ```python class Evaluator(object): """ Evaluator Base class. """ - def __init__(self): + def __init__(self, name, **kwargs): """ Different evaluator may has different metric states. E.g, Accuracy need two variables, total and right sample counts. Auc need four variables, `true_positives`, - `true_negatives`, `false_positives` and `false_negatives`. So every evaluator should create its needed variables and append the related mini-batch operator to main_program + `true_negatives`, `false_positives` and `false_negatives`. So every evaluator should create its needed variables and append to main_program The initialization of Evaluator should be responsible for: create metric states and append to the main_program - add mini-batch evaluator caculate operators to the main_program - add increment operator to accumulate the metric states """ pass - def clear(self): + def _update_ops(self, input, label, **kwargs) + """ + Add mini-batch evaluator caculate operators to the main_program. + Add increment operator to accumulate the metric states. + """ + + + def reset(self, executor, program=None): """ - clear metric states at the begin of each pass/user specified batch + Reset metric states at the begin of each pass/user specified batch number. + Execute the reset_program to reset the states. """ - return init_program + - def evaluate(self): + def eval(self, executor, program=None): """ Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. + Execute the eval_program and return the result. """ - return eval_program + return eval_result ``` diff --git a/python/paddle/v2/framework/evaluator.py b/python/paddle/v2/framework/evaluator.py index 252370ffde2b0c796fb115a2394eaf27bd31ca7d..664f65422c642c8061269ccec7e4f86d67a1e385 100644 --- a/python/paddle/v2/framework/evaluator.py +++ b/python/paddle/v2/framework/evaluator.py @@ -181,43 +181,6 @@ class Accuracy(Evaluator): return np.array(out[0]) -# Demo for composing low level ops to compute the F1 metric -class FScore(Evaluator): - def __init__(self, input, label, beta=1.0, **kwargs): - super(F1, self).__init__("FScore", **kwargs) - block = self._program.global_block() - g_tp = block.create_var( - name=unique_name("Tp"), persistable=True, dtype="int64", shape=[1]) - g_fn = block.create_var( - name=unique_name("Fn"), persistable=True, dtype="int64", shape=[1]) - g_fp = block.create_var( - name=unique_name("Fp"), persistable=True, dtype="int64", shape=[1]) - - self._states["Tp"] = g_tp - self._states["Fp"] = g_fp - self._states["Fn"] = g_fn - - def _update_ops(self): - block = self._program.global_block() - equal_out = block.create_var() - block.append_op( - type="equal", - inputs={"X": [input], - "Y": [label]}, - outputs={"Out": equal_out}) - - positive = block.create_var() - block.append_op( - type="sequence_pool", - inputs={"X": [equal_out]}, - outputs={"Out": positive}, - attrs={"pooltype": "SUM"}) - batch = block.create_var( - name=feed_var_name, - type=core.VarDesc.VarType.FEED_MINIBATCH, - persistable=True) - - # FIXME(dzh): add a decorator to call _update_ops automatically def accuracy(*args, **kwargs): cls = Accuracy(*args, **kwargs)