From cfbc92e6464cf91c11d7d0e36c002da2209eeb83 Mon Sep 17 00:00:00 2001 From: Dong Zhihong Date: Thu, 9 Nov 2017 15:14:36 -0800 Subject: [PATCH] "polish document" --- doc/design/evaluator.md | 29 ++++++++++++------- python/paddle/v2/framework/evaluator.py | 37 ------------------------- 2 files changed, 19 insertions(+), 47 deletions(-) diff --git a/doc/design/evaluator.md b/doc/design/evaluator.md index 771cb4d5f73..f43bad18390 100644 --- a/doc/design/evaluator.md +++ b/doc/design/evaluator.md @@ -15,35 +15,44 @@ Currently, every operation is expressed in the graph. we divide the evaluator pr 3. Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. When it comes to distributed training/Multi-GPU training, aggregate the value from different devices. ### Implementation -This design is shown in python API. There would be an abstract python interface and multiple inheritances for each evaluation method. +This design is shown in python API. +Each metric operator need to caculate the metric statistic and return the batch aware states, Python side responsible for accumulate the states for each pass. + ```python class Evaluator(object): """ Evaluator Base class. """ - def __init__(self): + def __init__(self, name, **kwargs): """ Different evaluator may has different metric states. E.g, Accuracy need two variables, total and right sample counts. Auc need four variables, `true_positives`, - `true_negatives`, `false_positives` and `false_negatives`. So every evaluator should create its needed variables and append the related mini-batch operator to main_program + `true_negatives`, `false_positives` and `false_negatives`. So every evaluator should create its needed variables and append to main_program The initialization of Evaluator should be responsible for: create metric states and append to the main_program - add mini-batch evaluator caculate operators to the main_program - add increment operator to accumulate the metric states """ pass - def clear(self): + def _update_ops(self, input, label, **kwargs) + """ + Add mini-batch evaluator caculate operators to the main_program. + Add increment operator to accumulate the metric states. + """ + + + def reset(self, executor, program=None): """ - clear metric states at the begin of each pass/user specified batch + Reset metric states at the begin of each pass/user specified batch number. + Execute the reset_program to reset the states. """ - return init_program + - def evaluate(self): + def eval(self, executor, program=None): """ Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. + Execute the eval_program and return the result. """ - return eval_program + return eval_result ``` diff --git a/python/paddle/v2/framework/evaluator.py b/python/paddle/v2/framework/evaluator.py index 252370ffde2..664f65422c6 100644 --- a/python/paddle/v2/framework/evaluator.py +++ b/python/paddle/v2/framework/evaluator.py @@ -181,43 +181,6 @@ class Accuracy(Evaluator): return np.array(out[0]) -# Demo for composing low level ops to compute the F1 metric -class FScore(Evaluator): - def __init__(self, input, label, beta=1.0, **kwargs): - super(F1, self).__init__("FScore", **kwargs) - block = self._program.global_block() - g_tp = block.create_var( - name=unique_name("Tp"), persistable=True, dtype="int64", shape=[1]) - g_fn = block.create_var( - name=unique_name("Fn"), persistable=True, dtype="int64", shape=[1]) - g_fp = block.create_var( - name=unique_name("Fp"), persistable=True, dtype="int64", shape=[1]) - - self._states["Tp"] = g_tp - self._states["Fp"] = g_fp - self._states["Fn"] = g_fn - - def _update_ops(self): - block = self._program.global_block() - equal_out = block.create_var() - block.append_op( - type="equal", - inputs={"X": [input], - "Y": [label]}, - outputs={"Out": equal_out}) - - positive = block.create_var() - block.append_op( - type="sequence_pool", - inputs={"X": [equal_out]}, - outputs={"Out": positive}, - attrs={"pooltype": "SUM"}) - batch = block.create_var( - name=feed_var_name, - type=core.VarDesc.VarType.FEED_MINIBATCH, - persistable=True) - - # FIXME(dzh): add a decorator to call _update_ops automatically def accuracy(*args, **kwargs): cls = Accuracy(*args, **kwargs) -- GitLab