From b02bc6cac8e544cbcce16ab723a955c89b075a9c Mon Sep 17 00:00:00 2001 From: malin10 Date: Mon, 27 Jul 2020 17:58:42 +0800 Subject: [PATCH] update metrics --- core/metrics/binary_class/auc.py | 11 ++ core/metrics/binary_class/precision_recall.py | 20 +++- core/metrics/pairwise_pn.py | 11 ++ core/metrics/recall_k.py | 102 ++++++++---------- tests/test_auc_metrics.py | 30 ++++-- tests/test_pairwise_pn.py | 18 +++- tests/test_precision_recall_metrics.py | 9 ++ tests/test_recall_k.py | 6 ++ 8 files changed, 131 insertions(+), 76 deletions(-) diff --git a/core/metrics/binary_class/auc.py b/core/metrics/binary_class/auc.py index 5d2fa6fc..9c111710 100755 --- a/core/metrics/binary_class/auc.py +++ b/core/metrics/binary_class/auc.py @@ -21,6 +21,7 @@ from paddlerec.core.metric import Metric from paddle.fluid.layers import nn, accuracy from paddle.fluid.initializer import Constant from paddle.fluid.layer_helper import LayerHelper +from paddle.fluid.layers.tensor import Variable class AUC(Metric): @@ -30,12 +31,22 @@ class AUC(Metric): def __init__(self, **kwargs): """ """ + if "input" not in kwargs or "label" not in kwargs: + raise ValueError("AUC expect input and label as inputs.") predict = kwargs.get("input") label = kwargs.get("label") curve = kwargs.get("curve", 'ROC') num_thresholds = kwargs.get("num_thresholds", 2**12 - 1) topk = kwargs.get("topk", 1) slide_steps = kwargs.get("slide_steps", 1) + + if not isinstance(predict, Variable): + raise ValueError("input must be Variable, but received %s" % + type(predict)) + if not isinstance(label, Variable): + raise ValueError("label must be Variable, but received %s" % + type(label)) + auc_out, batch_auc_out, [ batch_stat_pos, batch_stat_neg, stat_pos, stat_neg ] = fluid.layers.auc(predict, diff --git a/core/metrics/binary_class/precision_recall.py b/core/metrics/binary_class/precision_recall.py index 50438cb7..e09c26da 100755 --- a/core/metrics/binary_class/precision_recall.py +++ b/core/metrics/binary_class/precision_recall.py @@ -21,6 +21,7 @@ from paddlerec.core.metric import Metric from paddle.fluid.layers import nn, accuracy from paddle.fluid.initializer import Constant from paddle.fluid.layer_helper import LayerHelper +from paddle.fluid.layers.tensor import Variable class PrecisionRecall(Metric): @@ -30,12 +31,23 @@ class PrecisionRecall(Metric): def __init__(self, **kwargs): """ """ - helper = LayerHelper("PaddleRec_PrecisionRecall", **kwargs) + if "input" not in kwargs or "label" not in kwargs or "class_num" not in kwargs: + raise ValueError( + "PrecisionRecall expect input, label and class_num as inputs.") predict = kwargs.get("input") - origin_label = kwargs.get("label") - label = fluid.layers.cast(origin_label, dtype="int32") - label.stop_gradient = True + label = kwargs.get("label") num_cls = kwargs.get("class_num") + + if not isinstance(predict, Variable): + raise ValueError("input must be Variable, but received %s" % + type(predict)) + if not isinstance(label, Variable): + raise ValueError("label must be Variable, but received %s" % + type(label)) + + helper = LayerHelper("PaddleRec_PrecisionRecall", **kwargs) + label = fluid.layers.cast(label, dtype="int32") + label.stop_gradient = True max_probs, indices = fluid.layers.nn.topk(predict, k=1) indices = fluid.layers.cast(indices, dtype="int32") indices.stop_gradient = True diff --git a/core/metrics/pairwise_pn.py b/core/metrics/pairwise_pn.py index c3cc0d48..176bbad6 100755 --- a/core/metrics/pairwise_pn.py +++ b/core/metrics/pairwise_pn.py @@ -21,6 +21,7 @@ from paddlerec.core.metric import Metric from paddle.fluid.layers import nn, accuracy from paddle.fluid.initializer import Constant from paddle.fluid.layer_helper import LayerHelper +from paddle.fluid.layers.tensor import Variable class PosNegRatio(Metric): @@ -31,9 +32,19 @@ class PosNegRatio(Metric): def __init__(self, **kwargs): """ """ helper = LayerHelper("PaddleRec_PosNegRatio", **kwargs) + if "pos_score" not in kwargs or "neg_score" not in kwargs: + raise ValueError( + "PosNegRatio expect pos_score and neg_score as inputs.") pos_score = kwargs.get('pos_score') neg_score = kwargs.get('neg_score') + if not isinstance(pos_score, Variable): + raise ValueError("pos_score must be Variable, but received %s" % + type(pos_score)) + if not isinstance(neg_score, Variable): + raise ValueError("neg_score must be Variable, but received %s" % + type(neg_score)) + wrong = fluid.layers.cast( fluid.layers.less_equal(pos_score, neg_score), dtype='float32') wrong_cnt = fluid.layers.reduce_sum(wrong) diff --git a/core/metrics/recall_k.py b/core/metrics/recall_k.py index 7a8fcc71..be0372bb 100755 --- a/core/metrics/recall_k.py +++ b/core/metrics/recall_k.py @@ -21,6 +21,7 @@ from paddlerec.core.metric import Metric from paddle.fluid.layers import nn, accuracy from paddle.fluid.initializer import Constant from paddle.fluid.layer_helper import LayerHelper +from paddle.fluid.layers.tensor import Variable class RecallK(Metric): @@ -30,71 +31,58 @@ class RecallK(Metric): def __init__(self, **kwargs): """ """ + if "input" not in kwargs or "label" not in kwargs: + raise ValueError("RecallK expect input and label as inputs.") + predict = kwargs.get('input') + label = kwargs.get('label') + k = kwargs.get("k", 20) + + if not isinstance(predict, Variable): + raise ValueError("input must be Variable, but received %s" % + type(predict)) + if not isinstance(label, Variable): + raise ValueError("label must be Variable, but received %s" % + type(label)) + helper = LayerHelper("PaddleRec_RecallK", **kwargs) - predict = kwargs.get("input") - origin_label = kwargs.get("label") - label = fluid.layers.cast(origin_label, dtype="int32") - label.stop_gradient = True - num_cls = kwargs.get("class_num") - max_probs, indices = fluid.layers.nn.topk(predict, k=1) - indices = fluid.layers.cast(indices, dtype="int32") - indices.stop_gradient = True - - states_info, _ = helper.create_or_get_global_variable( - name="states_info", - persistable=True, - dtype='float32', - shape=[num_cls, 4]) - states_info.stop_gradient = True - - helper.set_variable_initializer( - states_info, Constant( - value=0.0, force_cpu=True)) - - batch_metrics, _ = helper.create_or_get_global_variable( - name="batch_metrics", - persistable=False, - dtype='float32', - shape=[6]) - accum_metrics, _ = helper.create_or_get_global_variable( - name="global_metrics", - persistable=False, - dtype='float32', - shape=[6]) - - batch_states = fluid.layers.fill_constant( - shape=[num_cls, 4], value=0.0, dtype="float32") - batch_states.stop_gradient = True + batch_accuracy = accuracy(predict, label, k) + global_ins_cnt, _ = helper.create_or_get_global_variable( + name="ins_cnt", persistable=True, dtype='float32', shape=[1]) + global_pos_cnt, _ = helper.create_or_get_global_variable( + name="pos_cnt", persistable=True, dtype='float32', shape=[1]) + + for var in [global_ins_cnt, global_pos_cnt]: + helper.set_variable_initializer( + var, Constant( + value=0.0, force_cpu=True)) + + tmp_ones = fluid.layers.fill_constant( + shape=fluid.layers.shape(label), dtype="float32", value=1.0) + batch_ins = fluid.layers.reduce_sum(tmp_ones) + batch_pos = batch_ins * batch_accuracy helper.append_op( - type="precision_recall", - attrs={'class_number': num_cls}, - inputs={ - 'MaxProbs': [max_probs], - 'Indices': [indices], - 'Labels': [label], - 'StatesInfo': [states_info] - }, - outputs={ - 'BatchMetrics': [batch_metrics], - 'AccumMetrics': [accum_metrics], - 'AccumStatesInfo': [batch_states] - }) + type="elementwise_add", + inputs={"X": [global_ins_cnt], + "Y": [batch_ins]}, + outputs={"Out": [global_ins_cnt]}) + helper.append_op( - type="assign", - inputs={'X': [batch_states]}, - outputs={'Out': [states_info]}) + type="elementwise_add", + inputs={"X": [global_pos_cnt], + "Y": [batch_pos]}, + outputs={"Out": [global_pos_cnt]}) - batch_states.stop_gradient = True - states_info.stop_gradient = True + self.acc = global_pos_cnt / global_ins_cnt - self._need_clear_list = [("states_info", "float32")] + self._need_clear_list = [("ins_cnt", "float32"), + ("pos_cnt", "float32")] + metric_name = "Recall@%d_ACC" % k self.metrics = dict() - self.metrics["precision_recall_f1"] = accum_metrics - self.metrics["accum_states"] = states_info - - # self.metrics["batch_metrics"] = batch_metrics + self.metrics["ins_cnt"] = global_ins_cnt + self.metrics["pos_cnt"] = global_pos_cnt + self.metrics[metric_name] = self.acc def get_result(self): return self.metrics diff --git a/tests/test_auc_metrics.py b/tests/test_auc_metrics.py index e6623f40..0fc55233 100644 --- a/tests/test_auc_metrics.py +++ b/tests/test_auc_metrics.py @@ -25,10 +25,14 @@ class TestAUC(unittest.TestCase): def setUp(self): self.ins_num = 64 self.batch_nums = 3 - self.probs = np.random.uniform(0, 1.0, - (self.ins_num, 2)).astype('float32') - self.labels = np.random.choice(range(2), self.ins_num).reshape( - (self.ins_num, 1)).astype('int64') + + self.datas = [] + for i in range(self.batch_nums): + probs = np.random.uniform(0, 1.0, + (self.ins_num, 2)).astype('float32') + labels = np.random.choice(range(2), self.ins_num).reshape( + (self.ins_num, 1)).astype('int64') + self.datas.append((probs, labels)) self.place = fluid.core.CPUPlace() @@ -37,7 +41,7 @@ class TestAUC(unittest.TestCase): curve='ROC', num_thresholds=self.num_thresholds) for i in range(self.batch_nums): - python_auc.update(self.probs, self.labels) + python_auc.update(self.datas[i][0], self.datas[i][1]) self.auc = np.array(python_auc.eval()) @@ -65,15 +69,21 @@ class TestAUC(unittest.TestCase): exe = fluid.Executor(self.place) exe.run(fluid.default_startup_program()) for i in range(self.batch_nums): - outs = exe.run(fluid.default_main_program(), - feed={'predict': self.probs, - 'label': self.labels}, - fetch_list=fetch_vars, - return_numpy=True) + outs = exe.run( + fluid.default_main_program(), + feed={'predict': self.datas[i][0], + 'label': self.datas[i][1]}, + fetch_list=fetch_vars, + return_numpy=True) outs = dict(zip(metric_keys, outs)) self.assertTrue(np.allclose(outs['AUC'], self.auc)) + def test_exception(self): + self.assertRaises(Exception, AUC) + self.assertRaises( + Exception, AUC, input=self.datas[0][0], label=self.datas[0][1]), + if __name__ == '__main__': unittest.main() diff --git a/tests/test_pairwise_pn.py b/tests/test_pairwise_pn.py index 96ced95d..3c8346e3 100644 --- a/tests/test_pairwise_pn.py +++ b/tests/test_pairwise_pn.py @@ -21,12 +21,12 @@ import paddle import paddle.fluid as fluid -class TestAUC(unittest.TestCase): +class TestPosNegRatio(unittest.TestCase): def setUp(self): self.ins_num = 64 self.batch_nums = 3 - self.probs = [] + self.datas = [] self.right_cnt = 0.0 self.wrong_cnt = 0.0 for i in range(self.batch_nums): @@ -40,7 +40,7 @@ class TestAUC(unittest.TestCase): 'int32') self.right_cnt += float(right_cnt) self.wrong_cnt += float(wrong_cnt) - self.probs.append((pos_score, neg_score)) + self.datas.append((pos_score, neg_score)) self.place = fluid.core.CPUPlace() @@ -68,8 +68,8 @@ class TestAUC(unittest.TestCase): for i in range(self.batch_nums): outs = exe.run(fluid.default_main_program(), feed={ - 'pos_score': self.probs[i][0], - 'neg_score': self.probs[i][1] + 'pos_score': self.datas[i][0], + 'neg_score': self.datas[i][1] }, fetch_list=fetch_vars, return_numpy=True) @@ -82,6 +82,14 @@ class TestAUC(unittest.TestCase): np.array((self.right_cnt + 1.0) / (self.wrong_cnt + 1.0 )))) + def test_exception(self): + self.assertRaises(Exception, PosNegRatio) + self.assertRaises( + Exception, + PosNegRatio, + pos_score=self.datas[0][0], + neg_score=self.datas[0][1]), + if __name__ == '__main__': unittest.main() diff --git a/tests/test_precision_recall_metrics.py b/tests/test_precision_recall_metrics.py index 73856227..3e302bf9 100644 --- a/tests/test_precision_recall_metrics.py +++ b/tests/test_precision_recall_metrics.py @@ -148,6 +148,15 @@ class TestPrecisionRecall(unittest.TestCase): self.assertTrue(np.allclose(outs['accum_states'], self.states)) self.assertTrue(np.allclose(outs['precision_recall_f1'], self.metrics)) + def test_exception(self): + self.assertRaises(Exception, PrecisionRecall) + self.assertRaises( + Exception, + PrecisionRecall, + input=self.datas[0][0], + label=self.datas[0][1], + class_num=self.cls_num) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_recall_k.py b/tests/test_recall_k.py index b2b92312..8d936798 100644 --- a/tests/test_recall_k.py +++ b/tests/test_recall_k.py @@ -85,6 +85,12 @@ class TestRecallK(unittest.TestCase): np.array(self.match_num / (self.ins_num * self.batch_nums)))) + def test_exception(self): + self.assertRaises(Exception, RecallK) + self.assertRaises( + Exception, RecallK, input=self.datas[0][0], + label=self.datas[0][1]), + if __name__ == '__main__': unittest.main() -- GitLab