# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Evaluation """ import sys import six import numpy as np def evaluate_ubuntu(file_path): """ Evaluate on ubuntu data """ def get_p_at_n_in_m(data, n, m, ind): """ Recall n at m """ pos_score = data[ind][0] curr = data[ind:ind + m] curr = sorted(curr, key=lambda x: x[0], reverse=True) if curr[n - 1][0] <= pos_score: return 1 return 0 data = [] with open(file_path, 'r') as file: for line in file: line = line.strip() tokens = line.split("\t") if len(tokens) != 2: continue data.append((float(tokens[0]), int(tokens[1]))) #assert len(data) % 10 == 0 p_at_1_in_2 = 0.0 p_at_1_in_10 = 0.0 p_at_2_in_10 = 0.0 p_at_5_in_10 = 0.0 length = len(data) // 10 for i in six.moves.xrange(0, length): ind = i * 10 assert data[ind][1] == 1 p_at_1_in_2 += get_p_at_n_in_m(data, 1, 2, ind) p_at_1_in_10 += get_p_at_n_in_m(data, 1, 10, ind) p_at_2_in_10 += get_p_at_n_in_m(data, 2, 10, ind) p_at_5_in_10 += get_p_at_n_in_m(data, 5, 10, ind) result_dict = { "1_in_2": p_at_1_in_2 / length, "1_in_10": p_at_1_in_10 / length, "2_in_10": p_at_2_in_10 / length, "5_in_10": p_at_5_in_10 / length } return result_dict def evaluate_douban(file_path): """ Evaluate douban data """ def mean_average_precision(sort_data): """ Evaluate mean average precision """ count_1 = 0 sum_precision = 0 for index in six.moves.xrange(len(sort_data)): if sort_data[index][1] == 1: count_1 += 1 sum_precision += 1.0 * count_1 / (index + 1) return sum_precision / count_1 def mean_reciprocal_rank(sort_data): """ Evaluate MRR """ sort_lable = [s_d[1] for s_d in sort_data] assert 1 in sort_lable return 1.0 / (1 + sort_lable.index(1)) def precision_at_position_1(sort_data): """ Evaluate precision """ if sort_data[0][1] == 1: return 1 else: return 0 def recall_at_position_k_in_10(sort_data, k): """" Evaluate recall """ sort_lable = [s_d[1] for s_d in sort_data] select_lable = sort_lable[:k] return 1.0 * select_lable.count(1) / sort_lable.count(1) def evaluation_one_session(data): """ Evaluate one session """ sort_data = sorted(data, key=lambda x: x[0], reverse=True) m_a_p = mean_average_precision(sort_data) m_r_r = mean_reciprocal_rank(sort_data) p_1 = precision_at_position_1(sort_data) r_1 = recall_at_position_k_in_10(sort_data, 1) r_2 = recall_at_position_k_in_10(sort_data, 2) r_5 = recall_at_position_k_in_10(sort_data, 5) return m_a_p, m_r_r, p_1, r_1, r_2, r_5 sum_m_a_p = 0 sum_m_r_r = 0 sum_p_1 = 0 sum_r_1 = 0 sum_r_2 = 0 sum_r_5 = 0 i = 0 total_num = 0 with open(file_path, 'r') as infile: for line in infile: if i % 10 == 0: data = [] tokens = line.strip().split('\t') data.append((float(tokens[0]), int(tokens[1]))) if i % 10 == 9: total_num += 1 m_a_p, m_r_r, p_1, r_1, r_2, r_5 = evaluation_one_session(data) sum_m_a_p += m_a_p sum_m_r_r += m_r_r sum_p_1 += p_1 sum_r_1 += r_1 sum_r_2 += r_2 sum_r_5 += r_5 i += 1 result_dict = { "MAP": 1.0 * sum_m_a_p / total_num, "MRR": 1.0 * sum_m_r_r / total_num, "P_1": 1.0 * sum_p_1 / total_num, "1_in_10": 1.0 * sum_r_1 / total_num, "2_in_10": 1.0 * sum_r_2 / total_num, "5_in_10": 1.0 * sum_r_5 / total_num } return result_dict