# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import yaml import paddle.fluid as fluid import logging from paddle.distributed.utils import get_logger __all__ = [] logger = get_logger(logging.INFO, name="metrics") # read metric config from yaml and init MetricMsg in fleet_wrapper def init_metric(metric_ptr, metric_yaml_path, cmatch_rank_var="", mask_var="", uid_var="", phase=-1, cmatch_rank_group="", ignore_rank=False, bucket_size=1000000): yaml_fobj = open(metric_yaml_path) if sys.version.startswith('2.7.13'): content = yaml.load(yaml_fobj) else: content = yaml.load(yaml_fobj, Loader=yaml.FullLoader) print("yaml metric config: \n") print(content) metric_runner_list = content['monitors'] if not metric_runner_list: metric_runner_list = [] for metric_runner in metric_runner_list: is_join = metric_runner['phase'] == 'JOINING' phase = 1 if is_join else 0 if metric_runner['method'] == 'AucCalculator': metric_ptr.init_metric( metric_runner['method'], metric_runner['name'], metric_runner['label'], metric_runner['target'], cmatch_rank_var, mask_var, uid_var, phase, cmatch_rank_group, ignore_rank, bucket_size) elif metric_runner['method'] == 'MultiTaskAucCalculator': metric_ptr.init_metric( metric_runner['method'], metric_runner['name'], metric_runner['label'], metric_runner['target'], metric_runner['cmatch_var'], mask_var, uid_var, phase, metric_runner['cmatch_group'], ignore_rank, bucket_size) elif metric_runner['method'] == 'CmatchRankAucCalculator': metric_ptr.init_metric( metric_runner['method'], metric_runner['name'], metric_runner['label'], metric_runner['target'], metric_runner['cmatch_var'], mask_var, uid_var, phase, metric_runner['cmatch_group'], metric_runner['ignore_rank'], bucket_size) elif metric_runner['method'] == 'MaskAucCalculator': metric_ptr.init_metric( metric_runner['method'], metric_runner['name'], metric_runner['label'], metric_runner['target'], cmatch_rank_var, metric_runner['mask'], uid_var, phase, cmatch_rank_group, ignore_rank, bucket_size) elif metric_runner['method'] == 'CmatchRankMaskAucCalculator': metric_ptr.init_metric( metric_runner['method'], metric_runner['name'], metric_runner['label'], metric_runner['target'], metric_runner['cmatch_var'], metric_runner['mask'], uid_var, phase, metric_runner['cmatch_group'], metric_runner['ignore_rank'], bucket_size) elif metric_runner['method'] == 'WuAucCalculator': metric_ptr.init_metric( metric_runner['method'], metric_runner['name'], metric_runner['label'], metric_runner['target'], cmatch_rank_var, mask_var, metric_runner['uid'], phase, cmatch_rank_group, ignore_rank, bucket_size) else: metric_ptr.init_metric( metric_runner['method'], metric_runner['name'], metric_runner['label'], metric_runner['target'], cmatch_rank_var, mask_var, phase, cmatch_rank_group, ignore_rank, bucket_size) def print_metric(metric_ptr, name): """ print the metric value. Print directly in back-end """ if name.find("wuauc") != -1: metric = metric_ptr.get_wuauc_metric_msg(name) monitor_msg = "%s: User Count=%.0f INS Count=%.0f UAUC=%.6f WUAUC=%.6f "\ % (name, metric[0], metric[1], metric[4], metric[5]) else: metric = metric_ptr.get_metric_msg(name) monitor_msg = "%s: AUC=%.6f BUCKET_ERROR=%.6f MAE=%.6f RMSE=%.6f "\ "Actual CTR=%.6f Predicted CTR=%.6f COPC=%.6f INS Count=%.0f"\ % (name, metric[0], metric[1], metric[2], metric[3], metric[4], metric[5], metric[6], metric[7]) # logger.info(monitor_msg) return monitor_msg def print_auc(metric_ptr, is_day, phase="all"): """ print metric according to stage and phase """ if is_day is True: stage = "day" stage_num = -1 else: stage = "pass" stage_num = 1 if phase == "join" else 0 metric_results = [] name_list = metric_ptr.get_metric_name_list(stage_num) if phase == "all": for name in name_list: if name.find(stage) != -1: metric_results.append(print_metric(metric_ptr, name=name)) else: for name in name_list: if name.find(stage) != -1 and name.find(phase) != -1: metric_results.append(print_metric(metric_ptr, name=name)) return metric_results