# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """this file is only used for continuous evaluation test!""" import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi from kpi import AccKpi each_step_duration_atis_slot_card1 = DurationKpi('each_step_duration_atis_slot_card1', 0.01, 0, actived=True) train_loss_atis_slot_card1 = CostKpi('train_loss_atis_slot_card1', 0.08, 0, actived=True) train_acc_atis_slot_card1 = CostKpi('train_acc_atis_slot_card1', 0.01, 0, actived=True) each_step_duration_atis_slot_card4 = DurationKpi('each_step_duration_atis_slot_card4', 0.06, 0, actived=True) train_loss_atis_slot_card4 = CostKpi('train_loss_atis_slot_card4', 0.03, 0, actived=True) train_acc_atis_slot_card4 = CostKpi('train_acc_atis_slot_card4', 0.01, 0, actived=True) tracking_kpis = [ each_step_duration_atis_slot_card1, train_loss_atis_slot_card1, train_acc_atis_slot_card1, each_step_duration_atis_slot_card4, train_loss_atis_slot_card4, train_acc_atis_slot_card4, ] def parse_log(log): ''' This method should be implemented by model developers. The suggestion: each line in the log should be key, value, for example: " train_cost\t1.0 test_cost\t1.0 train_cost\t1.0 train_cost\t1.0 train_acc\t1.2 " ''' for line in log.split('\n'): fs = line.strip().split('\t') print(fs) if len(fs) == 3 and fs[0] == 'kpis': kpi_name = fs[1] kpi_value = float(fs[2]) yield kpi_name, kpi_value def log_to_ce(log): kpi_tracker = {} for kpi in tracking_kpis: kpi_tracker[kpi.name] = kpi for (kpi_name, kpi_value) in parse_log(log): print(kpi_name, kpi_value) kpi_tracker[kpi_name].add_record(kpi_value) kpi_tracker[kpi_name].persist() if __name__ == '__main__': log = sys.stdin.read() log_to_ce(log)