# this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi imikolov_20_avg_ppl_kpi = CostKpi('lstm_language_model_loss', 0.02, 0) imikolov_20_pass_duration_kpi = DurationKpi( 'lstm_language_model_duration', 0.02, 0, actived=True) tracking_kpis = [ imikolov_20_avg_ppl_kpi, imikolov_20_pass_duration_kpi, ] def parse_log(log): ''' This method should be implemented by model developers. The suggestion: each line in the log should be key, value, for example: " train_cost\t1.0 test_cost\t1.0 train_cost\t1.0 train_cost\t1.0 train_acc\t1.2 " ''' for line in log.split('\n'): fs = line.strip().split('\t') print(fs) kpi_name = fs[0] kpi_value = float(fs[1]) yield kpi_name, kpi_value def log_to_ce(log): kpi_tracker = {} for kpi in tracking_kpis: kpi_tracker[kpi.name] = kpi for (kpi_name, kpi_value) in parse_log(log): print(kpi_name, kpi_value) kpi_tracker[kpi_name].add_record(kpi_value) kpi_tracker[kpi_name].persist() if __name__ == '__main__': log = sys.stdin.read() log_to_ce(log)