_ce.py

####this file is only used for continuous evaluation test!

import os
import sys
sys.path.append(os.environ['ceroot'])
from kpi import CostKpi, DurationKpi, AccKpi

#### NOTE kpi.py should shared in models in some way!!!!

train_acc_top1_kpi = AccKpi('train_acc_top1', 0.05, 0, desc='TOP1 ACC')
train_acc_top5_kpi = AccKpi(
    'train_acc_top5', 0.05, 0, actived=False, desc='TOP5 ACC')
train_cost_kpi = CostKpi('train_cost', 0.5, 0, actived=True, desc='train cost')
test_acc_top1_kpi = AccKpi('test_acc_top1', 0.05, 0, desc='TOP1 ACC')
test_acc_top5_kpi = AccKpi(
    'test_acc_top5', 0.05, 0, actived=False, desc='TOP5 ACC')
test_cost_kpi = CostKpi('test_cost', 1.0, 0, actived=True, desc='train cost')
train_speed_kpi = AccKpi(
    'train_speed',
    0.5,
    0,
    actived=True,
    unit_repr='seconds/image',
    desc='train speed in one GPU card')
train_acc_top1_card4_kpi = AccKpi(
    'train_acc_top1_card4', 0.05, 0, desc='TOP1 ACC')
train_acc_top5_card4_kpi = AccKpi(
    'train_acc_top5_card4', 0.05, 0, actived=False, desc='TOP5 ACC')
train_cost_card4_kpi = CostKpi(
    'train_cost_kpi', 0.3, 0, actived=True, desc='train cost')
test_acc_top1_card4_kpi = AccKpi(
    'test_acc_top1_card4', 0.05, 0, desc='TOP1 ACC')
test_acc_top5_card4_kpi = AccKpi(
    'test_acc_top5_card4', 0.05, 0, actived=False, desc='TOP5 ACC')
test_cost_card4_kpi = CostKpi(
    'test_cost_card4', 1.0, 0, actived=True, desc='train cost')
train_speed_card4_kpi = AccKpi(
    'train_speed_card4',
    0.5,
    0,
    actived=True,
    unit_repr='seconds/image',
    desc='train speed in four GPU card')
tracking_kpis = [
    train_acc_top1_kpi, train_acc_top5_kpi, train_cost_kpi, test_acc_top1_kpi,
    test_acc_top5_kpi, test_cost_kpi, train_speed_kpi, train_acc_top1_card4_kpi,
    train_acc_top5_card4_kpi, train_cost_card4_kpi, test_acc_top1_card4_kpi,
    test_acc_top5_card4_kpi, test_cost_card4_kpi, train_speed_card4_kpi
]


def parse_log(log):
    '''
    This method should be implemented by model developers.

    The suggestion:

    each line in the log should be key, value, for example:

    "
    train_cost\t1.0
    test_cost\t1.0
    train_cost\t1.0
    train_cost\t1.0
    train_acc\t1.2
    "
    '''
    for line in log.split('\n'):
        fs = line.strip().split('\t')
        print(fs)
        if len(fs) == 3 and fs[0] == 'kpis':
            print("-----%s" % fs)
            kpi_name = fs[1]
            kpi_value = float(fs[2])
            yield kpi_name, kpi_value


def log_to_ce(log):
    kpi_tracker = {}
    for kpi in tracking_kpis:
        kpi_tracker[kpi.name] = kpi

    for (kpi_name, kpi_value) in parse_log(log):
        print(kpi_name, kpi_value)
        kpi_tracker[kpi_name].add_record(kpi_value)
        kpi_tracker[kpi_name].persist()


if __name__ == '__main__':
    log = sys.stdin.read()
    print("*****")
    print(log)
    print("****")
    log_to_ce(log)