From e9abf856ef391eae335fa83351349c03786ffcd9 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 22 May 2019 11:01:56 +0000 Subject: [PATCH] add 02.recognize_digits ce files --- 02.recognize_digits/.run_ce.sh | 4 +++ 02.recognize_digits/_ce.py | 37 ++++++++++++++++++++++++ 02.recognize_digits/train.py | 53 +++++++++++++++++++++++----------- 3 files changed, 77 insertions(+), 17 deletions(-) create mode 100644 02.recognize_digits/.run_ce.sh create mode 100644 02.recognize_digits/_ce.py diff --git a/02.recognize_digits/.run_ce.sh b/02.recognize_digits/.run_ce.sh new file mode 100644 index 0000000..4c5ae21 --- /dev/null +++ b/02.recognize_digits/.run_ce.sh @@ -0,0 +1,4 @@ +#!/bin/bash +#This file is only used for continuous evaluation. +python train.py --enable_ce | python _ce.py + diff --git a/02.recognize_digits/_ce.py b/02.recognize_digits/_ce.py new file mode 100644 index 0000000..40491a5 --- /dev/null +++ b/02.recognize_digits/_ce.py @@ -0,0 +1,37 @@ +### This file is only used for continuous evaluation test! +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +import os +import sys +sys.path.append(os.environ['ceroot']) +from kpi import CostKpi +from kpi import AccKpi + +train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=True, desc='train cost') +test_cost_kpi = CostKpi('test_cost', 0.02, 0, actived=True, desc='test cost') +test_acc_kpi= AccKpi('test_acc', 0.02, 0, actived=True, desc='test acc') +tracking_kpis=[train_cost_kpi, test_cost_kpi, test_acc_kpi] + +def parse_log(log): + for line in log.split('\n'): + fs = line.strip().split('\t') + print(fs) + if len(fs) == 3 and fs[0] =='kpis': + kpi_name = fs[1] + kpi_value = float(fs[2]) + yield kpi_name,kpi_value + +def log_to_ce(log): + kpi_tracker = {} + for kpi in tracking_kpis: + kpi_tracker[kpi.name]=kpi + for (kpi_name, kpi_value) in parse_log(log): + print(kpi_name,kpi_value) + kpi_tracker[kpi_name].add_record(kpi_value) + kpi_tracker[kpi_name].persist() + +if __name__ == '__main__': + log = sys.stdin.read() + log_to_ce(log) + diff --git a/02.recognize_digits/train.py b/02.recognize_digits/train.py index 5525845..da59918 100644 --- a/02.recognize_digits/train.py +++ b/02.recognize_digits/train.py @@ -15,14 +15,19 @@ from __future__ import print_function import os +import argparse from PIL import Image import numpy import paddle import paddle.fluid as fluid -BATCH_SIZE = 64 -PASS_NUM = 5 - +def parse_args(): + parser = argparse.ArgumentParser("mnist") + parser.add_argument('--enable_ce', action='store_true', help="If set, run the task with continuous evaluation logs.") + parser.add_argument('--use_gpu', type=bool, default=False, help="Whether to use GPU or not.") + parser.add_argument('--num_epochs', type=int, default=5, help="number of epochs.") + args=parser.parse_args() + return args def loss_net(hidden, label): prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') @@ -68,6 +73,20 @@ def train(nn_type, params_filename=None): if use_cuda and not fluid.core.is_compiled_with_cuda(): return + + startup_program = fluid.default_startup_program() + main_program = fluid.default_main_program() + + if args.enable_ce: + train_reader = paddle.batch(paddle.dataset.mnist.train(), batch_size=BATCH_SIZE) + test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=BATCH_SIZE) + startup_program.random_seed = 90 + main_program.random_seed = 90 + else : + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), batch_size=BATCH_SIZE) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=BATCH_SIZE) img = fluid.layers.data(name='img', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') @@ -81,8 +100,7 @@ def train(nn_type, prediction, avg_loss, acc = net_conf(img, label) - test_program = fluid.default_main_program().clone(for_test=True) - + test_program = main_program.clone(for_test=True) optimizer = fluid.optimizer.Adam(learning_rate=0.001) optimizer.minimize(avg_loss) @@ -104,16 +122,9 @@ def train(nn_type, place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) - - train_reader = paddle.batch( - paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), - batch_size=BATCH_SIZE) - test_reader = paddle.batch( - paddle.dataset.mnist.test(), batch_size=BATCH_SIZE) + feeder = fluid.DataFeeder(feed_list=[img, label], place=place) - - exe.run(fluid.default_startup_program()) - main_program = fluid.default_main_program() + exe.run(startup_program) epochs = [epoch_id for epoch_id in range(PASS_NUM)] lists = [] @@ -143,12 +154,17 @@ def train(nn_type, exe, model_filename=model_filename, params_filename=params_filename) - + + if args.enable_ce: + print("kpis\ttrain_cost\t%f" % metrics[0] ) + print("kpis\ttest_cost\t%s" % avg_loss_val) + print("kpis\ttest_acc\t%s" % acc_val) + # find the best pass best = sorted(lists, key=lambda list: float(list[1]))[0] print('Best pass is %s, testing Avgcost is %s' % (best[0], best[1])) print('The classification accuracy is %.2f%%' % (float(best[2]) * 100)) - + def infer(use_cuda, save_dirname=None, @@ -210,7 +226,10 @@ def main(use_cuda, nn_type): if __name__ == '__main__': - use_cuda = False + args = parse_args() + BATCH_SIZE = 64 + PASS_NUM = args.num_epochs + use_cuda = args.use_gpu # predict = 'softmax_regression' # uncomment for Softmax # predict = 'multilayer_perceptron' # uncomment for MLP predict = 'convolutional_neural_network' # uncomment for LeNet5 -- GitLab