提交 84c8bc00 编写于 作者: u010070587's avatar u010070587 提交者: Yibing Liu

add 04.word2vec ce (#716)

* add ce

* add ce

* add 02.recognize_digits ce files

* modify ce file code style of 01.fit_a_line and 02.recognize_digits

* add 04.word2vec ce
上级 ec27c8eb
#!/bin/bash
#This file is only used for continuous evaluation.
python train.py --enable_ce | python _ce.py
### This file is only used for continuous evaluation test!
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import os
import sys
sys.path.append(os.environ['ceroot'])
from kpi import CostKpi
train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=True, desc='train cost')
test_cost_kpi = CostKpi('test_cost', 0.02, 0, actived=True, desc='test cost')
tracking_kpis = [train_cost_kpi, test_cost_kpi]
def parse_log(log):
for line in log.split('\n'):
fs = line.strip().split('\t')
print(fs)
if len(fs) == 3 and fs[0] == 'kpis':
print("-----%s" % fs)
kpi_name = fs[1]
kpi_value = float(fs[2])
yield kpi_name, kpi_value
def log_to_ce(log):
kpi_tracker = {}
for kpi in tracking_kpis:
kpi_tracker[kpi.name] = kpi
for (kpi_name, kpi_value) in parse_log(log):
print(kpi_name, kpi_value)
kpi_tracker[kpi_name].add_record(kpi_value)
kpi_tracker[kpi_name].persist()
if __name__ == '__main__':
log = sys.stdin.read()
log_to_ce(log)
01.fit_a_line/image/ranges.png

8.6 KB | W: | H:

01.fit_a_line/image/ranges.png

6.6 KB | W: | H:

01.fit_a_line/image/ranges.png
01.fit_a_line/image/ranges.png
01.fit_a_line/image/ranges.png
01.fit_a_line/image/ranges.png
  • 2-up
  • Swipe
  • Onion skin
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
from __future__ import print_function from __future__ import print_function
import sys import sys
import argparse
import math import math
import numpy import numpy
...@@ -23,6 +24,23 @@ import paddle ...@@ -23,6 +24,23 @@ import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
def parse_args():
parser = argparse.ArgumentParser("fit_a_line")
parser.add_argument(
'--enable_ce',
action='store_true',
help="If set, run the task with continuous evaluation logs.")
parser.add_argument(
'--use_gpu',
type=bool,
default=False,
help="Whether to use GPU or not.")
parser.add_argument(
'--num_epochs', type=int, default=100, help="number of epochs.")
args = parser.parse_args()
return args
# For training test cost # For training test cost
def train_test(executor, program, reader, feeder, fetch_list): def train_test(executor, program, reader, feeder, fetch_list):
accumulated = 1 * [0] accumulated = 1 * [0]
...@@ -52,21 +70,34 @@ def save_result(points1, points2): ...@@ -52,21 +70,34 @@ def save_result(points1, points2):
def main(): def main():
batch_size = 20 batch_size = 20
train_reader = paddle.batch(
paddle.reader.shuffle(paddle.dataset.uci_housing.train(), buf_size=500), if args.enable_ce:
batch_size=batch_size) train_reader = paddle.batch(
test_reader = paddle.batch( paddle.dataset.uci_housing.train(), batch_size=batch_size)
paddle.reader.shuffle(paddle.dataset.uci_housing.test(), buf_size=500), test_reader = paddle.batch(
batch_size=batch_size) paddle.dataset.uci_housing.test(), batch_size=batch_size)
else:
train_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.uci_housing.train(), buf_size=500),
batch_size=batch_size)
test_reader = paddle.batch(
paddle.reader.shuffle(
paddle.dataset.uci_housing.test(), buf_size=500),
batch_size=batch_size)
# feature vector of length 13 # feature vector of length 13
x = fluid.layers.data(name='x', shape=[13], dtype='float32') x = fluid.layers.data(name='x', shape=[13], dtype='float32')
y = fluid.layers.data(name='y', shape=[1], dtype='float32') y = fluid.layers.data(name='y', shape=[1], dtype='float32')
y_predict = fluid.layers.fc(input=x, size=1, act=None)
main_program = fluid.default_main_program() main_program = fluid.default_main_program()
startup_program = fluid.default_startup_program() startup_program = fluid.default_startup_program()
if args.enable_ce:
main_program.random_seed = 90
startup_program.random_seed = 90
y_predict = fluid.layers.fc(input=x, size=1, act=None)
cost = fluid.layers.square_error_cost(input=y_predict, label=y) cost = fluid.layers.square_error_cost(input=y_predict, label=y)
avg_loss = fluid.layers.mean(cost) avg_loss = fluid.layers.mean(cost)
...@@ -76,13 +107,13 @@ def main(): ...@@ -76,13 +107,13 @@ def main():
test_program = main_program.clone(for_test=True) test_program = main_program.clone(for_test=True)
# can use CPU or GPU # can use CPU or GPU
use_cuda = False use_cuda = args.use_gpu
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = fluid.Executor(place) exe = fluid.Executor(place)
# Specify the directory to save the parameters # Specify the directory to save the parameters
params_dirname = "fit_a_line.inference.model" params_dirname = "fit_a_line.inference.model"
num_epochs = 100 num_epochs = args.num_epochs
# main train loop. # main train loop.
feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
...@@ -126,6 +157,10 @@ def main(): ...@@ -126,6 +157,10 @@ def main():
fluid.io.save_inference_model(params_dirname, ['x'], [y_predict], fluid.io.save_inference_model(params_dirname, ['x'], [y_predict],
exe) exe)
if args.enable_ce and pass_id == args.num_epochs - 1:
print("kpis\ttrain_cost\t%f" % avg_loss_value[0])
print("kpis\ttest_cost\t%f" % test_metics[0])
infer_exe = fluid.Executor(place) infer_exe = fluid.Executor(place)
inference_scope = fluid.core.Scope() inference_scope = fluid.core.Scope()
...@@ -162,4 +197,5 @@ def main(): ...@@ -162,4 +197,5 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
args = parse_args()
main() main()
#!/bin/bash
#This file is only used for continuous evaluation.
python train.py --enable_ce | python _ce.py
### This file is only used for continuous evaluation test!
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import os
import sys
sys.path.append(os.environ['ceroot'])
from kpi import CostKpi
from kpi import AccKpi
train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=True, desc='train cost')
test_cost_kpi = CostKpi('test_cost', 0.02, 0, actived=True, desc='test cost')
test_acc_kpi = AccKpi('test_acc', 0.02, 0, actived=True, desc='test acc')
tracking_kpis = [train_cost_kpi, test_cost_kpi, test_acc_kpi]
def parse_log(log):
for line in log.split('\n'):
fs = line.strip().split('\t')
print(fs)
if len(fs) == 3 and fs[0] == 'kpis':
kpi_name = fs[1]
kpi_value = float(fs[2])
yield kpi_name, kpi_value
def log_to_ce(log):
kpi_tracker = {}
for kpi in tracking_kpis:
kpi_tracker[kpi.name] = kpi
for (kpi_name, kpi_value) in parse_log(log):
print(kpi_name, kpi_value)
kpi_tracker[kpi_name].add_record(kpi_value)
kpi_tracker[kpi_name].persist()
if __name__ == '__main__':
log = sys.stdin.read()
log_to_ce(log)
...@@ -15,13 +15,28 @@ ...@@ -15,13 +15,28 @@
from __future__ import print_function from __future__ import print_function
import os import os
import argparse
from PIL import Image from PIL import Image
import numpy import numpy
import paddle import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
BATCH_SIZE = 64
PASS_NUM = 5 def parse_args():
parser = argparse.ArgumentParser("mnist")
parser.add_argument(
'--enable_ce',
action='store_true',
help="If set, run the task with continuous evaluation logs.")
parser.add_argument(
'--use_gpu',
type=bool,
default=False,
help="Whether to use GPU or not.")
parser.add_argument(
'--num_epochs', type=int, default=5, help="number of epochs.")
args = parser.parse_args()
return args
def loss_net(hidden, label): def loss_net(hidden, label):
...@@ -69,6 +84,23 @@ def train(nn_type, ...@@ -69,6 +84,23 @@ def train(nn_type,
if use_cuda and not fluid.core.is_compiled_with_cuda(): if use_cuda and not fluid.core.is_compiled_with_cuda():
return return
startup_program = fluid.default_startup_program()
main_program = fluid.default_main_program()
if args.enable_ce:
train_reader = paddle.batch(
paddle.dataset.mnist.train(), batch_size=BATCH_SIZE)
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=BATCH_SIZE)
startup_program.random_seed = 90
main_program.random_seed = 90
else:
train_reader = paddle.batch(
paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500),
batch_size=BATCH_SIZE)
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=BATCH_SIZE)
img = fluid.layers.data(name='img', shape=[1, 28, 28], dtype='float32') img = fluid.layers.data(name='img', shape=[1, 28, 28], dtype='float32')
label = fluid.layers.data(name='label', shape=[1], dtype='int64') label = fluid.layers.data(name='label', shape=[1], dtype='int64')
...@@ -81,8 +113,7 @@ def train(nn_type, ...@@ -81,8 +113,7 @@ def train(nn_type,
prediction, avg_loss, acc = net_conf(img, label) prediction, avg_loss, acc = net_conf(img, label)
test_program = fluid.default_main_program().clone(for_test=True) test_program = main_program.clone(for_test=True)
optimizer = fluid.optimizer.Adam(learning_rate=0.001) optimizer = fluid.optimizer.Adam(learning_rate=0.001)
optimizer.minimize(avg_loss) optimizer.minimize(avg_loss)
...@@ -105,15 +136,8 @@ def train(nn_type, ...@@ -105,15 +136,8 @@ def train(nn_type,
exe = fluid.Executor(place) exe = fluid.Executor(place)
train_reader = paddle.batch(
paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500),
batch_size=BATCH_SIZE)
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=BATCH_SIZE)
feeder = fluid.DataFeeder(feed_list=[img, label], place=place) feeder = fluid.DataFeeder(feed_list=[img, label], place=place)
exe.run(startup_program)
exe.run(fluid.default_startup_program())
main_program = fluid.default_main_program()
epochs = [epoch_id for epoch_id in range(PASS_NUM)] epochs = [epoch_id for epoch_id in range(PASS_NUM)]
lists = [] lists = []
...@@ -144,6 +168,11 @@ def train(nn_type, ...@@ -144,6 +168,11 @@ def train(nn_type,
model_filename=model_filename, model_filename=model_filename,
params_filename=params_filename) params_filename=params_filename)
if args.enable_ce:
print("kpis\ttrain_cost\t%f" % metrics[0])
print("kpis\ttest_cost\t%s" % avg_loss_val)
print("kpis\ttest_acc\t%s" % acc_val)
# find the best pass # find the best pass
best = sorted(lists, key=lambda list: float(list[1]))[0] best = sorted(lists, key=lambda list: float(list[1]))[0]
print('Best pass is %s, testing Avgcost is %s' % (best[0], best[1])) print('Best pass is %s, testing Avgcost is %s' % (best[0], best[1]))
...@@ -210,7 +239,10 @@ def main(use_cuda, nn_type): ...@@ -210,7 +239,10 @@ def main(use_cuda, nn_type):
if __name__ == '__main__': if __name__ == '__main__':
use_cuda = False args = parse_args()
BATCH_SIZE = 64
PASS_NUM = args.num_epochs
use_cuda = args.use_gpu
# predict = 'softmax_regression' # uncomment for Softmax # predict = 'softmax_regression' # uncomment for Softmax
# predict = 'multilayer_perceptron' # uncomment for MLP # predict = 'multilayer_perceptron' # uncomment for MLP
predict = 'convolutional_neural_network' # uncomment for LeNet5 predict = 'convolutional_neural_network' # uncomment for LeNet5
......
#!/bin/bash
#This file is only used for continuous evaluation.
python train.py --enable_ce | python _ce.py
### This file is only used for continuous evaluation test!
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import os
import sys
sys.path.append(os.environ['ceroot'])
from kpi import CostKpi
train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=True, desc='train cost')
tracking_kpis = [train_cost_kpi]
def parse_log(log):
for line in log.split('\n'):
fs = line.strip().split('\t')
print(fs)
if len(fs) == 3 and fs[0] == 'kpis':
kpi_name = fs[1]
kpi_value = float(fs[2])
yield kpi_name, kpi_value
def log_to_ce(log):
kpi_tracker = {}
for kpi in tracking_kpis:
kpi_tracker[kpi.name] = kpi
for (kpi_name, kpi_value) in parse_log(log):
print(kpi_name, kpi_value)
kpi_tracker[kpi_name].add_record(kpi_value)
kpi_tracker[kpi_name].persist()
if __name__ == '__main__':
log = sys.stdin.read()
log_to_ce(log)
...@@ -18,19 +18,31 @@ import six ...@@ -18,19 +18,31 @@ import six
import numpy import numpy
import sys import sys
import math import math
import argparse
EMBED_SIZE = 32 EMBED_SIZE = 32
HIDDEN_SIZE = 256 HIDDEN_SIZE = 256
N = 5 N = 5
BATCH_SIZE = 100 BATCH_SIZE = 100
PASS_NUM = 100
use_cuda = False # set to True if training with GPU
word_dict = paddle.dataset.imikolov.build_dict() word_dict = paddle.dataset.imikolov.build_dict()
dict_size = len(word_dict) dict_size = len(word_dict)
def parse_args():
parser = argparse.ArgumentParser("word2vec")
parser.add_argument(
'--enable_ce',
action='store_true',
help='If set, run the task with continuous evaluation logs.')
parser.add_argument(
'--use_gpu', type=int, default=0, help='whether to use gpu')
parser.add_argument(
'--num_epochs', type=int, default=100, help='number of epoch')
args = parser.parse_args()
return args
def inference_program(words, is_sparse): def inference_program(words, is_sparse):
embed_first = fluid.layers.embedding( embed_first = fluid.layers.embedding(
...@@ -102,6 +114,10 @@ def train(if_use_cuda, params_dirname, is_sparse=True): ...@@ -102,6 +114,10 @@ def train(if_use_cuda, params_dirname, is_sparse=True):
main_program = fluid.default_main_program() main_program = fluid.default_main_program()
star_program = fluid.default_startup_program() star_program = fluid.default_startup_program()
if args.enable_ce:
main_program.random_seed = 90
star_program.random_seed = 90
predict_word = inference_program(word_list, is_sparse) predict_word = inference_program(word_list, is_sparse)
avg_cost = train_program(predict_word) avg_cost = train_program(predict_word)
test_program = main_program.clone(for_test=True) test_program = main_program.clone(for_test=True)
...@@ -153,6 +169,9 @@ def train(if_use_cuda, params_dirname, is_sparse=True): ...@@ -153,6 +169,9 @@ def train(if_use_cuda, params_dirname, is_sparse=True):
# Note 5.8 is a relatively high value. In order to get a better model, one should # Note 5.8 is a relatively high value. In order to get a better model, one should
# aim for avg_cost lower than 3.5. But the training could take longer time. # aim for avg_cost lower than 3.5. But the training could take longer time.
if outs[0] < 5.8: if outs[0] < 5.8:
if args.enable_ce:
print("kpis\ttrain_cost\t%f" % outs[0])
if params_dirname is not None: if params_dirname is not None:
fluid.io.save_inference_model(params_dirname, [ fluid.io.save_inference_model(params_dirname, [
'firstw', 'secondw', 'thirdw', 'fourthw' 'firstw', 'secondw', 'thirdw', 'fourthw'
...@@ -161,7 +180,6 @@ def train(if_use_cuda, params_dirname, is_sparse=True): ...@@ -161,7 +180,6 @@ def train(if_use_cuda, params_dirname, is_sparse=True):
step += 1 step += 1
if math.isnan(float(avg_cost_np[0])): if math.isnan(float(avg_cost_np[0])):
sys.exit("got NaN loss, training failed.") sys.exit("got NaN loss, training failed.")
raise AssertionError("Cost is too large {0:2.2}".format(avg_cost_np[0])) raise AssertionError("Cost is too large {0:2.2}".format(avg_cost_np[0]))
train_loop() train_loop()
...@@ -245,4 +263,7 @@ def main(use_cuda, is_sparse): ...@@ -245,4 +263,7 @@ def main(use_cuda, is_sparse):
if __name__ == '__main__': if __name__ == '__main__':
args = parse_args()
PASS_NUM = args.num_epochs
use_cuda = args.use_gpu # set to True if training with GPU
main(use_cuda=use_cuda, is_sparse=True) main(use_cuda=use_cuda, is_sparse=True)
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册