提交 3558a6d2 编写于 作者: P Paddle CI

fix code style by yapf

上级 448c59aa
...@@ -7,11 +7,11 @@ from kpi import CostKpi ...@@ -7,11 +7,11 @@ from kpi import CostKpi
from kpi import DurationKpi from kpi import DurationKpi
imikolov_20_avg_ppl_kpi = CostKpi('imikolov_20_avg_ppl', 0.2, 0) imikolov_20_avg_ppl_kpi = CostKpi('imikolov_20_avg_ppl', 0.2, 0)
imikolov_20_pass_duration_kpi = DurationKpi('imikolov_20_pass_duration', 0.02, imikolov_20_pass_duration_kpi = DurationKpi(
0, actived=True) 'imikolov_20_pass_duration', 0.02, 0, actived=True)
imikolov_20_avg_ppl_kpi_card4 = CostKpi('imikolov_20_avg_ppl_card4', 0.2, 0) imikolov_20_avg_ppl_kpi_card4 = CostKpi('imikolov_20_avg_ppl_card4', 0.2, 0)
imikolov_20_pass_duration_kpi_card4 = DurationKpi('imikolov_20_pass_duration_card4', 0.03, imikolov_20_pass_duration_kpi_card4 = DurationKpi(
0, actived=True) 'imikolov_20_pass_duration_card4', 0.03, 0, actived=True)
tracking_kpis = [ tracking_kpis = [
imikolov_20_avg_ppl_kpi, imikolov_20_avg_ppl_kpi,
...@@ -20,6 +20,7 @@ tracking_kpis = [ ...@@ -20,6 +20,7 @@ tracking_kpis = [
imikolov_20_pass_duration_kpi_card4, imikolov_20_pass_duration_kpi_card4,
] ]
def parse_log(log): def parse_log(log):
''' '''
This method should be implemented by model developers. This method should be implemented by model developers.
...@@ -38,7 +39,7 @@ def parse_log(log): ...@@ -38,7 +39,7 @@ def parse_log(log):
''' '''
for line in log.split('\n'): for line in log.split('\n'):
fs = line.strip().split('\t') fs = line.strip().split('\t')
print (fs) print(fs)
if len(fs) == 3 and fs[0] == 'kpis': if len(fs) == 3 and fs[0] == 'kpis':
kpi_name = fs[1] kpi_name = fs[1]
kpi_value = float(fs[2]) kpi_value = float(fs[2])
...@@ -51,7 +52,7 @@ def log_to_ce(log): ...@@ -51,7 +52,7 @@ def log_to_ce(log):
kpi_tracker[kpi.name] = kpi kpi_tracker[kpi.name] = kpi
for (kpi_name, kpi_value) in parse_log(log): for (kpi_name, kpi_value) in parse_log(log):
print (kpi_name, kpi_value) print(kpi_name, kpi_value)
kpi_tracker[kpi_name].add_record(kpi_value) kpi_tracker[kpi_name].add_record(kpi_value)
kpi_tracker[kpi_name].persist() kpi_tracker[kpi_name].persist()
...@@ -59,4 +60,3 @@ def log_to_ce(log): ...@@ -59,4 +60,3 @@ def log_to_ce(log):
if __name__ == '__main__': if __name__ == '__main__':
log = sys.stdin.read() log = sys.stdin.read()
log_to_ce(log) log_to_ce(log)
...@@ -13,6 +13,7 @@ import utils ...@@ -13,6 +13,7 @@ import utils
# random seed must set before configuring the network. # random seed must set before configuring the network.
fluid.default_startup_program().random_seed = 102 fluid.default_startup_program().random_seed = 102
def network(src, dst, vocab_size, hid_size, init_low_bound, init_high_bound): def network(src, dst, vocab_size, hid_size, init_low_bound, init_high_bound):
""" network definition """ """ network definition """
emb_lr_x = 10.0 emb_lr_x = 10.0
...@@ -97,7 +98,7 @@ def train(train_reader, ...@@ -97,7 +98,7 @@ def train(train_reader,
train_exe = fluid.ParallelExecutor(use_cuda=True, loss_name=avg_cost.name) train_exe = fluid.ParallelExecutor(use_cuda=True, loss_name=avg_cost.name)
total_time = 0.0 total_time = 0.0
fetch_list=[avg_cost.name] fetch_list = [avg_cost.name]
for pass_idx in xrange(pass_num): for pass_idx in xrange(pass_num):
epoch_idx = pass_idx + 1 epoch_idx = pass_idx + 1
print "epoch_%d start" % epoch_idx print "epoch_%d start" % epoch_idx
...@@ -111,8 +112,7 @@ def train(train_reader, ...@@ -111,8 +112,7 @@ def train(train_reader,
map(lambda x: x[0], data), place) map(lambda x: x[0], data), place)
lod_dst_wordseq = utils.to_lodtensor( lod_dst_wordseq = utils.to_lodtensor(
map(lambda x: x[1], data), place) map(lambda x: x[1], data), place)
ret_avg_cost = train_exe.run( ret_avg_cost = train_exe.run(feed={
feed={
"src_wordseq": lod_src_wordseq, "src_wordseq": lod_src_wordseq,
"dst_wordseq": lod_dst_wordseq "dst_wordseq": lod_dst_wordseq
}, },
...@@ -124,39 +124,44 @@ def train(train_reader, ...@@ -124,39 +124,44 @@ def train(train_reader,
t1 = time.time() t1 = time.time()
total_time += t1 - t0 total_time += t1 - t0
print "epoch:%d num_steps:%d time_cost(s):%f" % ( print "epoch:%d num_steps:%d time_cost(s):%f" % (epoch_idx, i,
epoch_idx, i, total_time / epoch_idx) total_time / epoch_idx)
if pass_idx == pass_num - 1: if pass_idx == pass_num - 1:
#Note: The following logs are special for CE monitoring. #Note: The following logs are special for CE monitoring.
#Other situations do not need to care about these logs. #Other situations do not need to care about these logs.
gpu_num = get_cards() gpu_num = get_cards()
if gpu_num == 1: if gpu_num == 1:
print("kpis imikolov_20_pass_duration %s" % (total_time / epoch_idx)) print("kpis imikolov_20_pass_duration %s" %
(total_time / epoch_idx))
print("kpis imikolov_20_avg_ppl %s" % newest_ppl) print("kpis imikolov_20_avg_ppl %s" % newest_ppl)
else: else:
print("kpis imikolov_20_pass_duration_card%s %s" % \ print("kpis imikolov_20_pass_duration_card%s %s" % \
(gpu_num, total_time / epoch_idx)) (gpu_num, total_time / epoch_idx))
print("kpis imikolov_20_avg_ppl_card%s %s" % (gpu_num, newest_ppl)) print("kpis imikolov_20_avg_ppl_card%s %s" %
(gpu_num, newest_ppl))
save_dir = "%s/epoch_%d" % (model_dir, epoch_idx) save_dir = "%s/epoch_%d" % (model_dir, epoch_idx)
feed_var_names = ["src_wordseq", "dst_wordseq"] feed_var_names = ["src_wordseq", "dst_wordseq"]
fetch_vars = [avg_cost] fetch_vars = [avg_cost]
fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, exe)
exe)
print("model saved in %s" % save_dir) print("model saved in %s" % save_dir)
print("finish training") print("finish training")
def get_cards(): def get_cards():
cards = os.environ.get('CUDA_VISIBLE_DEVICES') cards = os.environ.get('CUDA_VISIBLE_DEVICES')
num = len(cards.split(",")) num = len(cards.split(","))
return num return num
def train_net(): def train_net():
""" do training """ """ do training """
batch_size = 20 batch_size = 20
vocab, train_reader, test_reader = utils.prepare_data( vocab, train_reader, test_reader = utils.prepare_data(
batch_size=batch_size * get_cards(), buffer_size=1000, word_freq_threshold=0) batch_size=batch_size * get_cards(),
buffer_size=1000,
word_freq_threshold=0)
train( train(
train_reader=train_reader, train_reader=train_reader,
vocab=vocab, vocab=vocab,
......
...@@ -5,6 +5,7 @@ import numpy as np ...@@ -5,6 +5,7 @@ import numpy as np
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle import paddle
def to_lodtensor(data, place): def to_lodtensor(data, place):
""" convert to LODtensor """ """ convert to LODtensor """
seq_lens = [len(seq) for seq in data] seq_lens = [len(seq) for seq in data]
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册