Skip to content

M
models

PaddlePaddle / models
大约 1 年 前同步成功

通知 222
Star 6828
Fork 2962
M
models
提交 e56ead99 编写于 作者: C chengyao
浏览文件
操作

fix codestyle

上级 2d392828
显示空白变更内容
内联 并排
Showing with 270 addition and 312 deletion
fluid/text_classification/clouds/scdb_parallel_executor.py
浏览文件 @ e56ead99
......@@ -9,6 +9,7 @@ import os
import json
import random
def to_lodtensor(data, place):
"""
convert to LODtensor
......@@ -45,20 +46,22 @@ def data2tensor(data, place):
"""
data2tensor
"""
input_seq = to_lodtensor(map(lambda x:x[0], data), place)
input_seq = to_lodtensor(map(lambda x: x[0], data), place)
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
y_data = y_data.reshape([-1, 1])
return {"words": input_seq, "label": y_data}
def data2pred(data, place):
"""
data2tensor
"""
input_seq = to_lodtensor(map(lambda x:x[0], data), place)
input_seq = to_lodtensor(map(lambda x: x[0], data), place)
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
y_data = y_data.reshape([-1, 1])
return {"words": input_seq}
def load_dict(vocab):
"""
Load dict from vocab
......@@ -80,6 +83,7 @@ def save_dict(word_dict, vocab):
outstr = ("%s\t%s\n" % (k, v)).encode("gb18030")
fout.write(outstr)
def build_dict(fname):
"""
build word dict using trainset
......@@ -88,7 +92,8 @@ def build_dict(fname):
with open(fname, "r") as fin:
for line in fin:
try:
words = line.strip("\r\n").decode("gb18030").split("\t")[1].split(" ")
words = line.strip("\r\n").decode("gb18030").split("\t")[
1].split(" ")
except:
sys.stderr.write("[warning] build_dict: decode error\n")
continue
......@@ -133,7 +138,10 @@ def data_reader(fname, word_dict, is_dir=False):
continue
label = int(cols[0])
wids = [word_dict[x] if x in word_dict else unk_id for x in cols[1].split(" ")]
wids = [
word_dict[x] if x in word_dict else unk_id
for x in cols[1].split(" ")
]
all_data.append((wids, label))
random.shuffle(all_data)
......@@ -145,7 +153,8 @@ def data_reader(fname, word_dict, is_dir=False):
return reader
def scdb_train_data(train_dir="scdb_data/train_set/corpus.train.seg", w_dict=None):
def scdb_train_data(train_dir="scdb_data/train_set/corpus.train.seg",
w_dict=None):
"""
create train data
"""
......@@ -160,7 +169,8 @@ def scdb_test_data(test_file, w_dict):
return data_reader(test_file, w_dict)
def bow_net(data, label,
def bow_net(data,
label,
dict_dim,
emb_dim=128,
hid_dim=128,
......@@ -169,19 +179,12 @@ def bow_net(data, label,
"""
bow net
"""
emb = fluid.layers.embedding(input=data,
size=[dict_dim, emb_dim])
bow = fluid.layers.sequence_pool(
input=emb,
pool_type='sum')
emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim])
bow = fluid.layers.sequence_pool(input=emb, pool_type='sum')
bow_tanh = fluid.layers.tanh(bow)
fc_1 = fluid.layers.fc(input=bow_tanh,
size=hid_dim, act = "tanh")
fc_2 = fluid.layers.fc(input=fc_1,
size=hid_dim2, act = "tanh")
prediction = fluid.layers.fc(input=[fc_2],
size=class_dim,
act="softmax")
fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh")
fc_2 = fluid.layers.fc(input=fc_1, size=hid_dim2, act="tanh")
prediction = fluid.layers.fc(input=[fc_2], size=class_dim, act="softmax")
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
acc = fluid.layers.accuracy(input=prediction, label=label)
......@@ -189,7 +192,8 @@ def bow_net(data, label,
return avg_cost, acc, prediction
def cnn_net(data, label,
def cnn_net(data,
label,
dict_dim,
emb_dim=128,
hid_dim=128,
......@@ -199,21 +203,18 @@ def cnn_net(data, label,
"""
conv net
"""
emb = fluid.layers.embedding(input=data,
size=[dict_dim, emb_dim])
emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim])
conv_3 = fluid.nets.sequence_conv_pool(input=emb,
conv_3 = fluid.nets.sequence_conv_pool(
input=emb,
num_filters=hid_dim,
filter_size=win_size,
act="tanh",
pool_type="max")
fc_1 = fluid.layers.fc(input=[conv_3],
size=hid_dim2)
fc_1 = fluid.layers.fc(input=[conv_3], size=hid_dim2)
prediction = fluid.layers.fc(input=[fc_1],
size=class_dim,
act="softmax")
prediction = fluid.layers.fc(input=[fc_1], size=class_dim, act="softmax")
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
acc = fluid.layers.accuracy(input=prediction, label=label)
......@@ -221,7 +222,8 @@ def cnn_net(data, label,
return avg_cost, acc, prediction
def lstm_net(data, label,
def lstm_net(data,
label,
dict_dim,
emb_dim=128,
hid_dim=128,
......@@ -231,29 +233,22 @@ def lstm_net(data, label,
"""
lstm net
"""
emb = fluid.layers.embedding(input=data,
emb = fluid.layers.embedding(
input=data,
size=[dict_dim, emb_dim],
param_attr=fluid.ParamAttr(learning_rate=emb_lr))
fc0 = fluid.layers.fc(input=emb,
size=hid_dim * 4,
act='tanh')
fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4, act='tanh')
lstm_h, c = fluid.layers.dynamic_lstm(input=fc0,
size=hid_dim * 4,
is_reverse=False)
lstm_h, c = fluid.layers.dynamic_lstm(
input=fc0, size=hid_dim * 4, is_reverse=False)
lstm_max = fluid.layers.sequence_pool(input=lstm_h,
pool_type='max')
lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max')
lstm_max_tanh = fluid.layers.tanh(lstm_max)
fc1 = fluid.layers.fc(input=lstm_max_tanh,
size=hid_dim2,
act='tanh')
fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh')
prediction = fluid.layers.fc(input=fc1,
size=class_dim,
act='softmax')
prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax')
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
......@@ -262,7 +257,8 @@ def lstm_net(data, label,
return avg_cost, acc, prediction
def bilstm_net(data, label,
def bilstm_net(data,
label,
dict_dim,
emb_dim=128,
hid_dim=128,
......@@ -272,25 +268,20 @@ def bilstm_net(data, label,
"""
lstm net
"""
emb = fluid.layers.embedding(input=data,
emb = fluid.layers.embedding(
input=data,
size=[dict_dim, emb_dim],
param_attr=fluid.ParamAttr(learning_rate=emb_lr))
fc0 = fluid.layers.fc(input=emb,
size=hid_dim * 4,
act='tanh')
fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4, act='tanh')
rfc0 = fluid.layers.fc(input=emb,
size=hid_dim * 4,
act='tanh')
rfc0 = fluid.layers.fc(input=emb, size=hid_dim * 4, act='tanh')
lstm_h, c = fluid.layers.dynamic_lstm(input=fc0,
size=hid_dim * 4,
is_reverse=False)
lstm_h, c = fluid.layers.dynamic_lstm(
input=fc0, size=hid_dim * 4, is_reverse=False)
rlstm_h, c = fluid.layers.dynamic_lstm(input=rfc0,
size=hid_dim * 4,
is_reverse=True)
rlstm_h, c = fluid.layers.dynamic_lstm(
input=rfc0, size=hid_dim * 4, is_reverse=True)
lstm_last = fluid.layers.sequence_last_step(input=lstm_h)
rlstm_last = fluid.layers.sequence_last_step(input=rlstm_h)
......@@ -300,13 +291,9 @@ def bilstm_net(data, label,
lstm_concat = fluid.layers.concat(input=[lstm_last, rlstm_last], axis=1)
fc1 = fluid.layers.fc(input=lstm_concat,
size=hid_dim2,
act='tanh')
fc1 = fluid.layers.fc(input=lstm_concat, size=hid_dim2, act='tanh')
prediction = fluid.layers.fc(input=fc1,
size=class_dim,
act='softmax')
prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax')
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
......@@ -315,7 +302,8 @@ def bilstm_net(data, label,
return avg_cost, acc, prediction
def gru_net(data, label,
def gru_net(data,
label,
dict_dim,
emb_dim=128,
hid_dim=128,
......@@ -325,28 +313,21 @@ def gru_net(data, label,
"""
gru net
"""
emb = fluid.layers.embedding(input=data,
emb = fluid.layers.embedding(
input=data,
size=[dict_dim, emb_dim],
param_attr=fluid.ParamAttr(learning_rate=emb_lr))
fc0 = fluid.layers.fc(input=emb,
size=hid_dim * 3)
fc0 = fluid.layers.fc(input=emb, size=hid_dim * 3)
gru_h = fluid.layers.dynamic_gru(input=fc0,
size=hid_dim,
is_reverse=False)
gru_h = fluid.layers.dynamic_gru(input=fc0, size=hid_dim, is_reverse=False)
gru_max = fluid.layers.sequence_pool(input=gru_h,
pool_type='max')
gru_max = fluid.layers.sequence_pool(input=gru_h, pool_type='max')
gru_max_tanh = fluid.layers.tanh(gru_max)
fc1 = fluid.layers.fc(input=gru_max_tanh,
size=hid_dim2,
act='tanh')
fc1 = fluid.layers.fc(input=gru_max_tanh, size=hid_dim2, act='tanh')
prediction = fluid.layers.fc(input=fc1,
size=class_dim,
act='softmax')
prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax')
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
......@@ -354,9 +335,8 @@ def gru_net(data, label,
return avg_cost, acc, prediction
def infer(test_reader,
use_cuda,
model_path=None):
def infer(test_reader, use_cuda, model_path=None):
"""
inference function
"""
......@@ -379,7 +359,7 @@ def infer(test_reader,
for data in each_test_reader():
pred = exe.run(inference_program,
feed = data2pred(data, place),
feed=data2pred(data, place),
fetch_list=fetch_targets,
return_numpy=True)
......@@ -410,7 +390,8 @@ def infer(test_reader,
class2_acc = sum(class2_list) / len(class2_list)
class3_acc = sum(class3_list) / len(class3_list)
print("[test info] model_path: %s, class2_acc: %f, class3_acc: %f" % (model_path, class2_acc, class3_acc))
print("[test info] model_path: %s, class2_acc: %f, class3_acc: %f" %
(model_path, class2_acc, class3_acc))
def start_train(train_reader,
......@@ -427,18 +408,11 @@ def start_train(train_reader,
train network
"""
data = fluid.layers.data(
name="words",
shape=[1],
dtype="int64",
lod_level=1)
name="words", shape=[1], dtype="int64", lod_level=1)
label = fluid.layers.data(
name="label",
shape=[1],
dtype="int64")
label = fluid.layers.data(name="label", shape=[1], dtype="int64")
cost, acc, pred = network(
data, label, len(word_dict) + 1)
cost, acc, pred = network(data, label, len(word_dict) + 1)
sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=lr)
sgd_optimizer.minimize(cost)
......@@ -462,15 +436,13 @@ def start_train(train_reader,
avg_cost = total_cost / total_count
avg_acc = total_acc / total_count
print("[train info]: pass_id: %d, avg_acc: %f, avg_cost: %f" % (pass_id, avg_acc, avg_cost))
print("[train info]: pass_id: %d, avg_acc: %f, avg_cost: %f" %
(pass_id, avg_acc, avg_cost))
gpu_place = fluid.CUDAPlace(0)
save_exe = fluid.Executor(gpu_place)
epoch_model = save_dirname + "/" + "epoch" + str(pass_id)
fluid.io.save_inference_model(
epoch_model,
["words"],
pred, save_exe)
fluid.io.save_inference_model(epoch_model, ["words"], pred, save_exe)
infer(test_reader, False, epoch_model)
......@@ -500,9 +472,17 @@ def train_net(vocab="./thirdparty/train.vocab",
paddle.dataset.imdb.test(w_dict), buf_size=50000),
batch_size=128)
test_reader = [test_reader]
start_train(train_reader, test_reader, w_dict, bilstm_net, use_cuda=True,
parallel=False, save_dirname="scdb_bilstm_model", lr=0.05,
pass_num=10, batch_size=256)
start_train(
train_reader,
test_reader,
w_dict,
bilstm_net,
use_cuda=True,
parallel=False,
save_dirname="scdb_bilstm_model",
lr=0.05,
pass_num=10,
batch_size=256)
if __name__ == "__main__":
......
fluid/text_classification/clouds/scdb_single_card.py
浏览文件 @ e56ead99
......@@ -9,6 +9,7 @@ import os
import json
import random
def to_lodtensor(data, place):
"""
convert to LODtensor
......@@ -45,20 +46,22 @@ def data2tensor(data, place):
"""
data2tensor
"""
input_seq = to_lodtensor(map(lambda x:x[0], data), place)
input_seq = to_lodtensor(map(lambda x: x[0], data), place)
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
y_data = y_data.reshape([-1, 1])
return {"words": input_seq, "label": y_data}
def data2pred(data, place):
"""
data2tensor
"""
input_seq = to_lodtensor(map(lambda x:x[0], data), place)
input_seq = to_lodtensor(map(lambda x: x[0], data), place)
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
y_data = y_data.reshape([-1, 1])
return {"words": input_seq}
def load_dict(vocab):
"""
Load dict from vocab
......@@ -80,6 +83,7 @@ def save_dict(word_dict, vocab):
outstr = ("%s\t%s\n" % (k, v)).encode("gb18030")
fout.write(outstr)
def build_dict(fname):
"""
build word dict using trainset
......@@ -88,7 +92,8 @@ def build_dict(fname):
with open(fname, "r") as fin:
for line in fin:
try:
words = line.strip("\r\n").decode("gb18030").split("\t")[1].split(" ")
words = line.strip("\r\n").decode("gb18030").split("\t")[
1].split(" ")
except:
sys.stderr.write("[warning] build_dict: decode error\n")
continue
......@@ -133,7 +138,10 @@ def data_reader(fname, word_dict, is_dir=False):
continue
label = int(cols[0])
wids = [word_dict[x] if x in word_dict else unk_id for x in cols[1].split(" ")]
wids = [
word_dict[x] if x in word_dict else unk_id
for x in cols[1].split(" ")
]
all_data.append((wids, label))
random.shuffle(all_data)
......@@ -145,7 +153,8 @@ def data_reader(fname, word_dict, is_dir=False):
return reader
def scdb_train_data(train_dir="scdb_data/train_set/corpus.train.seg", w_dict=None):
def scdb_train_data(train_dir="scdb_data/train_set/corpus.train.seg",
w_dict=None):
"""
create train data
"""
......@@ -160,7 +169,8 @@ def scdb_test_data(test_file, w_dict):
return data_reader(test_file, w_dict)
def bow_net(data, label,
def bow_net(data,
label,
dict_dim,
emb_dim=128,
hid_dim=128,
......@@ -169,19 +179,12 @@ def bow_net(data, label,
"""
bow net
"""
emb = fluid.layers.embedding(input=data,
size=[dict_dim, emb_dim])
bow = fluid.layers.sequence_pool(
input=emb,
pool_type='sum')
emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim])
bow = fluid.layers.sequence_pool(input=emb, pool_type='sum')
bow_tanh = fluid.layers.tanh(bow)
fc_1 = fluid.layers.fc(input=bow_tanh,
size=hid_dim, act = "tanh")
fc_2 = fluid.layers.fc(input=fc_1,
size=hid_dim2, act = "tanh")
prediction = fluid.layers.fc(input=[fc_2],
size=class_dim,
act="softmax")
fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh")
fc_2 = fluid.layers.fc(input=fc_1, size=hid_dim2, act="tanh")
prediction = fluid.layers.fc(input=[fc_2], size=class_dim, act="softmax")
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
acc = fluid.layers.accuracy(input=prediction, label=label)
......@@ -189,7 +192,8 @@ def bow_net(data, label,
return avg_cost, acc, prediction
def cnn_net(data, label,
def cnn_net(data,
label,
dict_dim,
emb_dim=128,
hid_dim=128,
......@@ -199,21 +203,18 @@ def cnn_net(data, label,
"""
conv net
"""
emb = fluid.layers.embedding(input=data,
size=[dict_dim, emb_dim])
emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim])
conv_3 = fluid.nets.sequence_conv_pool(input=emb,
conv_3 = fluid.nets.sequence_conv_pool(
input=emb,
num_filters=hid_dim,
filter_size=win_size,
act="tanh",
pool_type="max")
fc_1 = fluid.layers.fc(input=[conv_3],
size=hid_dim2)
fc_1 = fluid.layers.fc(input=[conv_3], size=hid_dim2)
prediction = fluid.layers.fc(input=[fc_1],
size=class_dim,
act="softmax")
prediction = fluid.layers.fc(input=[fc_1], size=class_dim, act="softmax")
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
acc = fluid.layers.accuracy(input=prediction, label=label)
......@@ -221,7 +222,8 @@ def cnn_net(data, label,
return avg_cost, acc, prediction
def lstm_net(data, label,
def lstm_net(data,
label,
dict_dim,
emb_dim=128,
hid_dim=128,
......@@ -231,29 +233,22 @@ def lstm_net(data, label,
"""
lstm net
"""
emb = fluid.layers.embedding(input=data,
emb = fluid.layers.embedding(
input=data,
size=[dict_dim, emb_dim],
param_attr=fluid.ParamAttr(learning_rate=emb_lr))
fc0 = fluid.layers.fc(input=emb,
size=hid_dim * 4,
act='tanh')
fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4, act='tanh')
lstm_h, c = fluid.layers.dynamic_lstm(input=fc0,
size=hid_dim * 4,
is_reverse=False)
lstm_h, c = fluid.layers.dynamic_lstm(
input=fc0, size=hid_dim * 4, is_reverse=False)
lstm_max = fluid.layers.sequence_pool(input=lstm_h,
pool_type='max')
lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max')
lstm_max_tanh = fluid.layers.tanh(lstm_max)
fc1 = fluid.layers.fc(input=lstm_max_tanh,
size=hid_dim2,
act='tanh')
fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh')
prediction = fluid.layers.fc(input=fc1,
size=class_dim,
act='softmax')
prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax')
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
......@@ -262,7 +257,8 @@ def lstm_net(data, label,
return avg_cost, acc, prediction
def bilstm_net(data, label,
def bilstm_net(data,
label,
dict_dim,
emb_dim=128,
hid_dim=128,
......@@ -272,25 +268,20 @@ def bilstm_net(data, label,
"""
lstm net
"""
emb = fluid.layers.embedding(input=data,
emb = fluid.layers.embedding(
input=data,
size=[dict_dim, emb_dim],
param_attr=fluid.ParamAttr(learning_rate=emb_lr))
fc0 = fluid.layers.fc(input=emb,
size=hid_dim * 4,
act='tanh')
fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4, act='tanh')
rfc0 = fluid.layers.fc(input=emb,
size=hid_dim * 4,
act='tanh')
rfc0 = fluid.layers.fc(input=emb, size=hid_dim * 4, act='tanh')
lstm_h, c = fluid.layers.dynamic_lstm(input=fc0,
size=hid_dim * 4,
is_reverse=False)
lstm_h, c = fluid.layers.dynamic_lstm(
input=fc0, size=hid_dim * 4, is_reverse=False)
rlstm_h, c = fluid.layers.dynamic_lstm(input=rfc0,
size=hid_dim * 4,
is_reverse=True)
rlstm_h, c = fluid.layers.dynamic_lstm(
input=rfc0, size=hid_dim * 4, is_reverse=True)
lstm_last = fluid.layers.sequence_last_step(input=lstm_h)
rlstm_last = fluid.layers.sequence_last_step(input=rlstm_h)
......@@ -300,13 +291,9 @@ def bilstm_net(data, label,
lstm_concat = fluid.layers.concat(input=[lstm_last, rlstm_last], axis=1)
fc1 = fluid.layers.fc(input=lstm_concat,
size=hid_dim2,
act='tanh')
fc1 = fluid.layers.fc(input=lstm_concat, size=hid_dim2, act='tanh')
prediction = fluid.layers.fc(input=fc1,
size=class_dim,
act='softmax')
prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax')
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
......@@ -315,7 +302,8 @@ def bilstm_net(data, label,
return avg_cost, acc, prediction
def gru_net(data, label,
def gru_net(data,
label,
dict_dim,
emb_dim=128,
hid_dim=128,
......@@ -325,28 +313,21 @@ def gru_net(data, label,
"""
gru net
"""
emb = fluid.layers.embedding(input=data,
emb = fluid.layers.embedding(
input=data,
size=[dict_dim, emb_dim],
param_attr=fluid.ParamAttr(learning_rate=emb_lr))
fc0 = fluid.layers.fc(input=emb,
size=hid_dim * 3)
fc0 = fluid.layers.fc(input=emb, size=hid_dim * 3)
gru_h = fluid.layers.dynamic_gru(input=fc0,
size=hid_dim,
is_reverse=False)
gru_h = fluid.layers.dynamic_gru(input=fc0, size=hid_dim, is_reverse=False)
gru_max = fluid.layers.sequence_pool(input=gru_h,
pool_type='max')
gru_max = fluid.layers.sequence_pool(input=gru_h, pool_type='max')
gru_max_tanh = fluid.layers.tanh(gru_max)
fc1 = fluid.layers.fc(input=gru_max_tanh,
size=hid_dim2,
act='tanh')
fc1 = fluid.layers.fc(input=gru_max_tanh, size=hid_dim2, act='tanh')
prediction = fluid.layers.fc(input=fc1,
size=class_dim,
act='softmax')
prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax')
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
......@@ -355,9 +336,7 @@ def gru_net(data, label,
return avg_cost, acc, prediction
def infer(test_reader,
use_cuda,
model_path=None):
def infer(test_reader, use_cuda, model_path=None):
"""
inference function
"""
......@@ -380,7 +359,7 @@ def infer(test_reader,
for data in each_test_reader():
pred = exe.run(inference_program,
feed = data2pred(data, place),
feed=data2pred(data, place),
fetch_list=fetch_targets,
return_numpy=True)
......@@ -411,7 +390,8 @@ def infer(test_reader,
class2_acc = sum(class2_list) / len(class2_list)
class3_acc = sum(class3_list) / len(class3_list)
print("[test info] model_path: %s, class2_acc: %f, class3_acc: %f" % (model_path, class2_acc, class3_acc))
print("[test info] model_path: %s, class2_acc: %f, class3_acc: %f" %
(model_path, class2_acc, class3_acc))
def start_train(train_reader,
......@@ -428,18 +408,11 @@ def start_train(train_reader,
train network
"""
data = fluid.layers.data(
name="words",
shape=[1],
dtype="int64",
lod_level=1)
name="words", shape=[1], dtype="int64", lod_level=1)
label = fluid.layers.data(
name="label",
shape=[1],
dtype="int64")
label = fluid.layers.data(name="label", shape=[1], dtype="int64")
cost, acc, pred = network(
data, label, len(word_dict) + 1)
cost, acc, pred = network(data, label, len(word_dict) + 1)
sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=lr)
sgd_optimizer.minimize(cost)
......@@ -462,13 +435,11 @@ def start_train(train_reader,
avg_cost = total_cost / data_count
avg_acc = total_acc / data_count
print("[train info]: pass_id: %d, avg_acc: %f, avg_cost: %f" % (pass_id, avg_acc, avg_cost))
print("[train info]: pass_id: %d, avg_acc: %f, avg_cost: %f" %
(pass_id, avg_acc, avg_cost))
epoch_model = save_dirname + "/" + "epoch" + str(pass_id)
fluid.io.save_inference_model(
epoch_model,
["words"],
pred, exe)
fluid.io.save_inference_model(epoch_model, ["words"], pred, exe)
infer(test_reader, False, epoch_model)
......@@ -476,18 +447,25 @@ def train_net(vocab="./thirdparty/train.vocab",
train_dir="./train",
test_list=["car", "spot", "weibo", "lbs"]):
w_dict = scdb_word_dict(vocab=vocab)
test_files = [ "./thirdparty" + os.sep + f for f in test_list]
test_files = ["./thirdparty" + os.sep + f for f in test_list]
train_reader = paddle.batch(
scdb_train_data(train_dir, w_dict),
batch_size = 256)
scdb_train_data(train_dir, w_dict), batch_size=256)
test_reader = [paddle.batch(scdb_test_data(test_file, w_dict), batch_size = 50) \
for test_file in test_files]
start_train(train_reader, test_reader, w_dict, bow_net, use_cuda=False,
parallel=False, save_dirname="scdb_bow_model", lr=0.002,
pass_num=10, batch_size=256)
start_train(
train_reader,
test_reader,
w_dict,
bow_net,
use_cuda=False,
parallel=False,
save_dirname="scdb_bow_model",
lr=0.002,
pass_num=10,
batch_size=256)
if __name__ == "__main__":
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或