Skip to content

M
models

PaddlePaddle / models
大约 1 年 前同步成功

通知 222
Star 6828
Fork 2962
M
models
未验证 提交 aa0cb2e9 编写于 作者: P pkpk 提交者： GitHub
浏览文件
操作

Refine api of emotion detection and dialogue domain classification (#4308) 

* Update README.md (#4267)

* test=develop (#4269)

* 3d use new api (#4275)

* PointNet++ and PointRCNN use new API

* Update Readme of Dygraph BERT (#4277)

Fix some typos.

* Update run_classifier_multi_gpu.sh (#4279)

remove the CUDA_VISIBLE_DEVICES

* Update README.md (#4280)

* 17 update api (#4294)

* update1.7 save/load & fluid.data

* update datafeed to dataloader

* Update resnet_acnet.py (#4297)

Bias attr of square conv should be "False" rather than None during training mode.

* test=develop
Co-authored-by: NKaipeng Deng <dengkaipeng@baidu.com>
Co-authored-by: Nzhang wenhui <frankwhzhang@126.com>
Co-authored-by: Nparap1uie-s <parap1uie-s@users.noreply.github.com>
上级 872f494e
隐藏空白更改
内联 并排
Showing with 224 addition and 216 deletion
PaddleCV/image_classification/models/resnet_acnet.py
浏览文件 @ aa0cb2e9
......@@ -172,7 +172,7 @@ class ResNetACNet(object):
act=act if self.deploy else None,
param_attr=ParamAttr(name=name + "_acsquare_weights"),
bias_attr=ParamAttr(name=name + "_acsquare_bias")
if self.deploy else None,
if self.deploy else False,
name=name + '.acsquare.conv2d.output.1')
if self.deploy:
......
PaddleNLP/dialogue_domain_classification/run_classifier.py
浏览文件 @ aa0cb2e9
......@@ -16,7 +16,6 @@
# limitations under the License.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
......@@ -40,43 +39,55 @@ import math
np.random.seed(0)
random.seed(0)
parser = argparse.ArgumentParser(__doc__)
DEV_COUNT = 1
model_g = ArgumentGroup(parser, "model", "model configuration and paths.")
model_g.add_arg("init_checkpoint", str, None, "Init checkpoint to resume training from.")
model_g.add_arg("checkpoints", str, "./checkpoints", "Path to save checkpoints.")
model_g.add_arg("init_checkpoint", str, None,
"Init checkpoint to resume training from.")
model_g.add_arg("checkpoints", str, "./checkpoints",
"Path to save checkpoints.")
model_g.add_arg("config_path", str, "./data/input/model.conf", "Model conf.")
model_g.add_arg("build_dict", bool, False, "Build dict.")
train_g = ArgumentGroup(parser, "training", "training options.")
train_g.add_arg("cpu_num", int, 3, "Number of Threads.")
train_g.add_arg("epoch", int, 100, "Number of epoches for training.")
train_g.add_arg("learning_rate", float, 0.1, "Learning rate used to train with warmup.")
train_g.add_arg("save_steps", int, 1000, "The steps interval to save checkpoints.")
train_g.add_arg("validation_steps", int, 100, "The steps interval to evaluate model performance.")
train_g.add_arg("learning_rate", float, 0.1,
"Learning rate used to train with warmup.")
train_g.add_arg("save_steps", int, 1000,
"The steps interval to save checkpoints.")
train_g.add_arg("validation_steps", int, 100,
"The steps interval to evaluate model performance.")
train_g.add_arg("random_seed", int, 7, "random seed")
train_g.add_arg("threshold", float, 0.1, "When the confidence exceeds the threshold, the corresponding label is given.")
train_g.add_arg(
"threshold", float, 0.1,
"When the confidence exceeds the threshold, the corresponding label is given."
)
log_g = ArgumentGroup(parser, "logging", "logging related.")
log_g.add_arg("skip_steps", int, 10, "The steps interval to print loss.")
data_g = ArgumentGroup(parser, "data", "Data paths, vocab paths and data processing options")
data_g = ArgumentGroup(parser, "data",
"Data paths, vocab paths and data processing options")
data_g.add_arg("data_dir", str, "./data/input/", "Path to training data.")
data_g.add_arg("save_dir", str, "./data/output/", "Path to save.")
data_g.add_arg("max_seq_len", int, 50, "Tokens' number of the longest seqence allowed.")
data_g.add_arg("batch_size", int, 64, "The total number of examples in one batch for training.")
data_g.add_arg("max_seq_len", int, 50,
"Tokens' number of the longest seqence allowed.")
data_g.add_arg("batch_size", int, 64,
"The total number of examples in one batch for training.")
run_type_g = ArgumentGroup(parser, "run_type", "running type options.")
run_type_g.add_arg("use_cuda", bool, False, "If set, use GPU for training.")
# run_type_g.add_arg("use_fast_executor", bool, False, "If set, use fast parallel executor (in experiment).")
run_type_g.add_arg("do_train", bool, True, "Whether to perform evaluation on test data set.")
run_type_g.add_arg("do_eval", bool, True, "Whether to perform evaluation on test data set.")
run_type_g.add_arg("do_test", bool, True, "Whether to perform evaluation on test data set.")
run_type_g.add_arg("do_train", bool, True,
"Whether to perform evaluation on test data set.")
run_type_g.add_arg("do_eval", bool, True,
"Whether to perform evaluation on test data set.")
run_type_g.add_arg("do_test", bool, True,
"Whether to perform evaluation on test data set.")
args = parser.parse_args()
def get_score(pred_result, label, eval_phase):
"""[get precision recall and f-score]
......@@ -93,7 +104,7 @@ def get_score(pred_result, label, eval_phase):
total += 1
pred_labels = []
actual_labels = []
for j in range(1, len(pred_result[0])): # the 0 one is background
for j in range(1, len(pred_result[0])): # the 0 one is background
if pred_result[i][j] == 1:
pred_labels.append(j)
if label[i][j] == 1:
......@@ -106,12 +117,12 @@ def get_score(pred_result, label, eval_phase):
tp += 1
true_cnt += 1
elif len(pred_labels) == 0 and len(actual_labels) == 0:
true_cnt += 1
true_cnt += 1
try:
precision = tp * 1.0 / pred_pos_num
recall = tp * 1.0 / pos_num
f1 = 2 * precision * recall / (recall + precision)
except Exception as e:
except Exception as e:
precision = 0
recall = 0
f1 = 0
......@@ -139,7 +150,7 @@ def train(args, train_exe, build_res, place):
pred_label = build_res["pred_label"]
label = build_res["label"]
fetch_list = [cost.name, prediction.name, pred_label.name, label.name]
train_pyreader = build_res["train_pyreader"]
train_data_loader = build_res["train_data_loader"]
train_prog = build_res["train_prog"]
steps = 0
time_begin = time.time()
......@@ -147,22 +158,24 @@ def train(args, train_exe, build_res, place):
logger.info("Begin training")
for i in range(args.epoch):
try:
for data in train_pyreader():
for data in train_data_loader():
avg_cost_np, avg_pred_np, pred_label, label = train_exe.run(feed=data, program=compiled_prog, \
fetch_list=fetch_list)
steps += 1
if steps % int(args.skip_steps) == 0:
time_end = time.time()
used_time = time_end - time_begin
get_score(pred_label, label, eval_phase = "Train")
get_score(pred_label, label, eval_phase="Train")
logger.info('loss is {}'.format(avg_cost_np))
logger.info("epoch: %d, step: %d, speed: %f steps/s" % (i, steps, args.skip_steps / used_time))
logger.info("epoch: %d, step: %d, speed: %f steps/s" %
(i, steps, args.skip_steps / used_time))
time_begin = time.time()
if steps % args.save_steps == 0:
save_path = os.path.join(args.checkpoints,
"step_" + str(steps))
fluid.io.save_persistables(train_exe, save_path, train_prog)
logger.info("[save]step %d : save at %s" % (steps, save_path))
"step_" + str(steps))
fluid.io.save(train_prog, save_path)
logger.info("[save]step %d : save at %s" %
(steps, save_path))
if steps % args.validation_steps == 0:
if args.do_eval:
evaluate(args, test_exe, build_res, "eval")
......@@ -173,11 +186,16 @@ def train(args, train_exe, build_res, place):
logger.error("Train error : %s" % str(e))
exit(1)
save_path = os.path.join(args.checkpoints, "step_" + str(steps))
fluid.io.save_persistables(train_exe, save_path, train_prog)
fluid.io.save(train_prog, save_path)
logger.info("[save]step %d : save at %s" % (steps, save_path))
def evaluate(args, test_exe, build_res, eval_phase, save_result=False, id2intent=None):
def evaluate(args,
test_exe,
build_res,
eval_phase,
save_result=False,
id2intent=None):
"""[evaluate on dev/test dataset]
Arguments:
......@@ -193,7 +211,7 @@ def evaluate(args, test_exe, build_res, eval_phase, save_result=False, id2intent
save_result {bool} -- [description] (default: {False})
id2intent {[type]} -- [description] (default: {None})
"""
place = build_res["test_place"]
place = build_res["test_place"]
threshold = args.threshold
cost = build_res["cost"]
prediction = build_res["prediction"]
......@@ -203,29 +221,34 @@ def evaluate(args, test_exe, build_res, eval_phase, save_result=False, id2intent
total_cost, total_acc, pred_prob_list, pred_label_list, label_list = [], [], [], [], []
if eval_phase == "eval":
test_prog = build_res["eval_compiled_prog"]
test_pyreader = build_res["eval_pyreader"]
test_data_loader = build_res["eval_data_loader"]
elif eval_phase == "test":
test_prog = build_res["test_compiled_prog"]
test_pyreader = build_res["test_pyreader"]
test_data_loader = build_res["test_data_loader"]
else:
exit(1)
logger.info("-----------------------------------------------------------")
for data in test_pyreader():
for data in test_data_loader():
avg_cost_np, avg_pred_np, pred_label, label= test_exe.run(program=test_prog, fetch_list=fetch_list, feed=data, \
return_numpy=True)
total_cost.append(avg_cost_np)
pred_prob_list.extend(avg_pred_np)
pred_label_list.extend(pred_label)
label_list.extend(label)
if save_result:
logger.info("save result at : %s" % args.save_dir + "/" + eval_phase + ".rst")
logger.info("save result at : %s" % args.save_dir + "/" + eval_phase +
".rst")
save_dir = args.save_dir
if not os.path.exists(save_dir):
logger.warning("save dir not exists, and create it")
os.makedirs(save_dir)
fin = codecs.open(os.path.join(args.data_dir, eval_phase + ".txt"), "r", encoding="utf8")
fout = codecs.open(args.save_dir + "/" + eval_phase + ".rst", "w", encoding="utf8")
fin = codecs.open(
os.path.join(args.data_dir, eval_phase + ".txt"),
"r",
encoding="utf8")
fout = codecs.open(
args.save_dir + "/" + eval_phase + ".rst", "w", encoding="utf8")
for line in pred_prob_list:
query = fin.readline().rsplit("\t", 1)[0]
res = []
......@@ -236,18 +259,23 @@ def evaluate(args, test_exe, build_res, eval_phase, save_result=False, id2intent
if len(res) == 0:
res.append(id2intent[0])
fout.write("%s\t%s\n" % (query, "\2".join(sorted(res))))
fout.close()
fout.close()
fin.close()
logger.info("[%s] result: " % eval_phase)
get_score(pred_label_list, label_list, eval_phase)
logger.info('loss is {}'.format(sum(total_cost) * 1.0 / len(total_cost)))
logger.info("-----------------------------------------------------------")
def create_net(args, flow_data, class_dim, dict_dim, place, model_name="textcnn_net", is_infer=False):
"""[create network and pyreader]
def create_net(args,
flow_data,
class_dim,
dict_dim,
place,
model_name="textcnn_net",
is_infer=False):
"""[create network and loader]
Arguments:
flow_data {[type]} -- [description]
......@@ -266,29 +294,42 @@ def create_net(args, flow_data, class_dim, dict_dim, place, model_name="textcnn_
model = textcnn_net_multi_label
else:
return
char_list = fluid.data(name="char", shape=[None, args.max_seq_len, 1], dtype="int64", lod_level=0)
label = fluid.data(name="label", shape=[None, class_dim], dtype="float32", lod_level=0) # label data
reader = fluid.io.PyReader(feed_list=[char_list, label], capacity=args.batch_size * 10, iterable=True, \
return_list=False)
output = model(char_list, label, dict_dim,
emb_dim=flow_data["model"]["emb_dim"],
hid_dim=flow_data["model"]["hid_dim"],
hid_dim2=flow_data["model"]["hid_dim2"],
class_dim=class_dim,
win_sizes=flow_data["model"]["win_sizes"],
is_infer=is_infer,
threshold=args.threshold,
max_seq_len=args.max_seq_len)
char_list = fluid.data(
name="char",
shape=[None, args.max_seq_len, 1],
dtype="int64",
lod_level=0)
label = fluid.data(
name="label", shape=[None, class_dim], dtype="float32",
lod_level=0) # label data
data_loader = fluid.io.DataLoader.from_generator(
feed_list=[char_list, label],
capacity=args.batch_size * 10,
iterable=True,
return_list=False)
output = model(
char_list,
label,
dict_dim,
emb_dim=flow_data["model"]["emb_dim"],
hid_dim=flow_data["model"]["hid_dim"],
hid_dim2=flow_data["model"]["hid_dim2"],
class_dim=class_dim,
win_sizes=flow_data["model"]["win_sizes"],
is_infer=is_infer,
threshold=args.threshold,
max_seq_len=args.max_seq_len)
if is_infer:
prediction = output
return [reader, prediction]
return [data_loader, prediction]
else:
avg_cost, prediction, pred_label, label = output[0], output[1], output[2], output[3]
return [reader, avg_cost, prediction, pred_label, label]
avg_cost, prediction, pred_label, label = output[0], output[1], output[
2], output[3]
return [data_loader, avg_cost, prediction, pred_label, label]
def build_data_reader(args, char_dict, intent_dict):
"""[decorate samples for pyreader]
def build_data_loader(args, char_dict, intent_dict):
"""[decorate samples for dataloader]
Arguments:
args {[type]} -- [description]
......@@ -298,20 +339,22 @@ def build_data_reader(args, char_dict, intent_dict):
Returns:
[type] -- [description]
"""
reader_res = {}
loader_res = {}
if args.do_train:
train_processor = DataReader(char_dict, intent_dict, args.max_seq_len)
train_data_generator = train_processor.prepare_data(
data_path=args.data_dir + "train.txt",
batch_size=args.batch_size,
mode='train')
reader_res["train_data_generator"] = train_data_generator
loader_res["train_data_generator"] = train_data_generator
num_train_examples = train_processor._get_num_examples()
logger.info("Num train examples: %d" % num_train_examples)
logger.info("Num train steps: %d" % (math.ceil(num_train_examples * 1.0 / args.batch_size) * \
args.epoch // DEV_COUNT))
if math.ceil(num_train_examples * 1.0 / args.batch_size) // DEV_COUNT <= 0:
logger.error("Num of train steps is less than 0 or equals to 0, exit")
if math.ceil(num_train_examples * 1.0 /
args.batch_size) // DEV_COUNT <= 0:
logger.error(
"Num of train steps is less than 0 or equals to 0, exit")
exit(1)
if args.do_eval:
eval_processor = DataReader(char_dict, intent_dict, args.max_seq_len)
......@@ -319,7 +362,7 @@ def build_data_reader(args, char_dict, intent_dict):
data_path=args.data_dir + "eval.txt",
batch_size=args.batch_size,
mode='eval')
reader_res["eval_data_generator"] = eval_data_generator
loader_res["eval_data_generator"] = eval_data_generator
num_eval_examples = eval_processor._get_num_examples()
logger.info("Num eval examples: %d" % num_eval_examples)
if args.do_test:
......@@ -328,11 +371,12 @@ def build_data_reader(args, char_dict, intent_dict):
data_path=args.data_dir + "test.txt",
batch_size=args.batch_size,
mode='test')
reader_res["test_data_generator"] = test_data_generator
return reader_res
loader_res["test_data_generator"] = test_data_generator
return loader_res
def build_graph(args, model_config, num_labels, dict_dim, place, test_place, reader_res):
def build_graph(args, model_config, num_labels, dict_dim, place, test_place,
loader_res):
"""[build paddle graph]
Arguments:
......@@ -341,7 +385,7 @@ def build_graph(args, model_config, num_labels, dict_dim, place, test_place, rea
num_labels {[type]} -- [description]
dict_dim {[type]} -- [description]
place {[type]} -- [description]
reader_res {[type]} -- [description]
loader_res {[type]} -- [description]
Returns:
[type] -- [description]
......@@ -349,7 +393,7 @@ def build_graph(args, model_config, num_labels, dict_dim, place, test_place, rea
res = {}
cost, prediction, pred_label, label = None, None, None, None
train_prog = fluid.default_main_program()
startup_prog = fluid.default_startup_program()
eval_prog = train_prog.clone(for_test=True)
test_prog = train_prog.clone(for_test=True)
......@@ -358,36 +402,42 @@ def build_graph(args, model_config, num_labels, dict_dim, place, test_place, rea
if args.do_train:
with fluid.program_guard(train_prog, startup_prog):
with fluid.unique_name.guard():
train_pyreader, cost, prediction, pred_label, label = create_net(args, model_config, num_labels, \
train_data_loader, cost, prediction, pred_label, label = create_net(args, model_config, num_labels, \
dict_dim, place, model_name="textcnn_net")
train_pyreader.decorate_sample_list_generator(reader_res['train_data_generator'], places=place)
res["train_pyreader"] = train_pyreader
sgd_optimizer = fluid.optimizer.SGD(learning_rate=fluid.layers.exponential_decay(
learning_rate=args.learning_rate, decay_steps=1000, decay_rate=0.5, staircase=True))
train_data_loader.set_sample_list_generator(
loader_res['train_data_generator'], places=place)
res["train_data_loader"] = train_data_loader
sgd_optimizer = fluid.optimizer.SGD(
learning_rate=fluid.layers.exponential_decay(
learning_rate=args.learning_rate,
decay_steps=1000,
decay_rate=0.5,
staircase=True))
sgd_optimizer.minimize(cost)
if args.do_eval:
with fluid.program_guard(eval_prog, startup_prog):
with fluid.unique_name.guard():
eval_pyreader, cost, prediction, pred_label, label = create_net(args, model_config, num_labels, \
eval_data_loader, cost, prediction, pred_label, label = create_net(args, model_config, num_labels, \
dict_dim, test_place, model_name="textcnn_net")
eval_pyreader.decorate_sample_list_generator(reader_res['eval_data_generator'], places=test_place)
res["eval_pyreader"] = eval_pyreader
eval_data_loader.set_sample_list_generator(
loader_res['eval_data_generator'], places=test_place)
res["eval_data_loader"] = eval_data_loader
if args.do_test:
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_pyreader, cost, prediction, pred_label, label = create_net(args, model_config, num_labels, \
test_data_loader, cost, prediction, pred_label, label = create_net(args, model_config, num_labels, \
dict_dim, test_place, model_name="textcnn_net")
test_pyreader.decorate_sample_list_generator(reader_res['test_data_generator'], places=test_place)
res["test_pyreader"] = test_pyreader
test_data_loader.set_sample_list_generator(
loader_res['test_data_generator'], places=test_place)
res["test_data_loader"] = test_data_loader
res["cost"] = cost
res["prediction"] = prediction
res["label"] = label
res["pred_label"] = pred_label
res["train_prog"] =train_prog
res["train_prog"] = train_prog
res["eval_prog"] = eval_prog
res["test_prog"] = test_prog
return res
......@@ -421,22 +471,25 @@ def main(args):
id2intent[int(value)] = key
num_labels = len(intent_dict)
# build model
reader_res = build_data_reader(args, char_dict, intent_dict)
build_res = build_graph(args, model_config, num_labels, dict_dim, place, test_place, reader_res)
loader_res = build_data_loader(args, char_dict, intent_dict)
build_res = build_graph(args, model_config, num_labels, dict_dim, place,
test_place, loader_res)
build_res["place"] = place
build_res["test_place"] = test_place
if not (args.do_train or args.do_eval or args.do_test):
raise ValueError("For args `do_train`, `do_eval` and `do_test`, at "
"least one of them must be True.")
exe.run(startup_prog)
if args.init_checkpoint and args.init_checkpoint != "None":
try:
init_checkpoint(exe, args.init_checkpoint, main_program=startup_prog)
init_checkpoint(
exe, args.init_checkpoint, main_program=startup_prog)
logger.info("Load model from %s" % args.init_checkpoint)
except Exception as e:
logger.exception(str(e))
logger.error("Faild load model from %s [%s]" % (args.init_checkpoint, str(e)))
logger.error("Faild load model from %s [%s]" %
(args.init_checkpoint, str(e)))
build_strategy = fluid.compiler.BuildStrategy()
build_strategy.fuse_all_reduce_ops = False
exec_strategy = fluid.ExecutionStrategy()
......@@ -449,22 +502,23 @@ def main(args):
exec_strategy=exec_strategy)
build_res["compiled_prog"] = compiled_prog
if args.do_test:
test_compiled_prog = fluid.compiler.CompiledProgram(build_res["test_prog"])
test_compiled_prog = fluid.compiler.CompiledProgram(build_res[
"test_prog"])
build_res["test_compiled_prog"] = test_compiled_prog
if args.do_eval:
eval_compiled_prog = fluid.compiler.CompiledProgram(build_res["eval_prog"])
eval_compiled_prog = fluid.compiler.CompiledProgram(build_res[
"eval_prog"])
build_res["eval_compiled_prog"] = eval_compiled_prog
if args.do_train:
train(args, exe, build_res, place)
train(args, exe, build_res, place)
if args.do_eval:
evaluate(args, exe, build_res, "eval", \
save_result=True, id2intent=id2intent)
if args.do_test:
evaluate(args, exe, build_res, "test",\
evaluate(args, exe, build_res, "eval", \
save_result=True, id2intent=id2intent)
if args.do_test:
evaluate(args, exe, build_res, "test",\
save_result=True, id2intent=id2intent)
if __name__ == "__main__":
logger.info("the paddle version is %s" % paddle.__version__)
......
PaddleNLP/dialogue_domain_classification/utils.py
浏览文件 @ aa0cb2e9
......@@ -32,14 +32,13 @@ try:
except ImportError:
import ConfigParser as cp
random_seed = 7
logger = logging.getLogger()
format = "%(asctime)s - %(name)s - %(levelname)s -%(filename)s-%(lineno)4d -%(message)s"
# format = "%(levelname)8s: %(asctime)s: %(filename)s:%(lineno)4d %(message)s"
logging.basicConfig(format=format)
logger.setLevel(logging.INFO)
logger = logging.getLogger('Paddle-DDC')
logger = logging.getLogger('Paddle-DDC')
def str2bool(v):
......@@ -77,6 +76,7 @@ class ArgumentGroup(object):
Arguments:
object {[type]} -- [description]
"""
def __init__(self, parser, title, des):
self._group = parser.add_argument_group(title=title, description=des)
......@@ -107,6 +107,7 @@ class DataReader(object):
Returns:
[type] -- [description]
"""
def __init__(self, char_vocab, intent_dict, max_len):
self._char_vocab = char_vocab
self._intent_dict = intent_dict
......@@ -115,10 +116,10 @@ class DataReader(object):
self.all_data = []
self.max_len = max_len
self.padding_id = 0
def _get_num_examples(self):
return len(self.all_data)
def prepare_data(self, data_path, batch_size, mode):
"""
prepare data
......@@ -128,12 +129,17 @@ class DataReader(object):
# word_dict_path), "The given word dictionary dose not exist."
assert os.path.exists(data_path), "The given data file does not exist."
if mode == "train":
train_reader = fluid.io.batch(paddle.reader.shuffle(self.data_reader(data_path, self.max_len, shuffle=True),
buf_size=batch_size * 100), batch_size)
train_reader = fluid.io.batch(
paddle.reader.shuffle(
self.data_reader(
data_path, self.max_len, shuffle=True),
buf_size=batch_size * 100),
batch_size)
# train_reader = fluid.io.batch(self.data_reader(data_path), batch_size)
return train_reader
else:
test_reader = fluid.io.batch(self.data_reader(data_path, self.max_len), batch_size)
test_reader = fluid.io.batch(
self.data_reader(data_path, self.max_len), batch_size)
return test_reader
def data_reader(self, file_path, max_len, shuffle=False):
......@@ -141,7 +147,7 @@ class DataReader(object):
Convert query into id list
use fixed voc
"""
for line in codecs.open(file_path, "r", encoding="utf8"):
line = line.strip()
if isinstance(line, six.binary_type):
......@@ -150,7 +156,8 @@ class DataReader(object):
char_id_list = list(map(lambda x: 0 if x not in self._char_vocab else int(self._char_vocab[x]), \
list(query)))
if len(char_id_list) < max_len:
char_id_list.extend([self.padding_id] * (max_len - len(char_id_list)))
char_id_list.extend([self.padding_id] *
(max_len - len(char_id_list)))
char_id_list = char_id_list[:max_len]
intent_id_list = [self.padding_id] * self.intent_size
for item in intent.split('\2'):
......@@ -159,6 +166,7 @@ class DataReader(object):
if shuffle:
random.seed(random_seed)
random.shuffle(self.all_data)
def reader():
"""
reader
......@@ -166,6 +174,7 @@ class DataReader(object):
for char_id_list, intent_id_list in self.all_data:
# print char_id_list, intent_id
yield char_id_list, intent_id_list
return reader
......@@ -178,6 +187,7 @@ class DataProcesser(object):
Returns:
[type] -- [description]
"""
@staticmethod
def read_dict(filename):
"""
......@@ -211,7 +221,7 @@ class DataProcesser(object):
char_dict = {}
intent_dict = {}
# readfile
for line in codecs.open(filename):
for line in codecs.open(filename):
line = line.strip()
if isinstance(line, six.binary_type):
line = line.strip().decode("utf8", errors="ignore")
......@@ -227,7 +237,8 @@ class DataProcesser(object):
intent_dict[intent] = 0
intent_dict[intent] += 1
# save char dict
with codecs.open("%s/char.dict" % save_dir, "w", encoding="utf8") as f_out:
with codecs.open(
"%s/char.dict" % save_dir, "w", encoding="utf8") as f_out:
f_out.write("PAD\0020\n")
f_out.write("OOV\0021\n")
char_id = 2
......@@ -238,7 +249,8 @@ class DataProcesser(object):
f_out.write("%s\002%d\n" % (key, char_id))
char_id += 1
# save intent dict
with codecs.open("%s/domain.dict" % save_dir, "w", encoding="utf8") as f_out:
with codecs.open(
"%s/domain.dict" % save_dir, "w", encoding="utf8") as f_out:
f_out.write("SYS_OTHER\0020\n")
intent_id = 1
for key, value in intent_dict.items():
......@@ -247,7 +259,6 @@ class DataProcesser(object):
key = key.encode("utf8")
f_out.write("%s\002%d\n" % (key, intent_id))
intent_id += 1
class ConfigReader(object):
......@@ -282,49 +293,13 @@ class ConfigReader(object):
return flow_data
def init_pretraining_params(exe,
pretraining_params_path,
main_program,
use_fp16=False):
"""load params of pretrained model, NOT including moment, learning_rate"""
assert os.path.exists(pretraining_params_path
), "[%s] cann't be found." % pretraining_params_path
def _existed_params(var):
if not isinstance(var, fluid.framework.Parameter):
return False
return os.path.exists(os.path.join(pretraining_params_path, var.name))
fluid.io.load_vars(
exe,
pretraining_params_path,
main_program=main_program,
predicate=_existed_params)
print("Load pretraining parameters from {}.".format(
pretraining_params_path))
def init_checkpoint(exe, init_checkpoint_path, main_program):
"""
Init CheckPoint
"""
assert os.path.exists(
init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path
fluid.load(main_program, init_checkpoint_path, exe)
print("Load model from {}".format(init_checkpoint_path))
def existed_persitables(var):
"""
If existed presitabels
"""
if not fluid.io.is_persistable(var):
return False
return os.path.exists(os.path.join(init_checkpoint_path, var.name))
fluid.io.load_vars(
exe,
init_checkpoint_path,
main_program=main_program,
predicate=existed_persitables)
print ("Load model from {}".format(init_checkpoint_path))
def print_arguments(args):
"""
......@@ -350,5 +325,3 @@ def check_version(version='1.6.0'):
except Exception as e:
logger.error(err)
sys.exit(1)
PaddleNLP/emotion_detection/run_classifier.py
浏览文件 @ aa0cb2e9
......@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Emotion Detection Task
"""
......@@ -38,9 +37,7 @@ import reader
import utils
def create_model(args,
num_labels,
is_prediction=False):
def create_model(args, num_labels, is_prediction=False):
"""
Create Model for Emotion Detection
"""
......@@ -77,10 +74,17 @@ def create_model(args,
raise ValueError("Unknown network type!")
if is_prediction:
probs = network(data, seq_len, None, args.vocab_size, class_dim=num_labels, is_prediction=True)
probs = network(
data,
seq_len,
None,
args.vocab_size,
class_dim=num_labels,
is_prediction=True)
return loader, probs, [data.name, seq_len.name]
avg_loss, probs = network(data, seq_len, label, args.vocab_size, class_dim=num_labels)
avg_loss, probs = network(
data, seq_len, label, args.vocab_size, class_dim=num_labels)
num_seqs = fluid.layers.create_tensor(dtype='int64')
accuracy = fluid.layers.accuracy(input=probs, label=label, total=num_seqs)
return loader, avg_loss, accuracy, num_seqs
......@@ -142,9 +146,10 @@ def main(args):
exe = fluid.Executor(place)
task_name = args.task_name.lower()
processor = reader.EmoTectProcessor(data_dir=args.data_dir,
vocab_path=args.vocab_path,
random_seed=args.random_seed)
processor = reader.EmoTectProcessor(
data_dir=args.data_dir,
vocab_path=args.vocab_path,
random_seed=args.random_seed)
#num_labels = len(processor.get_labels())
num_labels = args.num_labels
......@@ -173,9 +178,7 @@ def main(args):
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
train_loader, loss, accuracy, num_seqs = create_model(
args,
num_labels=num_labels,
is_prediction=False)
args, num_labels=num_labels, is_prediction=False)
sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=args.lr)
sgd_optimizer.minimize(loss)
......@@ -189,37 +192,27 @@ def main(args):
if args.do_val:
if args.do_train:
test_data_generator = processor.data_generator(
batch_size=args.batch_size,
phase='dev',
epoch=1)
batch_size=args.batch_size, phase='dev', epoch=1)
else:
test_data_generator = processor.data_generator(
batch_size=args.batch_size,
phase='test',
epoch=1)
batch_size=args.batch_size, phase='test', epoch=1)
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_loader, loss, accuracy, num_seqs = create_model(
args,
num_labels=num_labels,
is_prediction=False)
args, num_labels=num_labels, is_prediction=False)
test_prog = test_prog.clone(for_test=True)
if args.do_infer:
infer_data_generator = processor.data_generator(
batch_size=args.batch_size,
phase='infer',
epoch=1)
batch_size=args.batch_size, phase='infer', epoch=1)
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
infer_loader, probs, _ = create_model(
args,
num_labels=num_labels,
is_prediction=True)
args, num_labels=num_labels, is_prediction=True)
test_prog = test_prog.clone(for_test=True)
exe.run(startup_prog)
......@@ -292,8 +285,9 @@ def main(args):
time_begin = time.time()
if steps % args.save_steps == 0:
save_path = os.path.join(args.save_checkpoint_dir, "step_" + str(steps))
fluid.io.save_persistables(exe, save_path, train_program)
save_path = os.path.join(args.save_checkpoint_dir,
"step_" + str(steps))
fluid.save(train_program, save_path)
if steps % args.validation_steps == 0:
# evaluate on dev set
......@@ -306,11 +300,11 @@ def main(args):
print("final step: %d " % steps)
if args.do_val:
evaluate(test_exe, test_prog, test_loader,
[loss.name, accuracy.name, num_seqs.name],
"dev")
[loss.name, accuracy.name, num_seqs.name], "dev")
save_path = os.path.join(args.save_checkpoint_dir, "step_" + str(steps))
fluid.io.save_persistables(exe, save_path, train_program)
save_path = os.path.join(args.save_checkpoint_dir,
"step_" + str(steps))
fluid.save(train_program, save_path)
train_loader.reset()
break
......@@ -334,15 +328,12 @@ def main(args):
if not args.do_train and args.do_val:
print("Final test result:")
evaluate(test_exe, test_prog, test_loader,
[loss.name, accuracy.name, num_seqs.name],
"test")
[loss.name, accuracy.name, num_seqs.name], "test")
# infer
if args.do_infer:
print("Final infer result:")
infer(test_exe, test_prog, infer_loader,
[probs.name],
"infer")
infer(test_exe, test_prog, infer_loader, [probs.name], "infer")
def get_cards():
......
PaddleNLP/emotion_detection/run_ernie_classifier.py
浏览文件 @ aa0cb2e9
......@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Emotion Detection Task, based on ERNIE
"""
......@@ -350,7 +349,7 @@ def main(args):
if steps % args.save_steps == 0:
save_path = os.path.join(args.save_checkpoint_dir, "step_" + str(steps))
fluid.io.save_persistables(exe, save_path, train_program)
fluid.save(train_program, save_path)
if steps % args.validation_steps == 0:
# evaluate dev set
......@@ -369,7 +368,7 @@ def main(args):
except fluid.core.EOFException:
save_path = os.path.join(args.save_checkpoint_dir, "step_" + str(steps))
fluid.io.save_persistables(exe, save_path, train_program)
fluid.save(train_program, save_path)
train_pyreader.reset()
break
......
PaddleNLP/emotion_detection/utils.py
浏览文件 @ aa0cb2e9
......@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
EmoTect utilities.
"""
......@@ -29,27 +28,13 @@ import paddle
import paddle.fluid as fluid
import numpy as np
def init_checkpoint(exe, init_checkpoint_path, main_program):
"""
Init CheckPoint
"""
assert os.path.exists(
init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path
def existed_persitables(var):
"""
If existed presitabels
"""
if not fluid.io.is_persistable(var):
return False
return os.path.exists(os.path.join(init_checkpoint_path, var.name))
fluid.io.load_vars(
exe,
init_checkpoint_path,
main_program=main_program,
predicate=existed_persitables)
print("Load model from {}".format(init_checkpoint_path))
fluid.load(main_program, init_checkpoint_path, exe)
def word2id(word_dict, query):
......@@ -57,8 +42,10 @@ def word2id(word_dict, query):
Convert word sequence into id list
"""
unk_id = len(word_dict)
wids = [word_dict[w] if w in word_dict else unk_id
for w in query.strip().split(" ")]
wids = [
word_dict[w] if w in word_dict else unk_id
for w in query.strip().split(" ")
]
return wids
......
PaddleRec/multi-task/MMoE/mmoe_train.py
浏览文件 @ aa0cb2e9
......@@ -114,7 +114,6 @@ loss, data_list = model(dict_dim, emb_dim)
sgd = fluid.optimizer.SGD(learning_rate=args.base_lr)
sgd.minimize(loss)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
feeder = fluid.DataFeeder(feed_list=data_list, place=place)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
for batch_id in range(100):
......
PaddleRec/multiview_simnet/train.py
浏览文件 @ aa0cb2e9
......@@ -136,17 +136,19 @@ def start_train(args):
startup_program = fluid.default_startup_program()
loop_program = fluid.default_main_program()
feeder = fluid.DataFeeder(feed_list=all_slots, place=place)
exe = fluid.Executor(place)
exe.run(startup_program)
loader = fluid.io.DataLoader.from_generator(
feed_list=all_slots, capacity=10000, iterable=True)
loader.set_sample_list_generator(train_reader, places=place)
total_time = 0
ce_info = []
for pass_id in range(args.epochs):
start_time = time.time()
for batch_id, data in enumerate(train_reader()):
for batch_id, data in enumerate(loader()):
loss_val, correct_val = exe.run(loop_program,
feed=feeder.feed(data),
feed=data,
fetch_list=[avg_cost, correct])
logger.info("TRAIN --> pass: {} batch_id: {} avg_cost: {}, acc: {}"
.format(pass_id, batch_id, loss_val,
......
PaddleRec/ssr/train.py
浏览文件 @ aa0cb2e9
......@@ -87,9 +87,12 @@ def train(args):
optimizer.minimize(avg_cost)
data_list = [var.name for var in train_input_data]
feeder = fluid.DataFeeder(feed_list=data_list, place=place)
print(data_list)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
loader = fluid.io.DataLoader.from_generator(
feed_list=train_input_data, capacity=10000, iterable=True)
loader.set_sample_list_generator(train_reader, places=place)
if parallel:
train_exe = fluid.ParallelExecutor(
use_cuda=use_cuda, loss_name=avg_cost.name)
......@@ -103,10 +106,10 @@ def train(args):
print("epoch_%d start" % epoch_idx)
t0 = time.time()
i = 0
for batch_id, data in enumerate(train_reader()):
for batch_id, data in enumerate(loader()):
i += 1
loss_val, correct_val = train_exe.run(
feed=feeder.feed(data), fetch_list=[avg_cost.name, acc.name])
feed=data, fetch_list=[avg_cost.name, acc.name])
ce_info.append(float(np.mean(correct_val)) / args.batch_size)
if i % args.print_batch == 0:
logger.info(
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或