From 8f2e0d59a1bf48b77b7287696b360985300f4ee5 Mon Sep 17 00:00:00 2001 From: zhang wenhui Date: Mon, 17 Feb 2020 12:07:27 +0800 Subject: [PATCH] 17 update api (#4294) * update1.7 save/load & fluid.data * update datafeed to dataloader --- PaddleRec/gru4rec/infer_sample_neg.py | 3 +-- PaddleRec/gru4rec/train_sample_neg.py | 2 +- PaddleRec/multi-task/MMoE/mmoe_train.py | 30 +++++++------------------ PaddleRec/multiview_simnet/train.py | 8 ++++--- PaddleRec/ssr/cluster_train.py | 2 +- PaddleRec/ssr/infer.py | 3 +-- PaddleRec/ssr/train.py | 11 +++++---- 7 files changed, 24 insertions(+), 35 deletions(-) diff --git a/PaddleRec/gru4rec/infer_sample_neg.py b/PaddleRec/gru4rec/infer_sample_neg.py index b77f3685..c09a5221 100644 --- a/PaddleRec/gru4rec/infer_sample_neg.py +++ b/PaddleRec/gru4rec/infer_sample_neg.py @@ -47,8 +47,7 @@ def infer(args, vocab_size, test_reader, use_cuda): for epoch in range(start_index, last_index + 1): copy_program = main_program.clone() model_path = model_dir + "/epoch_" + str(epoch) - fluid.io.load_params( - executor=exe, dirname=model_path, main_program=copy_program) + fluid.load(copy_program, model_path, exe) accum_num_recall = 0.0 accum_num_sum = 0.0 t0 = time.time() diff --git a/PaddleRec/gru4rec/train_sample_neg.py b/PaddleRec/gru4rec/train_sample_neg.py index fbb68705..b3f273f8 100644 --- a/PaddleRec/gru4rec/train_sample_neg.py +++ b/PaddleRec/gru4rec/train_sample_neg.py @@ -121,7 +121,7 @@ def train(): print("epoch:%d num_steps:%d time_cost(s):%f" % (epoch_idx, i, total_time / epoch_idx)) save_dir = "%s/epoch_%d" % (model_dir, epoch_idx) - fluid.io.save_params(executor=exe, dirname=save_dir) + fluid.save(fluid.default_main_program(), model_path=save_dir) print("model saved in %s" % save_dir) print("finish training") diff --git a/PaddleRec/multi-task/MMoE/mmoe_train.py b/PaddleRec/multi-task/MMoE/mmoe_train.py index 713ca393..d0720fc4 100644 --- a/PaddleRec/multi-task/MMoE/mmoe_train.py +++ b/PaddleRec/multi-task/MMoE/mmoe_train.py @@ -59,27 +59,14 @@ def mmoe_layer(inputs, expert_num=8, gate_num=3): def model(dict_dim, emb_dim): - label_like = fluid.layers.data( - name="label_like", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) - label_comment = fluid.layers.data( - name="label_comment", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) - label_share = fluid.layers.data( - name="label_share", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) - - a_data = fluid.layers.data( - name="a", shape=[-1, 1], dtype="int64", append_batch_size=False) + label_like = fluid.data( + name="label_like", shape=[-1, 1], dtype="int64", lod_level=0) + label_comment = fluid.data( + name="label_comment", shape=[-1, 1], dtype="int64", lod_level=0) + label_share = fluid.data( + name="label_share", shape=[-1, 1], dtype="int64", lod_level=0) + + a_data = fluid.data(name="a", shape=[-1, 1], dtype="int64") emb = fluid.layers.embedding(input=a_data, size=[dict_dim, emb_dim]) outs = mmoe_layer(emb, expert_num=8, gate_num=3) @@ -127,7 +114,6 @@ loss, data_list = model(dict_dim, emb_dim) sgd = fluid.optimizer.SGD(learning_rate=args.base_lr) sgd.minimize(loss) place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() -feeder = fluid.DataFeeder(feed_list=data_list, place=place) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) for batch_id in range(100): diff --git a/PaddleRec/multiview_simnet/train.py b/PaddleRec/multiview_simnet/train.py index 8f4072ad..fd8de506 100644 --- a/PaddleRec/multiview_simnet/train.py +++ b/PaddleRec/multiview_simnet/train.py @@ -136,17 +136,19 @@ def start_train(args): startup_program = fluid.default_startup_program() loop_program = fluid.default_main_program() - feeder = fluid.DataFeeder(feed_list=all_slots, place=place) exe = fluid.Executor(place) exe.run(startup_program) + loader = fluid.io.DataLoader.from_generator( + feed_list=all_slots, capacity=10000, iterable=True) + loader.set_sample_list_generator(train_reader, places=place) total_time = 0 ce_info = [] for pass_id in range(args.epochs): start_time = time.time() - for batch_id, data in enumerate(train_reader()): + for batch_id, data in enumerate(loader()): loss_val, correct_val = exe.run(loop_program, - feed=feeder.feed(data), + feed=data, fetch_list=[avg_cost, correct]) logger.info("TRAIN --> pass: {} batch_id: {} avg_cost: {}, acc: {}" .format(pass_id, batch_id, loss_val, diff --git a/PaddleRec/ssr/cluster_train.py b/PaddleRec/ssr/cluster_train.py index bcbdde7e..7cbe7ab4 100644 --- a/PaddleRec/ssr/cluster_train.py +++ b/PaddleRec/ssr/cluster_train.py @@ -121,7 +121,7 @@ def train_loop(main_program, avg_cost, acc, train_input_data, place, args, print("epoch:%d num_steps:%d time_cost(s):%f" % (epoch_idx, i, total_time / epoch_idx)) save_dir = "%s/epoch_%d" % (args.model_dir, epoch_idx) - fluid.io.save_params(executor=exe, dirname=save_dir) + fluid.save(fluid.default_main_program(), save_dir) print("model saved in %s" % save_dir) # only for ce diff --git a/PaddleRec/ssr/infer.py b/PaddleRec/ssr/infer.py index 3a44fad7..c1bc8a71 100644 --- a/PaddleRec/ssr/infer.py +++ b/PaddleRec/ssr/infer.py @@ -84,8 +84,7 @@ def infer(args, vocab_size, test_reader): for epoch in range(start_index, last_index + 1): copy_program = main_program.clone() model_path = model_dir + "/epoch_" + str(epoch) - fluid.io.load_params( - executor=exe, dirname=model_path, main_program=copy_program) + fluid.load(copy_program, model_path, exe) accum_num_recall = 0.0 accum_num_sum = 0.0 t0 = time.time() diff --git a/PaddleRec/ssr/train.py b/PaddleRec/ssr/train.py index a3e85d63..40380531 100644 --- a/PaddleRec/ssr/train.py +++ b/PaddleRec/ssr/train.py @@ -87,9 +87,12 @@ def train(args): optimizer.minimize(avg_cost) data_list = [var.name for var in train_input_data] - feeder = fluid.DataFeeder(feed_list=data_list, place=place) + print(data_list) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) + loader = fluid.io.DataLoader.from_generator( + feed_list=train_input_data, capacity=10000, iterable=True) + loader.set_sample_list_generator(train_reader, places=place) if parallel: train_exe = fluid.ParallelExecutor( use_cuda=use_cuda, loss_name=avg_cost.name) @@ -103,10 +106,10 @@ def train(args): print("epoch_%d start" % epoch_idx) t0 = time.time() i = 0 - for batch_id, data in enumerate(train_reader()): + for batch_id, data in enumerate(loader()): i += 1 loss_val, correct_val = train_exe.run( - feed=feeder.feed(data), fetch_list=[avg_cost.name, acc.name]) + feed=data, fetch_list=[avg_cost.name, acc.name]) ce_info.append(float(np.mean(correct_val)) / args.batch_size) if i % args.print_batch == 0: logger.info( @@ -121,7 +124,7 @@ def train(args): print("epoch:%d num_steps:%d time_cost(s):%f" % (epoch_idx, i, total_time / epoch_idx)) save_dir = "%s/epoch_%d" % (args.model_dir, epoch_idx) - fluid.io.save_params(executor=exe, dirname=save_dir) + fluid.save(fluid.default_main_program(), model_path=save_dir) print("model saved in %s" % save_dir) # only for ce -- GitLab