17 update api (#4294)

* update1.7 save/load & fluid.data

* update datafeed to dataloader

PaddleRec/gru4rec/infer_sample_neg.py

@@ -47,8 +47,7 @@ def infer(args, vocab_size, test_reader, use_cuda):
             for epoch in range(start_index, last_index + 1):
                 copy_program = main_program.clone()
                 model_path = model_dir + "/epoch_" + str(epoch)
-                fluid.io.load_params(
-                    executor=exe, dirname=model_path, main_program=copy_program)
+                fluid.load(copy_program, model_path, exe)
                 accum_num_recall = 0.0
                 accum_num_sum = 0.0
                 t0 = time.time()

PaddleRec/gru4rec/train_sample_neg.py

@@ -121,7 +121,7 @@ def train():
         print("epoch:%d num_steps:%d time_cost(s):%f" %
               (epoch_idx, i, total_time / epoch_idx))
         save_dir = "%s/epoch_%d" % (model_dir, epoch_idx)
-        fluid.io.save_params(executor=exe, dirname=save_dir)
+        fluid.save(fluid.default_main_program(), model_path=save_dir)
         print("model saved in %s" % save_dir)
 
     print("finish training")

PaddleRec/multi-task/MMoE/mmoe_train.py

@@ -59,27 +59,14 @@ def mmoe_layer(inputs, expert_num=8, gate_num=3):
 
 
 def model(dict_dim, emb_dim):
-    label_like = fluid.layers.data(
-        name="label_like",
-        shape=[-1, 1],
-        dtype="int64",
-        lod_level=0,
-        append_batch_size=False)
-    label_comment = fluid.layers.data(
-        name="label_comment",
-        shape=[-1, 1],
-        dtype="int64",
-        lod_level=0,
-        append_batch_size=False)
-    label_share = fluid.layers.data(
-        name="label_share",
-        shape=[-1, 1],
-        dtype="int64",
-        lod_level=0,
-        append_batch_size=False)
-
-    a_data = fluid.layers.data(
-        name="a", shape=[-1, 1], dtype="int64", append_batch_size=False)
+    label_like = fluid.data(
+        name="label_like", shape=[-1, 1], dtype="int64", lod_level=0)
+    label_comment = fluid.data(
+        name="label_comment", shape=[-1, 1], dtype="int64", lod_level=0)
+    label_share = fluid.data(
+        name="label_share", shape=[-1, 1], dtype="int64", lod_level=0)
+
+    a_data = fluid.data(name="a", shape=[-1, 1], dtype="int64")
     emb = fluid.layers.embedding(input=a_data, size=[dict_dim, emb_dim])
 
     outs = mmoe_layer(emb, expert_num=8, gate_num=3)
@@ -127,7 +114,6 @@ loss, data_list = model(dict_dim, emb_dim)
 sgd = fluid.optimizer.SGD(learning_rate=args.base_lr)
 sgd.minimize(loss)
 place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
-feeder = fluid.DataFeeder(feed_list=data_list, place=place)
 exe = fluid.Executor(place)
 exe.run(fluid.default_startup_program())
 for batch_id in range(100):

PaddleRec/multiview_simnet/train.py

@@ -136,17 +136,19 @@ def start_train(args):
     startup_program = fluid.default_startup_program()
     loop_program = fluid.default_main_program()
 
-    feeder = fluid.DataFeeder(feed_list=all_slots, place=place)
     exe = fluid.Executor(place)
     exe.run(startup_program)
+    loader = fluid.io.DataLoader.from_generator(
+        feed_list=all_slots, capacity=10000, iterable=True)
+    loader.set_sample_list_generator(train_reader, places=place)
 
     total_time = 0
     ce_info = []
     for pass_id in range(args.epochs):
         start_time = time.time()
-        for batch_id, data in enumerate(train_reader()):
+        for batch_id, data in enumerate(loader()):
             loss_val, correct_val = exe.run(loop_program,
-                                            feed=feeder.feed(data),
+                                            feed=data,
                                             fetch_list=[avg_cost, correct])
             logger.info("TRAIN --> pass: {} batch_id: {} avg_cost: {}, acc: {}"
                         .format(pass_id, batch_id, loss_val,

PaddleRec/ssr/cluster_train.py

@@ -121,7 +121,7 @@ def train_loop(main_program, avg_cost, acc, train_input_data, place, args,
         print("epoch:%d num_steps:%d time_cost(s):%f" %
               (epoch_idx, i, total_time / epoch_idx))
         save_dir = "%s/epoch_%d" % (args.model_dir, epoch_idx)
-        fluid.io.save_params(executor=exe, dirname=save_dir)
+        fluid.save(fluid.default_main_program(), save_dir)
         print("model saved in %s" % save_dir)
 
     # only for ce

PaddleRec/ssr/infer.py

@@ -84,8 +84,7 @@ def infer(args, vocab_size, test_reader):
             for epoch in range(start_index, last_index + 1):
                 copy_program = main_program.clone()
                 model_path = model_dir + "/epoch_" + str(epoch)
-                fluid.io.load_params(
-                    executor=exe, dirname=model_path, main_program=copy_program)
+                fluid.load(copy_program, model_path, exe)
                 accum_num_recall = 0.0
                 accum_num_sum = 0.0
                 t0 = time.time()

PaddleRec/ssr/train.py

@@ -87,9 +87,12 @@ def train(args):
     optimizer.minimize(avg_cost)
 
     data_list = [var.name for var in train_input_data]
-    feeder = fluid.DataFeeder(feed_list=data_list, place=place)
+    print(data_list)
     exe = fluid.Executor(place)
     exe.run(fluid.default_startup_program())
+    loader = fluid.io.DataLoader.from_generator(
+        feed_list=train_input_data, capacity=10000, iterable=True)
+    loader.set_sample_list_generator(train_reader, places=place)
     if parallel:
         train_exe = fluid.ParallelExecutor(
             use_cuda=use_cuda, loss_name=avg_cost.name)
@@ -103,10 +106,10 @@ def train(args):
         print("epoch_%d start" % epoch_idx)
         t0 = time.time()
         i = 0
-        for batch_id, data in enumerate(train_reader()):
+        for batch_id, data in enumerate(loader()):
             i += 1
             loss_val, correct_val = train_exe.run(
-                feed=feeder.feed(data), fetch_list=[avg_cost.name, acc.name])
+                feed=data, fetch_list=[avg_cost.name, acc.name])
             ce_info.append(float(np.mean(correct_val)) / args.batch_size)
             if i % args.print_batch == 0:
                 logger.info(
@@ -121,7 +124,7 @@ def train(args):
         print("epoch:%d num_steps:%d time_cost(s):%f" %
               (epoch_idx, i, total_time / epoch_idx))
         save_dir = "%s/epoch_%d" % (args.model_dir, epoch_idx)
-        fluid.io.save_params(executor=exe, dirname=save_dir)
+        fluid.save(fluid.default_main_program(), model_path=save_dir)
         print("model saved in %s" % save_dir)
 
     # only for ce