diff --git a/PaddleNLP/language_model/args.py b/PaddleNLP/language_model/args.py index eef0af99c71cd6ab672268f177b8762204987d87..958a32e3de70398324711f60a1bfb1fe3651cfc6 100644 --- a/PaddleNLP/language_model/args.py +++ b/PaddleNLP/language_model/args.py @@ -60,10 +60,10 @@ def parse_args(): default=False, help='Whether profiling the trainning [True|False]') parser.add_argument( - '--use_py_reader', + '--use_dataloader', type=str2bool, default=False, - help='Whether using py_reader to feed data [True|False]') + help='Whether using dataloader to feed data [True|False]') parser.add_argument( '--log_path', help='path of the log file. If not set, logs are printed to console') diff --git a/PaddleNLP/language_model/run.sh b/PaddleNLP/language_model/run.sh index d836c4acefbaadb2540a389e4410f32e689c8a5e..e8c711faf990da8c2566a99c3611fa838a0adf64 100644 --- a/PaddleNLP/language_model/run.sh +++ b/PaddleNLP/language_model/run.sh @@ -6,7 +6,7 @@ function run_train() { python train.py \ --data_path data/simple-examples/data/ \ --model_type small \ - --use_gpu True + --use_gpu True \ } run_train diff --git a/PaddleNLP/language_model/train.py b/PaddleNLP/language_model/train.py index 169c5c7089dd58f02de0a03b07949244081fc004..423d136ad8bdf328375c4b72e2687810b84931eb 100644 --- a/PaddleNLP/language_model/train.py +++ b/PaddleNLP/language_model/train.py @@ -124,10 +124,10 @@ def main(): init_scale=config.init_scale, dropout=config.dropout, rnn_model=config.rnn_model, - use_py_reader=args.use_py_reader) + use_dataloader=args.use_dataloader) - if args.use_py_reader: - py_reader = res_vars[-1] + if args.use_dataloader: + dataloader = res_vars[-1] res_vars = res_vars[:-1] loss, last_hidden, last_cell, feed_order = res_vars @@ -159,7 +159,7 @@ def main(): init_scale=config.init_scale, dropout=config.dropout, rnn_model=config.rnn_model, - use_py_reader=False) + use_dataloader=False) # Some op behaves differently for train and inference, we need to call # this clone function to ensure every op is right for inference. inference_program = inference_program.clone(for_test=True) @@ -176,8 +176,6 @@ def main(): exec_strategy.num_iteration_per_drop_scope = 100 build_strategy = fluid.BuildStrategy() - build_strategy.enable_inplace = True - build_strategy.memory_optimize = False build_strategy.fuse_all_optimizer_ops = True if args.parallel: @@ -310,7 +308,7 @@ def main(): ppl = np.exp(total_loss / iters) return ppl - def train_an_epoch_py_reader(epoch_id, batch_times): + def train_an_epoch_dataloader(epoch_id, batch_times): # get train epoch size log_interval = get_log_interval(len(train_data)) @@ -319,7 +317,7 @@ def main(): total_loss = 0 iters = 0 - py_reader.start() + dataloader.start() batch_id = 0 try: while True: @@ -361,14 +359,14 @@ def main(): batch_id += 1 except fluid.core.EOFException: - py_reader.reset() + dataloader.reset() batch_times.append(time.time() - batch_start_time) ppl = np.exp(total_loss / iters) return ppl def train(): - if args.use_py_reader: + if args.use_dataloader: def data_gen(): data_iter_size = config.batch_size // device_count @@ -380,14 +378,14 @@ def main(): y = y.reshape((-1, 1)) yield x, y - py_reader.decorate_tensor_provider(data_gen) + dataloader.set_batch_generator(data_gen) total_time = 0.0 for epoch_id in range(config.max_epoch): batch_times = [] epoch_start_time = time.time() - if args.use_py_reader: - train_ppl = train_an_epoch_py_reader(epoch_id, batch_times) + if args.use_dataloader: + train_ppl = train_an_epoch_dataloader(epoch_id, batch_times) else: train_ppl = train_an_epoch(epoch_id, batch_times) epoch_time = time.time() - epoch_start_time diff --git a/PaddleNLP/models/language_model/lm_model.py b/PaddleNLP/models/language_model/lm_model.py index 731d8f5ab832e9d7b5873aa940b900cc296f1a5e..4dc0f9f36fa4c143ac8f2a8cd7c0b7c00ddcbfa2 100644 --- a/PaddleNLP/models/language_model/lm_model.py +++ b/PaddleNLP/models/language_model/lm_model.py @@ -32,7 +32,7 @@ def lm_model(hidden_size, init_scale=0.1, dropout=None, rnn_model='static', - use_py_reader=False): + use_dataloader=False): def padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None): weight_1_arr = [] weight_2_arr = [] @@ -255,23 +255,23 @@ def lm_model(hidden_size, return real_res, last_hidden, last_cell batch_size_each = batch_size // fluid.core.get_cuda_device_count() - if use_py_reader: - feed_shapes = [[batch_size_each, num_steps, 1], - [batch_size_each * num_steps, 1]] - py_reader = fluid.layers.py_reader( - capacity=16, shapes=feed_shapes, dtypes=['int64', 'int64']) - x, y = fluid.layers.read_file(py_reader) - else: - x = layers.data( - name="x", - shape=[batch_size_each, num_steps, 1], - dtype='int64', - append_batch_size=False) - y = layers.data( - name="y", - shape=[batch_size_each * num_steps, 1], - dtype='int64', - append_batch_size=False) + x = layers.data( + name="x", + shape=[batch_size_each, num_steps, 1], + dtype='int64', + append_batch_size=False) + y = layers.data( + name="y", + shape=[batch_size_each * num_steps, 1], + dtype='int64', + append_batch_size=False) + + if use_dataloader: + dataloader = fluid.io.DataLoader.from_generator( + feed_list=[x, y], + capacity=16, + iterable=False, + use_double_buffer=True) init_hidden = layers.data( name="init_hidden", @@ -385,7 +385,7 @@ def lm_model(hidden_size, layers.assign(input=last_hidden, output=init_hidden) feeding_list = ['x', 'y', 'init_hidden', 'init_cell'] - if use_py_reader: - return loss, last_hidden, last_cell, feeding_list, py_reader + if use_dataloader: + return loss, last_hidden, last_cell, feeding_list, dataloader else: return loss, last_hidden, last_cell, feeding_list