diff --git a/PaddleNLP/language_model/args.py b/PaddleNLP/language_model/args.py index 958a32e3de70398324711f60a1bfb1fe3651cfc6..eef0af99c71cd6ab672268f177b8762204987d87 100644 --- a/PaddleNLP/language_model/args.py +++ b/PaddleNLP/language_model/args.py @@ -60,10 +60,10 @@ def parse_args(): default=False, help='Whether profiling the trainning [True|False]') parser.add_argument( - '--use_dataloader', + '--use_py_reader', type=str2bool, default=False, - help='Whether using dataloader to feed data [True|False]') + help='Whether using py_reader to feed data [True|False]') parser.add_argument( '--log_path', help='path of the log file. If not set, logs are printed to console') diff --git a/PaddleNLP/language_model/run.sh b/PaddleNLP/language_model/run.sh index e8c711faf990da8c2566a99c3611fa838a0adf64..d836c4acefbaadb2540a389e4410f32e689c8a5e 100644 --- a/PaddleNLP/language_model/run.sh +++ b/PaddleNLP/language_model/run.sh @@ -6,7 +6,7 @@ function run_train() { python train.py \ --data_path data/simple-examples/data/ \ --model_type small \ - --use_gpu True \ + --use_gpu True } run_train diff --git a/PaddleNLP/language_model/train.py b/PaddleNLP/language_model/train.py index 423d136ad8bdf328375c4b72e2687810b84931eb..169c5c7089dd58f02de0a03b07949244081fc004 100644 --- a/PaddleNLP/language_model/train.py +++ b/PaddleNLP/language_model/train.py @@ -124,10 +124,10 @@ def main(): init_scale=config.init_scale, dropout=config.dropout, rnn_model=config.rnn_model, - use_dataloader=args.use_dataloader) + use_py_reader=args.use_py_reader) - if args.use_dataloader: - dataloader = res_vars[-1] + if args.use_py_reader: + py_reader = res_vars[-1] res_vars = res_vars[:-1] loss, last_hidden, last_cell, feed_order = res_vars @@ -159,7 +159,7 @@ def main(): init_scale=config.init_scale, dropout=config.dropout, rnn_model=config.rnn_model, - use_dataloader=False) + use_py_reader=False) # Some op behaves differently for train and inference, we need to call # this clone function to ensure every op is right for inference. inference_program = inference_program.clone(for_test=True) @@ -176,6 +176,8 @@ def main(): exec_strategy.num_iteration_per_drop_scope = 100 build_strategy = fluid.BuildStrategy() + build_strategy.enable_inplace = True + build_strategy.memory_optimize = False build_strategy.fuse_all_optimizer_ops = True if args.parallel: @@ -308,7 +310,7 @@ def main(): ppl = np.exp(total_loss / iters) return ppl - def train_an_epoch_dataloader(epoch_id, batch_times): + def train_an_epoch_py_reader(epoch_id, batch_times): # get train epoch size log_interval = get_log_interval(len(train_data)) @@ -317,7 +319,7 @@ def main(): total_loss = 0 iters = 0 - dataloader.start() + py_reader.start() batch_id = 0 try: while True: @@ -359,14 +361,14 @@ def main(): batch_id += 1 except fluid.core.EOFException: - dataloader.reset() + py_reader.reset() batch_times.append(time.time() - batch_start_time) ppl = np.exp(total_loss / iters) return ppl def train(): - if args.use_dataloader: + if args.use_py_reader: def data_gen(): data_iter_size = config.batch_size // device_count @@ -378,14 +380,14 @@ def main(): y = y.reshape((-1, 1)) yield x, y - dataloader.set_batch_generator(data_gen) + py_reader.decorate_tensor_provider(data_gen) total_time = 0.0 for epoch_id in range(config.max_epoch): batch_times = [] epoch_start_time = time.time() - if args.use_dataloader: - train_ppl = train_an_epoch_dataloader(epoch_id, batch_times) + if args.use_py_reader: + train_ppl = train_an_epoch_py_reader(epoch_id, batch_times) else: train_ppl = train_an_epoch(epoch_id, batch_times) epoch_time = time.time() - epoch_start_time diff --git a/PaddleNLP/models/language_model/lm_model.py b/PaddleNLP/models/language_model/lm_model.py index 4dc0f9f36fa4c143ac8f2a8cd7c0b7c00ddcbfa2..731d8f5ab832e9d7b5873aa940b900cc296f1a5e 100644 --- a/PaddleNLP/models/language_model/lm_model.py +++ b/PaddleNLP/models/language_model/lm_model.py @@ -32,7 +32,7 @@ def lm_model(hidden_size, init_scale=0.1, dropout=None, rnn_model='static', - use_dataloader=False): + use_py_reader=False): def padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None): weight_1_arr = [] weight_2_arr = [] @@ -255,23 +255,23 @@ def lm_model(hidden_size, return real_res, last_hidden, last_cell batch_size_each = batch_size // fluid.core.get_cuda_device_count() - x = layers.data( - name="x", - shape=[batch_size_each, num_steps, 1], - dtype='int64', - append_batch_size=False) - y = layers.data( - name="y", - shape=[batch_size_each * num_steps, 1], - dtype='int64', - append_batch_size=False) - - if use_dataloader: - dataloader = fluid.io.DataLoader.from_generator( - feed_list=[x, y], - capacity=16, - iterable=False, - use_double_buffer=True) + if use_py_reader: + feed_shapes = [[batch_size_each, num_steps, 1], + [batch_size_each * num_steps, 1]] + py_reader = fluid.layers.py_reader( + capacity=16, shapes=feed_shapes, dtypes=['int64', 'int64']) + x, y = fluid.layers.read_file(py_reader) + else: + x = layers.data( + name="x", + shape=[batch_size_each, num_steps, 1], + dtype='int64', + append_batch_size=False) + y = layers.data( + name="y", + shape=[batch_size_each * num_steps, 1], + dtype='int64', + append_batch_size=False) init_hidden = layers.data( name="init_hidden", @@ -385,7 +385,7 @@ def lm_model(hidden_size, layers.assign(input=last_hidden, output=init_hidden) feeding_list = ['x', 'y', 'init_hidden', 'init_cell'] - if use_dataloader: - return loss, last_hidden, last_cell, feeding_list, dataloader + if use_py_reader: + return loss, last_hidden, last_cell, feeding_list, py_reader else: return loss, last_hidden, last_cell, feeding_list