diff --git a/PaddleRec/word2vec/cluster_train.py b/PaddleRec/word2vec/cluster_train.py index 4ea3476491c89cb412d69cd8a6bc550ce810877a..45cd9aec83e7aa5bb1b05669a3769a6592ae5489 100644 --- a/PaddleRec/word2vec/cluster_train.py +++ b/PaddleRec/word2vec/cluster_train.py @@ -134,10 +134,10 @@ def convert_python_to_tensor(weight, batch_size, sample_reader): return __reader__ -def train_loop(args, train_program, reader, py_reader, loss, trainer_id, +def train_loop(args, train_program, reader, data_loader, loss, trainer_id, weight): - py_reader.decorate_tensor_provider( + data_loader.set_batch_generator( convert_python_to_tensor(weight, args.batch_size, reader.train())) place = fluid.CPUPlace() @@ -149,7 +149,7 @@ def train_loop(args, train_program, reader, py_reader, loss, trainer_id, train_exe = exe for pass_id in range(args.num_passes): - py_reader.start() + data_loader.start() time.sleep(10) epoch_start = time.time() batch_id = 0 @@ -164,7 +164,7 @@ def train_loop(args, train_program, reader, py_reader, loss, trainer_id, logger.info( "TRAIN --> pass: {} batch: {} loss: {} reader queue:{}". format(pass_id, batch_id, - loss_val.mean(), py_reader.queue.size())) + loss_val.mean(), data_loader.queue.size())) if args.with_speed: if batch_id % 500 == 0 and batch_id != 0: elapsed = (time.time() - start) @@ -178,18 +178,18 @@ def train_loop(args, train_program, reader, py_reader, loss, trainer_id, model_dir = args.model_output_dir + '/pass-' + str( pass_id) + ('/batch-' + str(batch_id)) if trainer_id == 0: - fluid.io.save_params(executor=exe, dirname=model_dir) + fluid.save(fluid.default_main_program(), model_path=model_dir) print("model saved in %s" % model_dir) batch_id += 1 except fluid.core.EOFException: - py_reader.reset() + data_loader.reset() epoch_end = time.time() logger.info("Epoch: {0}, Train total expend: {1} ".format( pass_id, epoch_end - epoch_start)) model_dir = args.model_output_dir + '/pass-' + str(pass_id) if trainer_id == 0: - fluid.io.save_params(executor=exe, dirname=model_dir) + fluid.save(fluid.default_main_program(), model_path=model_dir) print("model saved in %s" % model_dir) @@ -210,7 +210,7 @@ def train(args): np_power = np.power(np.array(word2vec_reader.id_frequencys), 0.75) id_frequencys_pow = np_power / np_power.sum() - loss, py_reader = skip_gram_word2vec( + loss, data_loader = skip_gram_word2vec( word2vec_reader.dict_size, args.embedding_size, is_sparse=args.is_sparse, @@ -241,7 +241,7 @@ def train(args): elif args.role == "trainer": print("run trainer") train_loop(args, - t.get_trainer_program(), word2vec_reader, py_reader, loss, + t.get_trainer_program(), word2vec_reader, data_loader, loss, args.trainer_id, id_frequencys_pow) diff --git a/PaddleRec/word2vec/infer.py b/PaddleRec/word2vec/infer.py index 32f65d17de585ef4f1fc14797e300ef7f55ac877..892c9e78a98efd9f51ce6a7a6e2ad94d0b7b26e8 100644 --- a/PaddleRec/word2vec/infer.py +++ b/PaddleRec/word2vec/infer.py @@ -70,8 +70,7 @@ def infer_epoch(args, vocab_size, test_reader, use_cuda, i2w): for epoch in range(start_index, last_index + 1): copy_program = main_program.clone() model_path = model_dir + "/pass-" + str(epoch) - fluid.io.load_params( - executor=exe, dirname=model_path, main_program=copy_program) + fluid.load(copy_program, model_path, exe) accum_num = 0 accum_num_sum = 0.0 t0 = time.time() @@ -131,10 +130,7 @@ def infer_step(args, vocab_size, test_reader, use_cuda, i2w): copy_program = main_program.clone() model_path = model_dir + "/pass-" + str(epoch) + ( '/batch-' + str(batchid * args.print_step)) - fluid.io.load_params( - executor=exe, - dirname=model_path, - main_program=copy_program) + fluid.load(copy_program, model_path, exe) accum_num = 0 accum_num_sum = 0.0 t0 = time.time() diff --git a/PaddleRec/word2vec/net.py b/PaddleRec/word2vec/net.py index b8379b6696b3253f60079ccc4b4042e253d5ace4..54d29378fb6afe2d65bd12ec36eb508b778aefaf 100644 --- a/PaddleRec/word2vec/net.py +++ b/PaddleRec/word2vec/net.py @@ -22,20 +22,19 @@ import paddle.fluid as fluid def skip_gram_word2vec(dict_size, embedding_size, is_sparse=False, neg_num=5): - datas = [] + words = [] input_word = fluid.data(name="input_word", shape=[None, 1], dtype='int64') true_word = fluid.data(name='true_label', shape=[None, 1], dtype='int64') neg_word = fluid.data( name="neg_label", shape=[None, neg_num], dtype='int64') - datas.append(input_word) - datas.append(true_word) - datas.append(neg_word) + words.append(input_word) + words.append(true_word) + words.append(neg_word) - py_reader = fluid.layers.create_py_reader_by_data( - capacity=64, feed_list=datas, name='py_reader', use_double_buffer=True) + data_loader = fluid.io.DataLoader.from_generator( + capacity=64, feed_list=words, iterable=False) - words = fluid.layers.read_file(py_reader) init_width = 0.5 / embedding_size input_emb = fluid.embedding( input=words[0], @@ -104,7 +103,7 @@ def skip_gram_word2vec(dict_size, embedding_size, is_sparse=False, neg_num=5): fluid.layers.reduce_sum( neg_xent, dim=1)) avg_cost = fluid.layers.reduce_mean(cost) - return avg_cost, py_reader + return avg_cost, data_loader def infer_network(vocab_size, emb_size): diff --git a/PaddleRec/word2vec/train.py b/PaddleRec/word2vec/train.py index 929abf4dc4272fec154bc29415d9651c3dbd5b74..16de84d93247763ea56c44f0a18df70e2cee744c 100644 --- a/PaddleRec/word2vec/train.py +++ b/PaddleRec/word2vec/train.py @@ -121,10 +121,9 @@ def convert_python_to_tensor(weight, batch_size, sample_reader): return __reader__ -def train_loop(args, train_program, reader, py_reader, loss, trainer_id, +def train_loop(args, train_program, reader, data_loader, loss, trainer_id, weight): - - py_reader.decorate_tensor_provider( + data_loader.set_batch_generator( convert_python_to_tensor(weight, args.batch_size, reader.train())) place = fluid.CPUPlace() @@ -149,7 +148,7 @@ def train_loop(args, train_program, reader, py_reader, loss, trainer_id, exec_strategy=exec_strategy) for pass_id in range(args.num_passes): - py_reader.start() + data_loader.start() time.sleep(10) epoch_start = time.time() batch_id = 0 @@ -164,7 +163,7 @@ def train_loop(args, train_program, reader, py_reader, loss, trainer_id, logger.info( "TRAIN --> pass: {} batch: {} loss: {} reader queue:{}". format(pass_id, batch_id, - loss_val.mean(), py_reader.queue.size())) + loss_val.mean(), data_loader.queue.size())) if args.with_speed: if batch_id % 500 == 0 and batch_id != 0: elapsed = (time.time() - start) @@ -178,18 +177,18 @@ def train_loop(args, train_program, reader, py_reader, loss, trainer_id, model_dir = args.model_output_dir + '/pass-' + str( pass_id) + ('/batch-' + str(batch_id)) if trainer_id == 0: - fluid.io.save_params(executor=exe, dirname=model_dir) + fluid.save(fluid.default_main_program(), model_path=model_dir) print("model saved in %s" % model_dir) batch_id += 1 except fluid.core.EOFException: - py_reader.reset() + data_loader.reset() epoch_end = time.time() logger.info("Epoch: {0}, Train total expend: {1} ".format( pass_id, epoch_end - epoch_start)) model_dir = args.model_output_dir + '/pass-' + str(pass_id) if trainer_id == 0: - fluid.io.save_params(executor=exe, dirname=model_dir) + fluid.save(fluid.default_main_program(), model_path=model_dir) print("model saved in %s" % model_dir) @@ -215,7 +214,7 @@ def train(args): np_power = np.power(np.array(word2vec_reader.id_frequencys), 0.75) id_frequencys_pow = np_power / np_power.sum() - loss, py_reader = skip_gram_word2vec( + loss, data_loader = skip_gram_word2vec( word2vec_reader.dict_size, args.embedding_size, is_sparse=args.is_sparse, @@ -233,7 +232,7 @@ def train(args): # do local training logger.info("run local training") main_program = fluid.default_main_program() - train_loop(args, main_program, word2vec_reader, py_reader, loss, 0, + train_loop(args, main_program, word2vec_reader, data_loader, loss, 0, id_frequencys_pow)