diff --git a/PaddleNLP/PaddleTextGEN/seq2seq/infer.py b/PaddleNLP/PaddleTextGEN/seq2seq/infer.py index ab7daf6031ad8c4ff4324667d76e60a3bcc94c3e..921710259b12a647316c04070b57410054b69445 100644 --- a/PaddleNLP/PaddleTextGEN/seq2seq/infer.py +++ b/PaddleNLP/PaddleTextGEN/seq2seq/infer.py @@ -127,7 +127,8 @@ def infer(): dir_name = args.reload_model print("dir name", dir_name) - fluid.io.load_params(exe, dir_name) + dir_name = os.path.join(dir_name, "checkpoint") + fluid.load(main_program, dir_name, exe) train_data_iter = reader.get_data_iter(infer_data, 1, mode='eval') diff --git a/PaddleNLP/PaddleTextGEN/seq2seq/train.py b/PaddleNLP/PaddleTextGEN/seq2seq/train.py index e44d9a47692d4a527afc09486a02119d4037ea65..08c7d5a0a8ea0a1e6f48e3ae1da67ed362154658 100644 --- a/PaddleNLP/PaddleTextGEN/seq2seq/train.py +++ b/PaddleNLP/PaddleTextGEN/seq2seq/train.py @@ -229,10 +229,11 @@ def main(): % (epoch_id, epoch_time, sum(batch_times) / len(batch_times))) if not args.profile: - dir_name = os.path.join(args.model_path, - "epoch_" + str(epoch_id)) - print("begin to save", dir_name) - fluid.io.save_params(exe, dir_name, main_program=train_program) + save_path = os.path.join(args.model_path, + "epoch_" + str(epoch_id), + "checkpoint") + print("begin to save", save_path) + fluid.save(train_program, save_path) print("save finished") dev_ppl = eval(valid_data) print("dev ppl", dev_ppl) diff --git a/PaddleNLP/PaddleTextGEN/variational_seq2seq/infer.py b/PaddleNLP/PaddleTextGEN/variational_seq2seq/infer.py index c21fff3a5e13c3853c70988c458d12a18aef964c..2044602050fc5803cee8ef126ee37dc1641de856 100644 --- a/PaddleNLP/PaddleTextGEN/variational_seq2seq/infer.py +++ b/PaddleNLP/PaddleTextGEN/variational_seq2seq/infer.py @@ -88,7 +88,8 @@ def infer(): dir_name = args.reload_model print("dir name", dir_name) - fluid.io.load_params(exe, dir_name) + dir_name = os.path.join(dir_name, "checkpoint") + fluid.load(main_program, dir_name, exe) vocab, tar_id2vocab = get_vocab(args.dataset_prefix) infer_output = np.ones((batch_size, 1), dtype='int64') * BOS_ID diff --git a/PaddleNLP/PaddleTextGEN/variational_seq2seq/train.py b/PaddleNLP/PaddleTextGEN/variational_seq2seq/train.py index 98515a8329fba8508b01accbbed940ef2df65842..1a1883e74f3abc2adc0dece7c8cfdf6e281a431d 100644 --- a/PaddleNLP/PaddleTextGEN/variational_seq2seq/train.py +++ b/PaddleNLP/PaddleTextGEN/variational_seq2seq/train.py @@ -255,10 +255,11 @@ def main(): best_nll = test_nll best_ppl = test_ppl best_epoch_id = epoch_id - dir_name = os.path.join(args.model_path, - "epoch_" + str(best_epoch_id)) - print("save model {}".format(dir_name)) - fluid.io.save_params(exe, dir_name, main_program) + save_path = os.path.join(args.model_path, + "epoch_" + str(best_epoch_id), + "checkpoint") + print("save model {}".format(save_path)) + fluid.save(main_program, save_path) else: steps_not_improved += 1 if steps_not_improved == decay_ts: diff --git a/PaddleNLP/lexical_analysis/run_ernie_sequence_labeling.py b/PaddleNLP/lexical_analysis/run_ernie_sequence_labeling.py index 3ebed4c3a358039e5d9c63ae4ac41231fac8dbbb..1b809a28526fe2940dadef355551c63394aeac65 100644 --- a/PaddleNLP/lexical_analysis/run_ernie_sequence_labeling.py +++ b/PaddleNLP/lexical_analysis/run_ernie_sequence_labeling.py @@ -187,17 +187,17 @@ def do_train(args): end_time - start_time, train_pyreader.queue.size())) if steps % args.save_steps == 0: - save_path = os.path.join(args.model_save_dir, - "step_" + str(steps)) + save_path = os.path.join(args.model_save_dir, "step_" + str(steps), + "checkpoint") print("\tsaving model as %s" % (save_path)) - fluid.io.save_persistables(exe, save_path, train_program) + fluid.save(train_program, save_path) if steps % args.validation_steps == 0: evaluate(exe, test_program, test_pyreader, train_ret) - save_path = os.path.join(args.model_save_dir, "step_" + str(steps)) - fluid.io.save_persistables(exe, save_path, train_program) - + save_path = os.path.join(args.model_save_dir, "step_" + str(steps), + "checkpoint") + fluid.save(train_program, save_path) def do_eval(args): # init executor diff --git a/PaddleNLP/lexical_analysis/train.py b/PaddleNLP/lexical_analysis/train.py index cbd47e3e8e63e3be7ed0be8f7db84219eb43fe6f..5cc28987bb603e56fc33dd429eeab48bdbadf991 100644 --- a/PaddleNLP/lexical_analysis/train.py +++ b/PaddleNLP/lexical_analysis/train.py @@ -151,8 +151,9 @@ def do_train(args): # save checkpoints if step % args.save_steps == 0 and step != 0: save_path = os.path.join(args.model_save_dir, - "step_" + str(step)) - fluid.io.save_persistables(exe, save_path, train_program) + "step_" + str(step), + "checkpoint") + fluid.save(train_program, save_path) step += 1 if args.enable_ce: diff --git a/PaddleNLP/lexical_analysis/utils.py b/PaddleNLP/lexical_analysis/utils.py index d3ee614de23ad896eb826bbf86239bba5860a77d..c49843ae0a4b6b42e9742571867736e31101590a 100644 --- a/PaddleNLP/lexical_analysis/utils.py +++ b/PaddleNLP/lexical_analysis/utils.py @@ -199,23 +199,14 @@ def init_checkpoint(exe, init_checkpoint_path, main_program): """ assert os.path.exists( init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path - - def existed_persitables(var): - """ - If existed presitabels - """ - if not fluid.io.is_persistable(var): - return False - return os.path.exists(os.path.join(init_checkpoint_path, var.name)) - - fluid.io.load_vars( - exe, - init_checkpoint_path, - main_program=main_program, - predicate=existed_persitables) + + try: + checkpoint_path = os.path.join(init_checkpoint_path, "checkpoint") + fluid.load(main_program, checkpoint_path, exe) + except: + fluid.load(main_program, init_checkpoint_path, exe) print("Load model from {}".format(init_checkpoint_path)) - def init_pretraining_params(exe, pretraining_params_path, main_program, @@ -224,15 +215,6 @@ def init_pretraining_params(exe, assert os.path.exists(pretraining_params_path ), "[%s] cann't be found." % pretraining_params_path - def _existed_params(var): - if not isinstance(var, fluid.framework.Parameter): - return False - return os.path.exists(os.path.join(pretraining_params_path, var.name)) - - fluid.io.load_vars( - exe, - pretraining_params_path, - main_program=main_program, - predicate=_existed_params) + fluid.load(main_program, pretraining_params_path, exe) print("Load pretraining parameters from {}.".format( pretraining_params_path)) diff --git a/PaddleNLP/sentiment_classification/run_classifier.py b/PaddleNLP/sentiment_classification/run_classifier.py index 233a6582d2532f3a4966981771371c5c96290462..eb651716ce930e601c863c7c5e88bb3b5f889187 100644 --- a/PaddleNLP/sentiment_classification/run_classifier.py +++ b/PaddleNLP/sentiment_classification/run_classifier.py @@ -289,8 +289,9 @@ def main(args): if steps % args.save_steps == 0: save_path = os.path.join(args.checkpoints, - "step_" + str(steps)) - fluid.io.save_persistables(exe, save_path, train_program) + "step_" + str(steps), + "checkpoint") + fluid.save(train_program, save_path) if steps % args.validation_steps == 0: # evaluate dev set @@ -301,8 +302,9 @@ def main(args): "dev") except fluid.core.EOFException: - save_path = os.path.join(args.checkpoints, "step_" + str(steps)) - fluid.io.save_persistables(exe, save_path, train_program) + save_path = os.path.join(args.checkpoints, "step_" + str(steps), + "checkpoint") + fluid.save(train_program, save_path) train_reader.reset() break diff --git a/PaddleNLP/sentiment_classification/run_ernie_classifier.py b/PaddleNLP/sentiment_classification/run_ernie_classifier.py index 80f83fd11956817cfd0af01d1f4b3e596b85da75..13d1447185ea4b758c530d595407f937c95edf4d 100644 --- a/PaddleNLP/sentiment_classification/run_ernie_classifier.py +++ b/PaddleNLP/sentiment_classification/run_ernie_classifier.py @@ -353,8 +353,9 @@ def main(args): if steps % args.save_steps == 0: save_path = os.path.join(args.checkpoints, - "step_" + str(steps)) - fluid.io.save_persistables(exe, save_path, train_program) + "step_" + str(steps), + "checkpoint") + fluid.save(train_program, save_path) if steps % args.validation_steps == 0: # evaluate dev set @@ -364,8 +365,9 @@ def main(args): "dev") except fluid.core.EOFException: - save_path = os.path.join(args.checkpoints, "step_" + str(steps)) - fluid.io.save_persistables(exe, save_path, train_program) + save_path = os.path.join(args.checkpoints, "step_" + str(steps), + "checkpoint") + fluid.save(train_program, save_path) train_pyreader.reset() break diff --git a/PaddleNLP/sentiment_classification/utils.py b/PaddleNLP/sentiment_classification/utils.py index 9a6d648a0bff7fa968b9d41e474f5e196f42ca10..e6a2b6503ec89561f1d8314e09a3803c5ba0c915 100644 --- a/PaddleNLP/sentiment_classification/utils.py +++ b/PaddleNLP/sentiment_classification/utils.py @@ -63,20 +63,11 @@ def init_checkpoint(exe, init_checkpoint_path, main_program): """ assert os.path.exists( init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path - - def existed_persitables(var): - """ - If existed presitabels - """ - if not fluid.io.is_persistable(var): - return False - return os.path.exists(os.path.join(init_checkpoint_path, var.name)) - - fluid.io.load_vars( - exe, - init_checkpoint_path, - main_program=main_program, - predicate=existed_persitables) + try: + checkpoint_path = os.path.join(init_checkpoint_path, "checkpoint") + fluid.load(main_program, checkpoint_path, exe) + except: + fluid.load(main_program, init_checkpoint_path, exe) print("Load model from {}".format(init_checkpoint_path)) @@ -144,15 +135,6 @@ def init_pretraining_params(exe, assert os.path.exists(pretraining_params_path ), "[%s] cann't be found." % pretraining_params_path - def _existed_params(var): - if not isinstance(var, fluid.framework.Parameter): - return False - return os.path.exists(os.path.join(pretraining_params_path, var.name)) - - fluid.io.load_vars( - exe, - pretraining_params_path, - main_program=main_program, - predicate=_existed_params) + fluid.load(main_program, pretraining_params_path, exe) print("Load pretraining parameters from {}.".format( pretraining_params_path))