From 4d28bb18c8662aa499ac3957f13458882914118c Mon Sep 17 00:00:00 2001 From: Xing Wu Date: Mon, 24 Feb 2020 14:22:11 +0800 Subject: [PATCH] update save/load for v1.7 (#4332) * update save/load for v1.7 * update load api * update load api --- PaddleNLP/PaddleTextGEN/seq2seq/infer.py | 3 +- PaddleNLP/PaddleTextGEN/seq2seq/train.py | 9 +++--- .../variational_seq2seq/infer.py | 3 +- .../variational_seq2seq/train.py | 9 +++--- .../run_ernie_sequence_labeling.py | 12 +++---- PaddleNLP/lexical_analysis/train.py | 5 +-- PaddleNLP/lexical_analysis/utils.py | 32 ++++--------------- .../run_classifier.py | 10 +++--- .../run_ernie_classifier.py | 10 +++--- PaddleNLP/sentiment_classification/utils.py | 30 ++++------------- 10 files changed, 48 insertions(+), 75 deletions(-) diff --git a/PaddleNLP/PaddleTextGEN/seq2seq/infer.py b/PaddleNLP/PaddleTextGEN/seq2seq/infer.py index ab7daf60..92171025 100644 --- a/PaddleNLP/PaddleTextGEN/seq2seq/infer.py +++ b/PaddleNLP/PaddleTextGEN/seq2seq/infer.py @@ -127,7 +127,8 @@ def infer(): dir_name = args.reload_model print("dir name", dir_name) - fluid.io.load_params(exe, dir_name) + dir_name = os.path.join(dir_name, "checkpoint") + fluid.load(main_program, dir_name, exe) train_data_iter = reader.get_data_iter(infer_data, 1, mode='eval') diff --git a/PaddleNLP/PaddleTextGEN/seq2seq/train.py b/PaddleNLP/PaddleTextGEN/seq2seq/train.py index e44d9a47..08c7d5a0 100644 --- a/PaddleNLP/PaddleTextGEN/seq2seq/train.py +++ b/PaddleNLP/PaddleTextGEN/seq2seq/train.py @@ -229,10 +229,11 @@ def main(): % (epoch_id, epoch_time, sum(batch_times) / len(batch_times))) if not args.profile: - dir_name = os.path.join(args.model_path, - "epoch_" + str(epoch_id)) - print("begin to save", dir_name) - fluid.io.save_params(exe, dir_name, main_program=train_program) + save_path = os.path.join(args.model_path, + "epoch_" + str(epoch_id), + "checkpoint") + print("begin to save", save_path) + fluid.save(train_program, save_path) print("save finished") dev_ppl = eval(valid_data) print("dev ppl", dev_ppl) diff --git a/PaddleNLP/PaddleTextGEN/variational_seq2seq/infer.py b/PaddleNLP/PaddleTextGEN/variational_seq2seq/infer.py index c21fff3a..20446020 100644 --- a/PaddleNLP/PaddleTextGEN/variational_seq2seq/infer.py +++ b/PaddleNLP/PaddleTextGEN/variational_seq2seq/infer.py @@ -88,7 +88,8 @@ def infer(): dir_name = args.reload_model print("dir name", dir_name) - fluid.io.load_params(exe, dir_name) + dir_name = os.path.join(dir_name, "checkpoint") + fluid.load(main_program, dir_name, exe) vocab, tar_id2vocab = get_vocab(args.dataset_prefix) infer_output = np.ones((batch_size, 1), dtype='int64') * BOS_ID diff --git a/PaddleNLP/PaddleTextGEN/variational_seq2seq/train.py b/PaddleNLP/PaddleTextGEN/variational_seq2seq/train.py index 98515a83..1a1883e7 100644 --- a/PaddleNLP/PaddleTextGEN/variational_seq2seq/train.py +++ b/PaddleNLP/PaddleTextGEN/variational_seq2seq/train.py @@ -255,10 +255,11 @@ def main(): best_nll = test_nll best_ppl = test_ppl best_epoch_id = epoch_id - dir_name = os.path.join(args.model_path, - "epoch_" + str(best_epoch_id)) - print("save model {}".format(dir_name)) - fluid.io.save_params(exe, dir_name, main_program) + save_path = os.path.join(args.model_path, + "epoch_" + str(best_epoch_id), + "checkpoint") + print("save model {}".format(save_path)) + fluid.save(main_program, save_path) else: steps_not_improved += 1 if steps_not_improved == decay_ts: diff --git a/PaddleNLP/lexical_analysis/run_ernie_sequence_labeling.py b/PaddleNLP/lexical_analysis/run_ernie_sequence_labeling.py index 3ebed4c3..1b809a28 100644 --- a/PaddleNLP/lexical_analysis/run_ernie_sequence_labeling.py +++ b/PaddleNLP/lexical_analysis/run_ernie_sequence_labeling.py @@ -187,17 +187,17 @@ def do_train(args): end_time - start_time, train_pyreader.queue.size())) if steps % args.save_steps == 0: - save_path = os.path.join(args.model_save_dir, - "step_" + str(steps)) + save_path = os.path.join(args.model_save_dir, "step_" + str(steps), + "checkpoint") print("\tsaving model as %s" % (save_path)) - fluid.io.save_persistables(exe, save_path, train_program) + fluid.save(train_program, save_path) if steps % args.validation_steps == 0: evaluate(exe, test_program, test_pyreader, train_ret) - save_path = os.path.join(args.model_save_dir, "step_" + str(steps)) - fluid.io.save_persistables(exe, save_path, train_program) - + save_path = os.path.join(args.model_save_dir, "step_" + str(steps), + "checkpoint") + fluid.save(train_program, save_path) def do_eval(args): # init executor diff --git a/PaddleNLP/lexical_analysis/train.py b/PaddleNLP/lexical_analysis/train.py index cbd47e3e..5cc28987 100644 --- a/PaddleNLP/lexical_analysis/train.py +++ b/PaddleNLP/lexical_analysis/train.py @@ -151,8 +151,9 @@ def do_train(args): # save checkpoints if step % args.save_steps == 0 and step != 0: save_path = os.path.join(args.model_save_dir, - "step_" + str(step)) - fluid.io.save_persistables(exe, save_path, train_program) + "step_" + str(step), + "checkpoint") + fluid.save(train_program, save_path) step += 1 if args.enable_ce: diff --git a/PaddleNLP/lexical_analysis/utils.py b/PaddleNLP/lexical_analysis/utils.py index d3ee614d..c49843ae 100644 --- a/PaddleNLP/lexical_analysis/utils.py +++ b/PaddleNLP/lexical_analysis/utils.py @@ -199,23 +199,14 @@ def init_checkpoint(exe, init_checkpoint_path, main_program): """ assert os.path.exists( init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path - - def existed_persitables(var): - """ - If existed presitabels - """ - if not fluid.io.is_persistable(var): - return False - return os.path.exists(os.path.join(init_checkpoint_path, var.name)) - - fluid.io.load_vars( - exe, - init_checkpoint_path, - main_program=main_program, - predicate=existed_persitables) + + try: + checkpoint_path = os.path.join(init_checkpoint_path, "checkpoint") + fluid.load(main_program, checkpoint_path, exe) + except: + fluid.load(main_program, init_checkpoint_path, exe) print("Load model from {}".format(init_checkpoint_path)) - def init_pretraining_params(exe, pretraining_params_path, main_program, @@ -224,15 +215,6 @@ def init_pretraining_params(exe, assert os.path.exists(pretraining_params_path ), "[%s] cann't be found." % pretraining_params_path - def _existed_params(var): - if not isinstance(var, fluid.framework.Parameter): - return False - return os.path.exists(os.path.join(pretraining_params_path, var.name)) - - fluid.io.load_vars( - exe, - pretraining_params_path, - main_program=main_program, - predicate=_existed_params) + fluid.load(main_program, pretraining_params_path, exe) print("Load pretraining parameters from {}.".format( pretraining_params_path)) diff --git a/PaddleNLP/sentiment_classification/run_classifier.py b/PaddleNLP/sentiment_classification/run_classifier.py index 233a6582..eb651716 100644 --- a/PaddleNLP/sentiment_classification/run_classifier.py +++ b/PaddleNLP/sentiment_classification/run_classifier.py @@ -289,8 +289,9 @@ def main(args): if steps % args.save_steps == 0: save_path = os.path.join(args.checkpoints, - "step_" + str(steps)) - fluid.io.save_persistables(exe, save_path, train_program) + "step_" + str(steps), + "checkpoint") + fluid.save(train_program, save_path) if steps % args.validation_steps == 0: # evaluate dev set @@ -301,8 +302,9 @@ def main(args): "dev") except fluid.core.EOFException: - save_path = os.path.join(args.checkpoints, "step_" + str(steps)) - fluid.io.save_persistables(exe, save_path, train_program) + save_path = os.path.join(args.checkpoints, "step_" + str(steps), + "checkpoint") + fluid.save(train_program, save_path) train_reader.reset() break diff --git a/PaddleNLP/sentiment_classification/run_ernie_classifier.py b/PaddleNLP/sentiment_classification/run_ernie_classifier.py index 80f83fd1..13d14471 100644 --- a/PaddleNLP/sentiment_classification/run_ernie_classifier.py +++ b/PaddleNLP/sentiment_classification/run_ernie_classifier.py @@ -353,8 +353,9 @@ def main(args): if steps % args.save_steps == 0: save_path = os.path.join(args.checkpoints, - "step_" + str(steps)) - fluid.io.save_persistables(exe, save_path, train_program) + "step_" + str(steps), + "checkpoint") + fluid.save(train_program, save_path) if steps % args.validation_steps == 0: # evaluate dev set @@ -364,8 +365,9 @@ def main(args): "dev") except fluid.core.EOFException: - save_path = os.path.join(args.checkpoints, "step_" + str(steps)) - fluid.io.save_persistables(exe, save_path, train_program) + save_path = os.path.join(args.checkpoints, "step_" + str(steps), + "checkpoint") + fluid.save(train_program, save_path) train_pyreader.reset() break diff --git a/PaddleNLP/sentiment_classification/utils.py b/PaddleNLP/sentiment_classification/utils.py index 9a6d648a..e6a2b650 100644 --- a/PaddleNLP/sentiment_classification/utils.py +++ b/PaddleNLP/sentiment_classification/utils.py @@ -63,20 +63,11 @@ def init_checkpoint(exe, init_checkpoint_path, main_program): """ assert os.path.exists( init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path - - def existed_persitables(var): - """ - If existed presitabels - """ - if not fluid.io.is_persistable(var): - return False - return os.path.exists(os.path.join(init_checkpoint_path, var.name)) - - fluid.io.load_vars( - exe, - init_checkpoint_path, - main_program=main_program, - predicate=existed_persitables) + try: + checkpoint_path = os.path.join(init_checkpoint_path, "checkpoint") + fluid.load(main_program, checkpoint_path, exe) + except: + fluid.load(main_program, init_checkpoint_path, exe) print("Load model from {}".format(init_checkpoint_path)) @@ -144,15 +135,6 @@ def init_pretraining_params(exe, assert os.path.exists(pretraining_params_path ), "[%s] cann't be found." % pretraining_params_path - def _existed_params(var): - if not isinstance(var, fluid.framework.Parameter): - return False - return os.path.exists(os.path.join(pretraining_params_path, var.name)) - - fluid.io.load_vars( - exe, - pretraining_params_path, - main_program=main_program, - predicate=_existed_params) + fluid.load(main_program, pretraining_params_path, exe) print("Load pretraining parameters from {}.".format( pretraining_params_path)) -- GitLab