From fba10b6bb5dfbb4f5b5f10e946636655dc2ab1e0 Mon Sep 17 00:00:00 2001 From: Jiabin Yang Date: Fri, 7 Jun 2019 09:50:24 +0800 Subject: [PATCH] test=develop, refine api (#17883) * test=develop, refine api * test=develop, fix bug when error occured on save_persistable with no optimizer * test=develop, refine waring * test=develop, refine example code and comments --- python/paddle/fluid/dygraph/checkpoint.py | 87 ++++++++++--------- .../unittests/test_imperative_checkpoint.py | 2 +- .../test_imperative_save_load_optimizer.py | 4 +- 3 files changed, 50 insertions(+), 43 deletions(-) diff --git a/python/paddle/fluid/dygraph/checkpoint.py b/python/paddle/fluid/dygraph/checkpoint.py index 46cdada9e..528494055 100644 --- a/python/paddle/fluid/dygraph/checkpoint.py +++ b/python/paddle/fluid/dygraph/checkpoint.py @@ -24,13 +24,10 @@ import warnings __all__ = ['save_persistables', 'load_persistables'] -def save_persistables(model_dict, - optimizer=None, - dirname='save_dir', - filename=None): +def save_persistables(model_dict, dirname='save_dir', optimizers=None): """ This function filters out all variables in layer.parameters from the - give `layer` and then trys to load these variables from the folder + give `layer`, and optimizer's learning rate decay and then trys to load these variables from the folder `dirname` or the file `filename`. Use the `dirname` to specify the folder where persistable variables were @@ -43,9 +40,7 @@ def save_persistables(model_dict, be saved. If it is None, nothing will be deal. dirname(str): The directory path. - filename(str|None): The file which saved all variables. If variables were - saved in different files, set it to None. - Default: None + optimizers(fluid.Optimizer|list(fluid.Optimizer)|None): The optimizers to be saved Returns: @@ -57,7 +52,7 @@ def save_persistables(model_dict, num_layers=num_layers, num_steps=num_steps, init_scale=init_scale) - + sgd = fluid.optimizer.SGD(learning_rate=0.01) x_data = np.arange(12).reshape(4, 3).astype('int64') y_data = np.arange(1, 13).reshape(4, 3).astype('int64') x_data = x_data.reshape((-1, num_steps, 1)) @@ -72,12 +67,14 @@ def save_persistables(model_dict, init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model(x, y, init_hidden, init_cell) + dy_loss.backward() + sgd.minimize(dy_loss) + ptb_model.clear_gradient() param_path = "./my_paddle_model" - fluid.dygraph.save_persistables(ptb_model.state_dict(), dirname=param_path, - layer=ptb_model) + fluid.dygraph.save_persistables(ptb_model.state_dict(), dirname=param_path, sgd) """ if isinstance(model_dict, collections.OrderedDict): - _save_var_to_file(model_dict, optimizer, dirname, filename) + _save_var_to_file(model_dict, optimizers, dirname, None) def load_persistables(dirname='save_dir'): @@ -92,18 +89,19 @@ def load_persistables(dirname='save_dir'): Args: dirname(str): The directory path. default is save_dir - optimizer(Optimizer): Optimizer to be saved Returns: dict: The parameter-dict resumed from file + optimizer dict: The optimizer Examples: .. code-block:: python my_layer = layer(fluid.Layer) param_path = "./my_paddle_model" - - param_dict = fluid.dygraph.load_persistables(my_layer.parameters(), param_path) + sgd = SGDOptimizer(learning_rate=1e-3) + param_dict, optimizer_dict = fluid.dygraph.load_persistables(my_layer.parameters(), param_path) param_1 = param_dict['PtbModel_0.w_1'] + sgd.load(optimizer_dict) """ return _load_var_from_file(dirname) @@ -123,32 +121,38 @@ def _save_var_to_file(stat_dict, optimizers, file_dir, file_name): 'file_path': os.path.join(file_dir, os.path.normpath(each_var.name)) }) - if isinstance(optimizers, (list, tuple)): - optimizers = optimizers + + if optimizers is not None: + if isinstance(optimizers, (list, tuple)): + optimizers = optimizers + else: + optimizers = [optimizers] + if os.path.exists( + os.path.join(file_dir, os.path.normpath("optimizers"))): + pass + else: + os.mkdir(os.path.join(file_dir, os.path.normpath("optimizers"))) + for optimizer in optimizers: + if isinstance(optimizer._learning_rate, + learning_rate_scheduler.LearningRateDecay): + try: + f = open( + os.path.join(file_dir, "optimizers", + os.path.normpath(str(optimizer._name))), + "wb") + pickle.dump(optimizer._learning_rate, f, 2) + f.close() + except (): + raise IOError("Can't load %s", + os.path.join( + file_dir, "optimizers", + os.path.normpath(str(optimizer._name)))) + else: + warnings.warn( + "Optimizer not saved, Only optimizer with 'LearningRateDecay' under DyGraph mode need to be saved" + ) else: - optimizers = [optimizers] - if os.path.exists(os.path.join(file_dir, os.path.normpath("optimizers"))): pass - else: - os.mkdir(os.path.join(file_dir, os.path.normpath("optimizers"))) - for optimizer in optimizers: - if isinstance(optimizer._learning_rate, - learning_rate_scheduler.LearningRateDecay): - try: - f = open( - os.path.join(file_dir, "optimizers", - os.path.normpath(str(optimizer._name))), "wb") - pickle.dump(optimizer._learning_rate, f, 2) - f.close() - except (): - raise IOError("Can't load %s", - os.path.join( - file_dir, "optimizers", - os.path.normpath(str(optimizer._name)))) - else: - warnings.warn( - "Optimizer not saved, Only optimizer with 'LearningRateDecay' under DyGraph mode need to be saved" - ) if file_name is not None: save_var_list = [] @@ -213,7 +217,10 @@ def _load_var_from_file(file_dir): file_dir, "optimizers", os.path.normpath(str(optimizer._name)))) if len(load_optimizer_map) == 0: - warnings.warn("No optimizer loaded") + print( + "No optimizer loaded. If you didn't save optimizer, please ignore this. The program can still work with new optimizer. " + ) + pass return load_var_map, load_optimizer_map diff --git a/python/paddle/fluid/tests/unittests/test_imperative_checkpoint.py b/python/paddle/fluid/tests/unittests/test_imperative_checkpoint.py index 6a4330df7..b7c3695ee 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_checkpoint.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_checkpoint.py @@ -144,7 +144,7 @@ class TestDygraphCheckpoint(unittest.TestCase): avg_loss.backward() sgd.minimize(avg_loss) - fluid.dygraph.save_persistables(mnist.state_dict(), [sgd], + fluid.dygraph.save_persistables(mnist.state_dict(), "save_dir") mnist.clear_gradients() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_save_load_optimizer.py b/python/paddle/fluid/tests/unittests/test_imperative_save_load_optimizer.py index 4aad828b0..a65db433e 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_save_load_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_save_load_optimizer.py @@ -78,8 +78,8 @@ class TestImperativeOptimizerBase(unittest.TestCase): optimizer.minimize(avg_loss) optimizer2.minimize(avg_loss) mlp.clear_gradients() - fluid.dygraph.save_persistables( - mlp.state_dict(), [optimizer, optimizer2], "save_dir_2") + fluid.dygraph.save_persistables(mlp.state_dict(), "save_dir_2", + [optimizer, optimizer2]) if batch_id == 2: break -- GitLab