From 5119f262ecd4fa9879cc6d1efd6197383c312149 Mon Sep 17 00:00:00 2001 From: hong <43953930+phlrain@users.noreply.github.com> Date: Wed, 30 Oct 2019 10:16:36 +0800 Subject: [PATCH] Cherry pick save load new feature (#20877) * Serialize to pickle format (#20820) test=develop * save load problem fix and new feature add (#20823) * fix persistable; * fix save load bugs; test=develop * fix bug; test=develop * add example for new io api; test=develop * addd example; test=develop --- paddle/fluid/framework/executor.h | 2 + paddle/fluid/pybind/pybind.cc | 59 +++- python/paddle/fluid/__init__.py | 2 +- python/paddle/fluid/core.py | 2 + python/paddle/fluid/io.py | 217 +++++++++++- .../tests/unittests/test_static_save_load.py | 325 +++++++++++++++++- 6 files changed, 570 insertions(+), 37 deletions(-) diff --git a/paddle/fluid/framework/executor.h b/paddle/fluid/framework/executor.h index 6ca50b70031..13bbe29d736 100644 --- a/paddle/fluid/framework/executor.h +++ b/paddle/fluid/framework/executor.h @@ -126,6 +126,8 @@ class Executor { Scope* scope, Dataset* dataset); void RunFromDataset(std::shared_ptr trainer); + const platform::Place GetPlace() const { return place_; } + private: const platform::Place place_; }; diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 7161efeb539..59b7fbd6ac0 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -237,6 +237,56 @@ static std::vector inline GetNameList( return vec_res; } +static void inline CreateVariableIfNotExit( + const py::handle &py_handle, const framework::Scope &scope, + const framework::Executor *exe = nullptr) { + std::vector vec_res; + + PyObject *py_obj = py_handle.ptr(); // get underlying PyObject + // Python None is not nullptr in C++! + if (!py_obj || py_obj == Py_None) { + PADDLE_THROW("Save parameter list is None"); + } + + if (PyList_Check(py_obj)) { + size_t len = PyList_GET_SIZE(py_obj); + + vec_res.reserve(len); + + const char *kNameField = "name"; + const char *kVarDescField = "desc"; + + for (size_t i = 0; i < len; ++i) { + PyObject *py_name = + PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField); + PADDLE_ENFORCE_NOT_NULL(py_name); + auto para_name = PyObjectCast(py_name); + Py_DECREF(py_name); + + auto var = scope.FindVar(para_name); + if (var == nullptr) { + PADDLE_ENFORCE_NE(exe, nullptr, + "Parameter not Initialized, " + "Please set argument [executor] not None " + "or run startup program first"); + PyObject *py_var_desc = + PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kVarDescField); + PADDLE_ENFORCE_NOT_NULL(py_var_desc); + auto var_desc = PyObjectCast(py_var_desc); + Py_DECREF(py_var_desc); + var = const_cast(&scope)->Var(para_name); + auto *tensor_temp = var->GetMutable(); + tensor_temp->Resize(framework::make_ddim(var_desc.GetShape())); + tensor_temp->mutable_data(exe->GetPlace(), var_desc.GetDataType()); + } + } + } else { + PADDLE_THROW("Set parameter should be a list"); + } + + return; +} + #ifdef PADDLE_WITH_AVX PYBIND11_MODULE(core_avx, m) { #else @@ -285,11 +335,18 @@ PYBIND11_MODULE(core_noavx, m) { m.def("_load_static_dict", [](const std::string &str_file_name, const py::handle &vec_var_list, - const Scope &scope) { + const Scope &scope, const Executor *executor) { std::vector vec_name_list = GetNameList(vec_var_list); + CreateVariableIfNotExit(vec_var_list, scope, executor); LoadStaticNameListFromDisk(str_file_name, vec_name_list, scope); }); + m.def("_create_loaded_parameter", + [](const py::handle &vec_var_list, const Scope &scope, + const Executor *executor) { + CreateVariableIfNotExit(vec_var_list, scope, executor); + }); + m.def("_save_dygraph_dict", [](const std::string &str_file_name, const PyNameVarBaseMap &state_dict) { auto vec_var_base_list = GetVarBaseList(state_dict); diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index 79c05d1e7b7..6265932b09e 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -86,7 +86,7 @@ from paddle.fluid.layers.math_op_patch import monkey_patch_variable from . import install_check from .dygraph.nn import * from .dygraph.layers import * -from .io import save, load +from .io import save, load, load_program_state, set_program_state from .dygraph.checkpoint import save_dygraph, load_dygraph Tensor = LoDTensor diff --git a/python/paddle/fluid/core.py b/python/paddle/fluid/core.py index e241c66501d..b0908534a7c 100644 --- a/python/paddle/fluid/core.py +++ b/python/paddle/fluid/core.py @@ -181,6 +181,7 @@ if avx_supported(): from .core_avx import _load_static_dict from .core_avx import _save_dygraph_dict from .core_avx import _load_dygraph_dict + from .core_avx import _create_loaded_parameter except Exception as e: if has_avx_core: raise e @@ -214,6 +215,7 @@ if load_noavx: from .core_noavx import _load_static_dict from .core_noavx import _save_dygraph_dict from .core_noavx import _load_dygraph_dict + from .core_noavx import _create_loaded_parameter except Exception as e: if has_noavx_core: sys.stderr.write( diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index d57aa0a2489..6f58a700a53 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -19,6 +19,7 @@ import errno import warnings import six import logging +import pickle from functools import reduce import numpy as np @@ -40,9 +41,19 @@ from .. import compat as cpt batch = paddle.batch __all__ = [ - 'save_vars', 'save_params', 'save_persistables', 'load_vars', 'load_params', - 'load_persistables', 'save_inference_model', 'load_inference_model', - 'batch', 'save', 'load' + 'save_vars', + 'save_params', + 'save_persistables', + 'load_vars', + 'load_params', + 'load_persistables', + 'save_inference_model', + 'load_inference_model', + 'batch', + 'save', + 'load', + 'load_program_state', + 'set_program_state', ] + reader.__all__ + paddle.reader.__all__ _logger = get_logger( @@ -96,7 +107,10 @@ def is_persistable(var): def is_belong_to_optimizer(var): - return var.belong_to_optimizer + if not isinstance(var, Parameter): + return is_persistable(var) + + return False def _clone_var_in_block_(block, var): @@ -1505,15 +1519,21 @@ def save(program, model_path): assert base_name != "", \ "model_path MUST be format of dirname/filename [dirname\\filename in Window], Now filename is empty str" + def get_tensor(var): + t = global_scope().find_var(var.name).get_tensor() + return np.array(t) + parameter_list = list(filter(is_parameter, program.list_vars())) - paddle.fluid.core._save_static_dict(model_path + ".pdparams", - parameter_list, global_scope()) + param_dict = {p.name: get_tensor(p) for p in parameter_list} + with open(model_path + ".pdparams", 'wb') as f: + pickle.dump(param_dict, f) optimizer_var_list = list( filter(is_belong_to_optimizer, program.list_vars())) - paddle.fluid.core._save_static_dict(model_path + ".pdopt", - optimizer_var_list, global_scope()) + opt_dict = {p.name: get_tensor(p) for p in optimizer_var_list} + with open(model_path + ".pdopt", 'wb') as f: + pickle.dump(opt_dict, f) main_program = program.clone() program.desc.flush() @@ -1524,16 +1544,16 @@ def save(program, model_path): f.write(program.desc.serialize_to_string()) -def load(program, model_path): +def load(program, model_path, executor=None): """ This function filter out parameters and optimizer information from program, and then get corresponding value from file. - An exception will throw if shape or dtype of the parameters is not match between program and loaded file. - - NOTICE: This function MUST called after run start_up_program + An exception will throw if shape or dtype of the parameters is not match. Args: - program: The program to be load - model_path: The file prefix store the program + program(Program): The program will be loaded + model_path(str): The file prefix store the program + executor(Executor, optional): The executor used for initialize the parameter + When startup program is not run. Returns: None @@ -1550,13 +1570,39 @@ def load(program, model_path): """ + assert executor is None or isinstance(executor, Executor) + parameter_file_name = model_path + ".pdparams" assert os.path.exists(parameter_file_name), \ - "Parameter file [{}] not exits".format( parameter_file_name) + "Parameter file [{}] not exits".format(parameter_file_name) + + def set_var(var, ndarray): + t = global_scope().find_var(var.name).get_tensor() + p = t._place() + if p.is_cpu_place(): + place = paddle.fluid.CPUPlace() + elif p.is_cuda_pinned_place(): + place = paddle.fluid.CUDAPinnedPlace() + else: + p = paddle.fluid.core.Place() + p.set_place(t._place()) + place = paddle.fluid.CUDAPlace(p.gpu_device_id()) + + t.set(ndarray, place) parameter_list = list(filter(is_parameter, program.list_vars())) - paddle.fluid.core._load_static_dict(parameter_file_name, parameter_list, - global_scope()) + + if executor: + paddle.fluid.core._create_loaded_parameter(parameter_list, + global_scope(), + executor._default_executor) + with open(parameter_file_name, 'rb') as f: + load_dict = pickle.load(f) + for v in parameter_list: + assert v.name in load_dict, \ + "Can not find [{}] in model file [{}]".format( + v.name, parameter_file_name) + set_var(v, load_dict[v.name]) optimizer_var_list = list( filter(is_belong_to_optimizer, program.list_vars())) @@ -1565,5 +1611,138 @@ def load(program, model_path): opt_file_name = model_path + ".pdopt" assert os.path.exists(opt_file_name), \ "Optimizer file [{}] not exits".format( opt_file_name) - paddle.fluid.core._load_static_dict(opt_file_name, optimizer_var_list, - global_scope()) + + if executor: + paddle.fluid.core._create_loaded_parameter( + optimizer_var_list, global_scope(), executor._default_executor) + + with open(opt_file_name, 'rb') as f: + load_dict = pickle.load(f) + for v in optimizer_var_list: + assert v.name in load_dict, \ + "Can not find [{}] in model file [{}]".format( + v.name, opt_file_name) + set_var(v, load_dict[v.name]) + + +def load_program_state(model_path): + """ + Load program state from local file + + Args: + model_path(str): The file prefix store the program + Returns: + state_dict(dict): the dict store Parameter and optimizer information + + Examples: + .. code-block:: python + + import paddle.fluid as fluid + x = fluid.data( name="x", shape=[10, 10], dtype='float32') + y = fluid.layers.fc( x, 10) + z = fluid.layers.fc( y, 10) + + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run( fluid.default_startup_program() ) + prog = fluid.default_main_program() + + fluid.save( prog, "./temp") + program_state = fluid.load_program_state( "./temp") + + fluid.set_program_state( prog, program_state) + + """ + parameter_file_name = model_path + ".pdparams" + assert os.path.exists(parameter_file_name), \ + "Parameter file [{}] not exits".format( parameter_file_name) + + with open(parameter_file_name, 'rb') as f: + para_dict = pickle.load(f) + + opt_file_name = model_path + ".pdopt" + if os.path.exists(opt_file_name): + with open(opt_file_name, 'rb') as f: + opti_dict = pickle.load(f) + + para_dict.update(opti_dict) + + return para_dict + + +def set_program_state(program, state_dict): + """ + Set program parameter from state_dict + + An exception will throw if shape or dtype of the parameters is not match. + + NOTICE: This function MUST called after run start_up_program + + Args: + program(Program): The program to be set + state_dict(dict): the dict store Parameter and optimizer information + Returns: + None + + Examples: + .. code-block:: python + + import paddle.fluid as fluid + x = fluid.data( name="x", shape=[10, 10], dtype='float32') + y = fluid.layers.fc( x, 10) + z = fluid.layers.fc( y, 10) + + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run( fluid.default_startup_program() ) + prog = fluid.default_main_program() + + fluid.save( prog, "./temp") + program_state = fluid.load_program_state( "./temp") + + """ + parameter_list = list(filter(is_persistable, program.list_vars())) + + used_para_list = {} + for para in parameter_list: + var_temp = paddle.fluid.global_scope().find_var(para.name) + assert var_temp != None, \ + "Variable [ {} ] Not found, Please make sure run startup program".format( para.name ) + if para.name in state_dict: + # set value from state dict + orig_para_np = np.array(var_temp.get_tensor()) + new_para_np = state_dict[para.name] + assert orig_para_np.shape == new_para_np.shape, \ + "Shape not matching: the Program requires a parameter with a shape of ({}), " \ + "while the loaded parameter (namely [ {} ]) has a shape of ({})." \ + .format(orig_para_np.shape, para.name, new_para_np.shape) + assert orig_para_np.dtype == new_para_np.dtype, \ + "Dtype not matching: the Program requires a parameter with a dtype of ({}), " \ + "while the loaded parameter (namely [ {} ]) has a dtype of ({})." \ + .format(orig_para_np.dtype, para.name, new_para_np.dtype) + + ten = var_temp.get_tensor() + ten_place = ten._place() + + assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \ + "Place not support, only support CPUPlace and GPUPlace, now is {}".format( str(ten_place)) + py_place = paddle.fluid.CPUPlace() + if ten_place.is_cuda_pinned_place(): + place = paddle.fluid.CUDAPinnedPlace() + elif ten_place.is_gpu_place(): + p = paddle.fluid.core.Place() + p.set_place(ten_place) + py_place = paddle.fluid.CUDAPlace(p.gpu_device_id()) + + ten.set(new_para_np, py_place) + + used_para_list[para.name] = 1 + + unused_para_list = [] + for k, v in state_dict.items(): + if k not in used_para_list: + unused_para_list.append(k) + if len(unused_para_list) > 0: + warnings.warn( + "This list is not set, Because of Paramerter not found in program. There are: {}". + format(" ".join(unused_para_list))) diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load.py b/python/paddle/fluid/tests/unittests/test_static_save_load.py index 5d54c9ebc81..7cfd9bd5a27 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load.py @@ -15,6 +15,7 @@ from __future__ import print_function import unittest +import paddle import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.dygraph.nn import Embedding @@ -22,8 +23,10 @@ import paddle.fluid.framework as framework from paddle.fluid.optimizer import Adam from paddle.fluid.dygraph.base import to_variable from test_imperative_base import new_program_scope +from paddle.fluid.executor import global_scope import numpy as np import six +import pickle class SimpleLSTMRNN(fluid.Layer): @@ -210,7 +213,7 @@ class PtbModel(fluid.Layer): return loss, last_hidden, last_cell -class TestDygraphPtbRnn(unittest.TestCase): +class TestSaveLoadBase(unittest.TestCase): def test_ptb_rnn_cpu_float32(self): seed = 90 hidden_size = 10 @@ -281,8 +284,7 @@ class TestDygraphPtbRnn(unittest.TestCase): main_program = framework.default_main_program() base_map = {} for var in main_program.list_vars(): - if isinstance(var, - framework.Parameter) or var.belong_to_optimizer: + if isinstance(var, framework.Parameter) or var.persistable: t = np.array(fluid.global_scope().find_var(var.name) .get_tensor()) # make sure all the paramerter or optimzier var have been update @@ -293,8 +295,7 @@ class TestDygraphPtbRnn(unittest.TestCase): # set var to zero for var in main_program.list_vars(): - if isinstance(var, - framework.Parameter) or var.belong_to_optimizer: + if isinstance(var, framework.Parameter) or var.persistable: ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) @@ -303,18 +304,17 @@ class TestDygraphPtbRnn(unittest.TestCase): # make sure all the paramerter or optimzier var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "./test_1") + fluid.load(main_program, "./test_1", exe) for var in main_program.list_vars(): - if isinstance(var, - framework.Parameter) or var.belong_to_optimizer: + if isinstance(var, framework.Parameter) or var.persistable: new_t = np.array(fluid.global_scope().find_var(var.name) .get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) -class TestDygraphPtbRnnPartial(unittest.TestCase): +class TestSaveLoadPartial(unittest.TestCase): def test_ptb_rnn_cpu_float32(self): seed = 90 hidden_size = 10 @@ -393,8 +393,7 @@ class TestDygraphPtbRnnPartial(unittest.TestCase): main_program = framework.default_main_program() base_map = {} for var in main_program.list_vars(): - if isinstance(var, - framework.Parameter) or var.belong_to_optimizer: + if isinstance(var, framework.Parameter) or var.persistable: t = np.array(fluid.global_scope().find_var(var.name) .get_tensor()) # make sure all the paramerter or optimzier var have been update @@ -405,8 +404,7 @@ class TestDygraphPtbRnnPartial(unittest.TestCase): # set var to zero for var in main_program.list_vars(): - if isinstance(var, - framework.Parameter) or var.belong_to_optimizer: + if isinstance(var, framework.Parameter) or var.persistable: ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) @@ -415,11 +413,10 @@ class TestDygraphPtbRnnPartial(unittest.TestCase): # make sure all the paramerter or optimzier var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(test_program, "./test_1") + fluid.load(test_program, "./test_1", None) for var in test_program.list_vars(): - if isinstance(var, - framework.Parameter) or var.belong_to_optimizer: + if isinstance(var, framework.Parameter) or var.persistable: print(var.name) new_t = np.array(fluid.global_scope().find_var(var.name) .get_tensor()) @@ -427,5 +424,301 @@ class TestDygraphPtbRnnPartial(unittest.TestCase): self.assertTrue(np.array_equal(new_t, base_t)) +class TestSaveLoadSetStateDict(unittest.TestCase): + def test_ptb_rnn_cpu_float32(self): + seed = 90 + hidden_size = 10 + vocab_size = 1000 + num_layers = 1 + num_steps = 3 + init_scale = 0.1 + batch_size = 4 + batch_num = 200 + + with new_program_scope(): + fluid.default_startup_program().random_seed = seed + fluid.default_main_program().random_seed = seed + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale) + + place = fluid.CPUPlace() if not core.is_compiled_with_cuda( + ) else fluid.CUDAPlace(0) + exe = fluid.Executor(place) + sgd = Adam(learning_rate=1e-3) + x = fluid.layers.data( + name="x", shape=[-1, num_steps, 1], dtype='int64') + y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32') + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32') + + static_loss, static_last_hidden, static_last_cell = ptb_model( + x, y, init_hidden, init_cell) + sgd.minimize(static_loss) + static_param_updated = dict() + static_param_init = dict() + + out = exe.run(framework.default_startup_program()) + + static_loss_value = None + static_last_cell_value = None + static_last_hidden_value = None + for i in range(batch_num): + x_data = np.arange(12).reshape(4, 3).astype('int64') + y_data = np.arange(1, 13).reshape(4, 3).astype('int64') + x_data = x_data.reshape((-1, num_steps, 1)) + y_data = y_data.reshape((-1, 1)) + init_hidden_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32') + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32') + fetch_list = [static_loss, static_last_hidden, static_last_cell] + out = exe.run(fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data + }, + fetch_list=fetch_list) + static_loss_value = out[0] + static_last_hidden_value = out[1] + static_last_cell_value = out[2] + + # get value before save + main_program = framework.default_main_program() + base_map = {} + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimzier var have been update + self.assertTrue(np.sum(np.abs(t)) != 0) + base_map[var.name] = t + + fluid.save(main_program, "./test_1") + + # set var to zero + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + ten = fluid.global_scope().find_var(var.name).get_tensor() + ten.set(np.zeros_like(np.array(ten)), place) + + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimzier var have been set to zero + self.assertTrue(np.sum(np.abs(new_t)) == 0) + + fluid.load(main_program, "./test_1", exe) + + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + base_t = base_map[var.name] + self.assertTrue(np.array_equal(new_t, base_t)) + + +class TestProgramStatePartial(unittest.TestCase): + def test_ptb_rnn_cpu_float32(self): + seed = 90 + hidden_size = 10 + vocab_size = 1000 + num_layers = 1 + num_steps = 3 + init_scale = 0.1 + batch_size = 4 + batch_num = 200 + + with new_program_scope(): + fluid.default_startup_program().random_seed = seed + fluid.default_main_program().random_seed = seed + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale) + + place = fluid.CPUPlace() if not core.is_compiled_with_cuda( + ) else fluid.CUDAPlace(0) + exe = fluid.Executor(place) + sgd = Adam(learning_rate=1e-3) + x = fluid.layers.data( + name="x", shape=[-1, num_steps, 1], dtype='int64') + y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32') + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32') + + static_loss, static_last_hidden, static_last_cell = ptb_model( + x, y, init_hidden, init_cell) + + test_program = fluid.default_main_program().clone(for_test=True) + + add_1 = fluid.layers.fc(static_last_hidden, + size=hidden_size, + num_flatten_dims=2, + bias_attr=False) + + sgd.minimize(static_loss) + static_param_updated = dict() + static_param_init = dict() + + out = exe.run(framework.default_startup_program()) + + static_loss_value = None + static_last_cell_value = None + static_last_hidden_value = None + for i in range(batch_num): + x_data = np.arange(12).reshape(4, 3).astype('int64') + y_data = np.arange(1, 13).reshape(4, 3).astype('int64') + x_data = x_data.reshape((-1, num_steps, 1)) + y_data = y_data.reshape((-1, 1)) + init_hidden_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32') + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32') + fetch_list = [static_loss, static_last_hidden, static_last_cell] + out = exe.run(fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data + }, + fetch_list=fetch_list) + static_loss_value = out[0] + static_last_hidden_value = out[1] + static_last_cell_value = out[2] + + # get value before save + main_program = framework.default_main_program() + base_map = {} + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimzier var have been update + self.assertTrue(np.sum(np.abs(t)) != 0) + base_map[var.name] = t + + fluid.save(main_program, "./test_1") + + # set var to zero + for var in main_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + ten = fluid.global_scope().find_var(var.name).get_tensor() + ten.set(np.zeros_like(np.array(ten)), place) + + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimzier var have been set to zero + self.assertTrue(np.sum(np.abs(new_t)) == 0) + + #fluid.load(test_program, "./test_1", None ) + program_state = fluid.load_program_state("./test_1") + fluid.set_program_state(test_program, program_state) + + for var in test_program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + print(var.name) + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + base_t = base_map[var.name] + self.assertTrue(np.array_equal(new_t, base_t)) + + +class TestVariableInit(unittest.TestCase): + def test_variable_init(self): + + x = fluid.data(name="x", shape=[10, 10], dtype='float32') + y = fluid.layers.fc(x, 10) + z = fluid.layers.fc(y, 10) + + place = fluid.CPUPlace() if not core.is_compiled_with_cuda( + ) else fluid.CUDAPlace(0) + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) + + fluid.save(fluid.default_main_program(), "./test_path") + + def set_var(var, ndarray): + t = var.get_tensor() + p = t._place() + if p.is_cpu_place(): + place = paddle.fluid.CPUPlace() + elif p.is_cuda_pinned_place(): + place = paddle.fluid.CUDAPinnedPlace() + else: + p = paddle.fluid.core.Place() + p.set_place(t._place()) + place = paddle.fluid.CUDAPlace(p.gpu_device_id()) + + t.set(ndarray, place) + + program = fluid.default_main_program() + new_scope = fluid.core.Scope() + + place = fluid.CPUPlace() if not core.is_compiled_with_cuda( + ) else fluid.CUDAPlace(0) + exe = fluid.Executor(place) + parameter_list = list( + filter(fluid.io.is_parameter, program.list_vars())) + + fluid.core._create_loaded_parameter(parameter_list, new_scope, + exe._default_executor) + parameter_file_name = "./test_path.pdparams" + with open(parameter_file_name, 'rb') as f: + load_dict = pickle.load(f) + + for v in parameter_list: + assert v.name in load_dict, \ + "Can not find [{}] in model file [{}]".format( + v.name, parameter_file_name) + new_v = new_scope.find_var(v.name) + set_var(new_v, load_dict[v.name]) + + opt_list = list( + filter(fluid.io.is_belong_to_optimizer, program.list_vars())) + + fluid.core._create_loaded_parameter(opt_list, new_scope, + exe._default_executor) + opt_file_name = "./test_path.pdopt" + with open(opt_file_name, 'rb') as f: + load_dict = pickle.load(f) + + for v in opt_list: + assert v.name in load_dict, \ + "Can not find [{}] in model file [{}]".format( + v.name, opt_file_name) + + new_v = new_scope.find_var(v.name) + set_var(new_v, load_dict[v.name]) + + base_map = {} + for var in program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimzier var have been update + base_map[var.name] = t + + for var in program.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + new_t = np.array(new_scope.find_var(var.name).get_tensor()) + base_t = base_map[var.name] + + self.assertTrue(np.array_equal(new_t, base_t)) + + if __name__ == '__main__': unittest.main() -- GitLab