diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index 6f66d0f044afaf0c7f024f538ae546a80af4f801..88027e46d27bc8747a27e38b70946248463eaa61 100644 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -709,11 +709,12 @@ set_tests_properties(test_nearest_interp_v2_op PROPERTIES TIMEOUT 120) set_tests_properties(test_trilinear_interp_op PROPERTIES TIMEOUT 120) set_tests_properties(test_bicubic_interp_v2_op PROPERTIES TIMEOUT 120) set_tests_properties(test_gather_op PROPERTIES TIMEOUT 120) +set_tests_properties(test_static_save_load PROPERTIES TIMEOUT 120) if (WIN32) - set_tests_properties(test_static_save_load PROPERTIES TIMEOUT 900) + set_tests_properties(test_static_save_load_large PROPERTIES TIMEOUT 900) set_tests_properties(test_paddle_save_load PROPERTIES TIMEOUT 250) else() - set_tests_properties(test_static_save_load PROPERTIES TIMEOUT 600) + set_tests_properties(test_static_save_load_large PROPERTIES TIMEOUT 600) set_tests_properties(test_paddle_save_load PROPERTIES TIMEOUT 150) endif() set_tests_properties(test_imperative_selected_rows_to_lod_tensor PROPERTIES TIMEOUT 120) diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load.py b/python/paddle/fluid/tests/unittests/test_static_save_load.py index 68d0e07e0cf2d8c1ca4b626a2afa2eb165a834b1..ca66aa47266ce272f8bc2ffe9757eed15e526999 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load.py @@ -1313,78 +1313,6 @@ class TestProgramStateOldSave(unittest.TestCase): self.assertTrue(np.array_equal(new_t, base_t)) -class TestStaticSaveLoadLargeParameters(unittest.TestCase): - def test_large_parameters_static_save(self): - # enable static mode - paddle.enable_static() - LARGE_PARAM = 2**26 - with new_program_scope(): - # create network - x = paddle.static.data( - name="static_save_load_large_x", - shape=[None, 10], - dtype='float32') - z = paddle.static.nn.fc(x, LARGE_PARAM, bias_attr=False) - place = paddle.CPUPlace() - exe = paddle.static.Executor(place) - exe.run(paddle.static.default_startup_program()) - prog = paddle.static.default_main_program() - - inputs = np.random.randn(1, 10).astype("float32") - result_z = exe.run(program=prog, - feed={"static_save_load_large_x": inputs}, - fetch_list=[z.name]) - base_map = {} - for var in prog.list_vars(): - if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var(var.name) - .get_tensor()) - # make sure all the paramerter or optimizer var have been update - self.assertTrue(np.sum(np.abs(t)) != 0) - base_map[var.name] = t - - path = os.path.join("test_static_save_load_large_param", - "static_save") - paddle.fluid.save(prog, path) - # set var to zero - for var in prog.list_vars(): - if isinstance(var, framework.Parameter) or var.persistable: - ten = fluid.global_scope().find_var(var.name).get_tensor() - ten.set(np.zeros_like(np.array(ten)), place) - - new_t = np.array(fluid.global_scope().find_var(var.name) - .get_tensor()) - self.assertTrue(np.sum(np.abs(new_t)) == 0) - - paddle.fluid.load(prog, path) - - for var in prog.list_vars(): - if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var(var.name) - .get_tensor()) - base_t = base_map[var.name] - self.assertTrue(np.array_equal(new_t, base_t)) - - # set var to zero - for var in prog.list_vars(): - if isinstance(var, framework.Parameter) or var.persistable: - ten = fluid.global_scope().find_var(var.name).get_tensor() - ten.set(np.zeros_like(np.array(ten)), place) - - new_t = np.array(fluid.global_scope().find_var(var.name) - .get_tensor()) - self.assertTrue(np.sum(np.abs(new_t)) == 0) - - program_state = fluid.load_program_state(path) - fluid.set_program_state(prog, program_state) - for var in prog.list_vars(): - if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var(var.name) - .get_tensor()) - base_t = base_map[var.name] - self.assertTrue(np.array_equal(new_t, base_t)) - - class TestProgramStateOldSaveSingleModel(unittest.TestCase): def test_ptb_rnn_cpu_float32(self): seed = 90 diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py new file mode 100644 index 0000000000000000000000000000000000000000..08413d711be55b0b7072b3d2dfd0dc5efdeb8462 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py @@ -0,0 +1,94 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import paddle +import paddle.fluid as fluid +import paddle.fluid.framework as framework +from test_imperative_base import new_program_scope + +import numpy as np +import six +import pickle +import os + + +class TestStaticSaveLoadLargeParameters(unittest.TestCase): + def test_large_parameters_static_save(self): + # enable static mode + paddle.enable_static() + LARGE_PARAM = 2**26 + with new_program_scope(): + # create network + x = paddle.static.data( + name="static_save_load_large_x", + shape=[None, 10], + dtype='float32') + z = paddle.static.nn.fc(x, LARGE_PARAM, bias_attr=False) + place = paddle.CPUPlace() + exe = paddle.static.Executor(place) + exe.run(paddle.static.default_startup_program()) + prog = paddle.static.default_main_program() + + base_map = {} + for var in prog.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + # make sure all the paramerter or optimizer var have been update + self.assertTrue(np.sum(np.abs(t)) != 0) + base_map[var.name] = t + + path = os.path.join("test_static_save_load_large_param", + "static_save") + paddle.fluid.save(prog, path) + # set var to zero + for var in prog.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + ten = fluid.global_scope().find_var(var.name).get_tensor() + ten.set(np.zeros_like(np.array(ten)), place) + + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + self.assertTrue(np.sum(np.abs(new_t)) == 0) + + paddle.fluid.load(prog, path) + + for var in prog.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + base_t = base_map[var.name] + self.assertTrue(np.array_equal(new_t, base_t)) + + # set var to zero + for var in prog.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + ten = fluid.global_scope().find_var(var.name).get_tensor() + ten.set(np.zeros_like(np.array(ten)), place) + + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + self.assertTrue(np.sum(np.abs(new_t)) == 0) + + program_state = fluid.load_program_state(path) + fluid.set_program_state(prog, program_state) + for var in prog.list_vars(): + if isinstance(var, framework.Parameter) or var.persistable: + new_t = np.array(fluid.global_scope().find_var(var.name) + .get_tensor()) + base_t = base_map[var.name] + self.assertTrue(np.array_equal(new_t, base_t)) diff --git a/tools/__pycache__/static_mode_white_list.cpython-35.pyc b/tools/__pycache__/static_mode_white_list.cpython-35.pyc deleted file mode 100644 index 7dae6374903af069c516bb436c2fcea7f761dcd1..0000000000000000000000000000000000000000 Binary files a/tools/__pycache__/static_mode_white_list.cpython-35.pyc and /dev/null differ