From 6aeb60aa994bbff0d5adfa7944eda7a9a8544ce1 Mon Sep 17 00:00:00 2001 From: minghaoBD <79566150+minghaoBD@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:47:14 +0800 Subject: [PATCH] [ASP] fix some bugs of asp (#43853) --- python/paddle/fluid/contrib/sparsity/asp.py | 4 ++ .../contrib/sparsity/supported_layer_list.py | 24 +++++++++ .../asp/test_asp_customized_pruning.py | 52 ++++++++++++------- .../asp/test_asp_optimize_dynamic.py | 33 +++++++++--- .../unittests/asp/test_asp_optimize_static.py | 32 +++++++++--- .../unittests/asp/test_asp_pruning_dynamic.py | 14 +++-- .../unittests/asp/test_asp_pruning_static.py | 17 ++++-- .../tests/unittests/asp/test_asp_save_load.py | 32 +++++++++--- .../asp/test_fleet_with_asp_dynamic.py | 32 +++++++++--- .../asp/test_fleet_with_asp_sharding.py | 16 ++++-- .../asp/test_fleet_with_asp_static.py | 48 ++++++++++++----- python/paddle/static/sparsity/__init__.py | 3 +- 12 files changed, 233 insertions(+), 74 deletions(-) diff --git a/python/paddle/fluid/contrib/sparsity/asp.py b/python/paddle/fluid/contrib/sparsity/asp.py index 0710ee9c72..40c96e0ce3 100644 --- a/python/paddle/fluid/contrib/sparsity/asp.py +++ b/python/paddle/fluid/contrib/sparsity/asp.py @@ -722,6 +722,10 @@ class ASPHelper(object): if param_name in supported_layers_and_prune_func_map: return True + # The parameter's name is neither in *.* format nor added to supported_layers_and_prune_func_map, return False. + if len(param_name_list) == 1: + return False + param_name_no_weight_suffix = param_name_list[0] param_type_suffix = param_name_list[1] layer_name = param_name_no_weight_suffix[:param_name_no_weight_suffix. diff --git a/python/paddle/fluid/contrib/sparsity/supported_layer_list.py b/python/paddle/fluid/contrib/sparsity/supported_layer_list.py index d9d8c262ad..8cd422d0d7 100644 --- a/python/paddle/fluid/contrib/sparsity/supported_layer_list.py +++ b/python/paddle/fluid/contrib/sparsity/supported_layer_list.py @@ -15,14 +15,38 @@ import numpy as np import paddle +import copy from paddle.fluid.contrib import sparsity import threading +import logging +from ...log_helper import get_logger __all__ = ['add_supported_layer'] +_logger = get_logger(__name__, + logging.INFO, + fmt='%(asctime)s-%(levelname)s: %(message)s') + def _default_pruning(weight_nparray, m, n, func_name, param_name): + # if the to-be-pruned dimension's size is smaller than m, we don't prune it. This strong assertion is required by the inference from cuSparseLT. + shape = weight_nparray.shape + weight_pruned_nparray = copy.deepcopy(weight_nparray) + weight_sparse_mask = np.ones_like(weight_pruned_nparray) + exlude_cond_shape2 = len(shape) == 2 and shape[0] < m + exlude_cond_shape4 = len(shape) == 4 and shape[1] < m + if exlude_cond_shape2: + _logger.warning( + '{} is not pruned because the first dimension of {} is smaller than {}' + .format(param_name, shape, m)) + return weight_pruned_nparray, weight_sparse_mask + if exlude_cond_shape4: + _logger.warning( + '{} is not pruned because the second dimension of {} is smaller than {}' + .format(param_name, shape, m)) + return weight_pruned_nparray, weight_sparse_mask + checked_func_name = sparsity.CheckMethod.get_checking_method(func_name) # The double transpose ops here make sure pruning direction consistent with cuSparseLt. diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py b/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py index 4ee7c2a99f..27b4361852 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py @@ -224,12 +224,20 @@ class TestASPStaticCustomerizedPruneFunc(unittest.TestCase): self.assertLessEqual( np.sum(mat.flatten() - static_tensor.flatten()), 1e-4) else: - self.assertTrue( - sparsity.check_sparsity( - mat.T, - func_name=sparsity.CheckMethod.CHECK_1D, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + sparsity.check_sparsity( + mat.T, + func_name=sparsity.CheckMethod.CHECK_1D, + n=2, + m=4)) self.assertEqual(supported_layer_count, self.supported_layer_count_ref) def test_training_pruning(self): @@ -264,18 +272,26 @@ class TestASPStaticCustomerizedPruneFunc(unittest.TestCase): np.sum(mat_mask.flatten() - static_tensor_mask.flatten()), 1e-4) else: - self.assertTrue( - sparsity.check_sparsity( - mat.T, - func_name=sparsity.CheckMethod.CHECK_1D, - n=2, - m=4)) - self.assertTrue( - sparsity.check_sparsity( - mat_mask.T, - func_name=sparsity.CheckMethod.CHECK_1D, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + sparsity.check_sparsity(mat.T, n=2, m=4)) + self.assertFalse( + sparsity.check_sparsity(mat_mask.T, n=2, m=4)) + else: + self.assertTrue( + sparsity.check_sparsity( + mat.T, + func_name=sparsity.CheckMethod.CHECK_1D, + n=2, + m=4)) + self.assertTrue( + sparsity.check_sparsity( + mat_mask.T, + func_name=sparsity.CheckMethod.CHECK_1D, + n=2, + m=4)) self.assertEqual(supported_layer_count, self.supported_layer_count_ref) diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py index b58fea9b77..44bca526f9 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py @@ -131,10 +131,18 @@ class TestASPDynamicOptimize(unittest.TestCase): if ASPHelper._is_supported_layer( paddle.static.default_main_program(), param.name): mat = param.numpy() - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) def test_asp_training_with_amp(self): self.optimizer = paddle.incubate.asp.decorate(self.optimizer) @@ -165,10 +173,19 @@ class TestASPDynamicOptimize(unittest.TestCase): if ASPHelper._is_supported_layer( paddle.static.default_main_program(), param.name): mat = param.numpy() - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py index 4fdfe21de0..c6b3d00cac 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py @@ -143,10 +143,18 @@ class TestASPStaticOptimize(unittest.TestCase): if ASPHelper._is_supported_layer(self.main_program, param.name): mat = np.array(fluid.global_scope().find_var( param.name).get_tensor()) - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) def test_asp_training_with_amp(self): if core.is_compiled_with_cuda(): @@ -172,10 +180,18 @@ class TestASPStaticOptimize(unittest.TestCase): if ASPHelper._is_supported_layer(self.main_program, param.name): mat = np.array(fluid.global_scope().find_var( param.name).get_tensor()) - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) def __get_param_names(self, params): param_names = [] diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py index fd592785a2..41eb7dfd8c 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py @@ -85,9 +85,17 @@ class TestASPDynamicPruningBase(unittest.TestCase): if ASPHelper._is_supported_layer( paddle.static.default_main_program(), param.name): mat = param.numpy() - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity( - mat.T, func_name=self.mask_check_func, n=2, m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, func_name=self.mask_check_func, n=2, m=4)) class TestASPDynamicPruning1D(TestASPDynamicPruningBase): diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py index 6f137e086e..1bb5c1477b 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py @@ -43,7 +43,8 @@ class TestASPStaticPruningBase(unittest.TestCase): padding=2, act="relu") hidden = fluid.layers.fc(input=hidden, size=32, act='softmax') - prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') + hidden = fluid.layers.fc(input=hidden, size=3, act='softmax') + prediction = fluid.layers.fc(input=hidden, size=3, act='softmax') return img, label, prediction with fluid.program_guard(self.main_program, self.startup_program): @@ -88,9 +89,17 @@ class TestASPStaticPruningBase(unittest.TestCase): if ASPHelper._is_supported_layer(self.main_program, param.name): mat = np.array(fluid.global_scope().find_var( param.name).get_tensor()) - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity( - mat.T, func_name=self.mask_check_func, n=2, m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, func_name=self.mask_check_func, n=2, m=4)) class TestASPStaticPruning1D(TestASPStaticPruningBase): diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py b/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py index 710bbcc658..dc5316d254 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py @@ -103,10 +103,18 @@ class TestASPDynamicOptimize(unittest.TestCase): if ASPHelper._is_supported_layer( paddle.static.default_main_program(), param.name): mat = param.numpy() - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) class TestASPStaticOptimize(unittest.TestCase): @@ -171,10 +179,18 @@ class TestASPStaticOptimize(unittest.TestCase): if ASPHelper._is_supported_layer(prog, param.name): mat = np.array(fluid.global_scope().find_var( param.name).get_tensor()) - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py index 7aaf1fd33a..09f6096574 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py +++ b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py @@ -90,10 +90,18 @@ class TestFleetWithASPDynamic(unittest.TestCase): if ASPHelper._is_supported_layer( paddle.static.default_main_program(), param.name): mat = param.numpy() - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) class TestFleetWithASPAMPDynamic(unittest.TestCase): @@ -146,10 +154,18 @@ class TestFleetWithASPAMPDynamic(unittest.TestCase): if ASPHelper._is_supported_layer( paddle.static.default_main_program(), param.name): mat = param.numpy() - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py index 1feb3e28c1..bd4e40ce6c 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py +++ b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py @@ -110,10 +110,18 @@ class TestFleetWithASPSharding(unittest.TestCase): if ASPHelper._is_supported_layer(train_prog, param.name): mat = np.array(fluid.global_scope().find_var( param.name).get_tensor()) - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py index 23110bb7ff..4465dee4d7 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py +++ b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py @@ -85,10 +85,18 @@ class TestFleetWithASPStatic(unittest.TestCase): if ASPHelper._is_supported_layer(train_prog, param.name): mat = np.array(fluid.global_scope().find_var( param.name).get_tensor()) - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) class TestFleetWithASPAMPStatic(unittest.TestCase): @@ -146,10 +154,18 @@ class TestFleetWithASPAMPStatic(unittest.TestCase): if ASPHelper._is_supported_layer(train_prog, param.name): mat = np.array(fluid.global_scope().find_var( param.name).get_tensor()) - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) def test_with_asp_and_pure_fp16(self): fleet.init(is_collective=True) @@ -187,10 +203,18 @@ class TestFleetWithASPAMPStatic(unittest.TestCase): if ASPHelper._is_supported_layer(train_prog, param.name): mat = np.array(fluid.global_scope().find_var( param.name).get_tensor()) - self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + if (len(param.shape) == 4 + and param.shape[1] < 4) or (len(param.shape) == 2 + and param.shape[0] < 4): + self.assertFalse( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) + else: + self.assertTrue( + paddle.fluid.contrib.sparsity.check_sparsity(mat.T, + n=2, + m=4)) if __name__ == "__main__": diff --git a/python/paddle/static/sparsity/__init__.py b/python/paddle/static/sparsity/__init__.py index 11ff30c78e..8d3166b19a 100644 --- a/python/paddle/static/sparsity/__init__.py +++ b/python/paddle/static/sparsity/__init__.py @@ -17,6 +17,7 @@ from ...fluid.contrib.sparsity import calculate_density #noqa: F401 from ...fluid.contrib.sparsity import decorate #noqa: F401 from ...fluid.contrib.sparsity import prune_model #noqa: F401 from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401 +from ...fluid.contrib.sparsity import add_supported_layer #noqa: F401 from ...fluid.contrib import sparsity #noqa: F401 @@ -27,5 +28,5 @@ def set_excluded_layers(main_program, param_names): __all__ = [ #noqa 'calculate_density', 'decorate', 'prune_model', 'set_excluded_layers', - 'reset_excluded_layers' + 'reset_excluded_layers', 'add_supported_layer' ] -- GitLab