Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 1 年 前同步成功

通知 2298
Star 20931
Fork 5422
P
Paddle
未验证 提交 6aeb60aa 编写于 作者: M minghaoBD 提交者： GitHub
浏览文件
操作

[ASP] fix some bugs of asp (#43853)

上级 b34b54db
隐藏空白更改
内联 并排
Showing with 233 addition and 74 deletion
python/paddle/fluid/contrib/sparsity/asp.py
浏览文件 @ 6aeb60aa
...@@ -722,6 +722,10 @@ class ASPHelper(object): ...@@ -722,6 +722,10 @@ class ASPHelper(object):
if param_name in supported_layers_and_prune_func_map: if param_name in supported_layers_and_prune_func_map:
return True return True
# The parameter's name is neither in *.* format nor added to supported_layers_and_prune_func_map, return False.
if len(param_name_list) == 1:
return False
param_name_no_weight_suffix = param_name_list[0] param_name_no_weight_suffix = param_name_list[0]
param_type_suffix = param_name_list[1] param_type_suffix = param_name_list[1]
layer_name = param_name_no_weight_suffix[:param_name_no_weight_suffix. layer_name = param_name_no_weight_suffix[:param_name_no_weight_suffix.
......
python/paddle/fluid/contrib/sparsity/supported_layer_list.py
浏览文件 @ 6aeb60aa
...@@ -15,14 +15,38 @@ ...@@ -15,14 +15,38 @@
import numpy as np import numpy as np
import paddle import paddle
import copy
from paddle.fluid.contrib import sparsity from paddle.fluid.contrib import sparsity
import threading import threading
import logging
from ...log_helper import get_logger
__all__ = ['add_supported_layer'] __all__ = ['add_supported_layer']
_logger = get_logger(__name__,
logging.INFO,
fmt='%(asctime)s-%(levelname)s: %(message)s')
def _default_pruning(weight_nparray, m, n, func_name, param_name): def _default_pruning(weight_nparray, m, n, func_name, param_name):
# if the to-be-pruned dimension's size is smaller than m, we don't prune it. This strong assertion is required by the inference from cuSparseLT.
shape = weight_nparray.shape
weight_pruned_nparray = copy.deepcopy(weight_nparray)
weight_sparse_mask = np.ones_like(weight_pruned_nparray)
exlude_cond_shape2 = len(shape) == 2 and shape[0] < m
exlude_cond_shape4 = len(shape) == 4 and shape[1] < m
if exlude_cond_shape2:
_logger.warning(
'{} is not pruned because the first dimension of {} is smaller than {}'
.format(param_name, shape, m))
return weight_pruned_nparray, weight_sparse_mask
if exlude_cond_shape4:
_logger.warning(
'{} is not pruned because the second dimension of {} is smaller than {}'
.format(param_name, shape, m))
return weight_pruned_nparray, weight_sparse_mask
checked_func_name = sparsity.CheckMethod.get_checking_method(func_name) checked_func_name = sparsity.CheckMethod.get_checking_method(func_name)
# The double transpose ops here make sure pruning direction consistent with cuSparseLt. # The double transpose ops here make sure pruning direction consistent with cuSparseLt.
......
python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py
浏览文件 @ 6aeb60aa
...@@ -224,12 +224,20 @@ class TestASPStaticCustomerizedPruneFunc(unittest.TestCase): ...@@ -224,12 +224,20 @@ class TestASPStaticCustomerizedPruneFunc(unittest.TestCase):
self.assertLessEqual( self.assertLessEqual(
np.sum(mat.flatten() - static_tensor.flatten()), 1e-4) np.sum(mat.flatten() - static_tensor.flatten()), 1e-4)
else: else:
self.assertTrue( if (len(param.shape) == 4
sparsity.check_sparsity( and param.shape[1] < 4) or (len(param.shape) == 2
mat.T, and param.shape[0] < 4):
func_name=sparsity.CheckMethod.CHECK_1D, self.assertFalse(
n=2, paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
m=4)) n=2,
m=4))
else:
self.assertTrue(
sparsity.check_sparsity(
mat.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
self.assertEqual(supported_layer_count, self.supported_layer_count_ref) self.assertEqual(supported_layer_count, self.supported_layer_count_ref)
def test_training_pruning(self): def test_training_pruning(self):
...@@ -264,18 +272,26 @@ class TestASPStaticCustomerizedPruneFunc(unittest.TestCase): ...@@ -264,18 +272,26 @@ class TestASPStaticCustomerizedPruneFunc(unittest.TestCase):
np.sum(mat_mask.flatten() - np.sum(mat_mask.flatten() -
static_tensor_mask.flatten()), 1e-4) static_tensor_mask.flatten()), 1e-4)
else: else:
self.assertTrue( if (len(param.shape) == 4
sparsity.check_sparsity( and param.shape[1] < 4) or (len(param.shape) == 2
mat.T, and param.shape[0] < 4):
func_name=sparsity.CheckMethod.CHECK_1D, self.assertFalse(
n=2, sparsity.check_sparsity(mat.T, n=2, m=4))
m=4)) self.assertFalse(
self.assertTrue( sparsity.check_sparsity(mat_mask.T, n=2, m=4))
sparsity.check_sparsity( else:
mat_mask.T, self.assertTrue(
func_name=sparsity.CheckMethod.CHECK_1D, sparsity.check_sparsity(
n=2, mat.T,
m=4)) func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
self.assertTrue(
sparsity.check_sparsity(
mat_mask.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
self.assertEqual(supported_layer_count, self.supported_layer_count_ref) self.assertEqual(supported_layer_count, self.supported_layer_count_ref)
......
python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py
浏览文件 @ 6aeb60aa
...@@ -131,10 +131,18 @@ class TestASPDynamicOptimize(unittest.TestCase): ...@@ -131,10 +131,18 @@ class TestASPDynamicOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer( if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name): paddle.static.default_main_program(), param.name):
mat = param.numpy() mat = param.numpy()
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
def test_asp_training_with_amp(self): def test_asp_training_with_amp(self):
self.optimizer = paddle.incubate.asp.decorate(self.optimizer) self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
...@@ -165,10 +173,19 @@ class TestASPDynamicOptimize(unittest.TestCase): ...@@ -165,10 +173,19 @@ class TestASPDynamicOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer( if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name): paddle.static.default_main_program(), param.name):
mat = param.numpy() mat = param.numpy()
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if __name__ == '__main__': if __name__ == '__main__':
......
python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py
浏览文件 @ 6aeb60aa
...@@ -143,10 +143,18 @@ class TestASPStaticOptimize(unittest.TestCase): ...@@ -143,10 +143,18 @@ class TestASPStaticOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer(self.main_program, param.name): if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var( mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor()) param.name).get_tensor())
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
def test_asp_training_with_amp(self): def test_asp_training_with_amp(self):
if core.is_compiled_with_cuda(): if core.is_compiled_with_cuda():
...@@ -172,10 +180,18 @@ class TestASPStaticOptimize(unittest.TestCase): ...@@ -172,10 +180,18 @@ class TestASPStaticOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer(self.main_program, param.name): if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var( mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor()) param.name).get_tensor())
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
def __get_param_names(self, params): def __get_param_names(self, params):
param_names = [] param_names = []
......
python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py
浏览文件 @ 6aeb60aa
...@@ -85,9 +85,17 @@ class TestASPDynamicPruningBase(unittest.TestCase): ...@@ -85,9 +85,17 @@ class TestASPDynamicPruningBase(unittest.TestCase):
if ASPHelper._is_supported_layer( if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name): paddle.static.default_main_program(), param.name):
mat = param.numpy() mat = param.numpy()
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity( and param.shape[1] < 4) or (len(param.shape) == 2
mat.T, func_name=self.mask_check_func, n=2, m=4)) and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, func_name=self.mask_check_func, n=2, m=4))
class TestASPDynamicPruning1D(TestASPDynamicPruningBase): class TestASPDynamicPruning1D(TestASPDynamicPruningBase):
......
python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py
浏览文件 @ 6aeb60aa
...@@ -43,7 +43,8 @@ class TestASPStaticPruningBase(unittest.TestCase): ...@@ -43,7 +43,8 @@ class TestASPStaticPruningBase(unittest.TestCase):
padding=2, padding=2,
act="relu") act="relu")
hidden = fluid.layers.fc(input=hidden, size=32, act='softmax') hidden = fluid.layers.fc(input=hidden, size=32, act='softmax')
prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') hidden = fluid.layers.fc(input=hidden, size=3, act='softmax')
prediction = fluid.layers.fc(input=hidden, size=3, act='softmax')
return img, label, prediction return img, label, prediction
with fluid.program_guard(self.main_program, self.startup_program): with fluid.program_guard(self.main_program, self.startup_program):
...@@ -88,9 +89,17 @@ class TestASPStaticPruningBase(unittest.TestCase): ...@@ -88,9 +89,17 @@ class TestASPStaticPruningBase(unittest.TestCase):
if ASPHelper._is_supported_layer(self.main_program, param.name): if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var( mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor()) param.name).get_tensor())
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity( and param.shape[1] < 4) or (len(param.shape) == 2
mat.T, func_name=self.mask_check_func, n=2, m=4)) and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, func_name=self.mask_check_func, n=2, m=4))
class TestASPStaticPruning1D(TestASPStaticPruningBase): class TestASPStaticPruning1D(TestASPStaticPruningBase):
......
python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py
浏览文件 @ 6aeb60aa
...@@ -103,10 +103,18 @@ class TestASPDynamicOptimize(unittest.TestCase): ...@@ -103,10 +103,18 @@ class TestASPDynamicOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer( if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name): paddle.static.default_main_program(), param.name):
mat = param.numpy() mat = param.numpy()
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
class TestASPStaticOptimize(unittest.TestCase): class TestASPStaticOptimize(unittest.TestCase):
...@@ -171,10 +179,18 @@ class TestASPStaticOptimize(unittest.TestCase): ...@@ -171,10 +179,18 @@ class TestASPStaticOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer(prog, param.name): if ASPHelper._is_supported_layer(prog, param.name):
mat = np.array(fluid.global_scope().find_var( mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor()) param.name).get_tensor())
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if __name__ == '__main__': if __name__ == '__main__':
......
python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py
浏览文件 @ 6aeb60aa
...@@ -90,10 +90,18 @@ class TestFleetWithASPDynamic(unittest.TestCase): ...@@ -90,10 +90,18 @@ class TestFleetWithASPDynamic(unittest.TestCase):
if ASPHelper._is_supported_layer( if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name): paddle.static.default_main_program(), param.name):
mat = param.numpy() mat = param.numpy()
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
class TestFleetWithASPAMPDynamic(unittest.TestCase): class TestFleetWithASPAMPDynamic(unittest.TestCase):
...@@ -146,10 +154,18 @@ class TestFleetWithASPAMPDynamic(unittest.TestCase): ...@@ -146,10 +154,18 @@ class TestFleetWithASPAMPDynamic(unittest.TestCase):
if ASPHelper._is_supported_layer( if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name): paddle.static.default_main_program(), param.name):
mat = param.numpy() mat = param.numpy()
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if __name__ == "__main__": if __name__ == "__main__":
......
python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py
浏览文件 @ 6aeb60aa
...@@ -110,10 +110,18 @@ class TestFleetWithASPSharding(unittest.TestCase): ...@@ -110,10 +110,18 @@ class TestFleetWithASPSharding(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name): if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var( mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor()) param.name).get_tensor())
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if __name__ == "__main__": if __name__ == "__main__":
......
python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py
浏览文件 @ 6aeb60aa
...@@ -85,10 +85,18 @@ class TestFleetWithASPStatic(unittest.TestCase): ...@@ -85,10 +85,18 @@ class TestFleetWithASPStatic(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name): if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var( mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor()) param.name).get_tensor())
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
class TestFleetWithASPAMPStatic(unittest.TestCase): class TestFleetWithASPAMPStatic(unittest.TestCase):
...@@ -146,10 +154,18 @@ class TestFleetWithASPAMPStatic(unittest.TestCase): ...@@ -146,10 +154,18 @@ class TestFleetWithASPAMPStatic(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name): if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var( mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor()) param.name).get_tensor())
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
def test_with_asp_and_pure_fp16(self): def test_with_asp_and_pure_fp16(self):
fleet.init(is_collective=True) fleet.init(is_collective=True)
...@@ -187,10 +203,18 @@ class TestFleetWithASPAMPStatic(unittest.TestCase): ...@@ -187,10 +203,18 @@ class TestFleetWithASPAMPStatic(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name): if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var( mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor()) param.name).get_tensor())
self.assertTrue( if (len(param.shape) == 4
paddle.fluid.contrib.sparsity.check_sparsity(mat.T, and param.shape[1] < 4) or (len(param.shape) == 2
n=2, and param.shape[0] < 4):
m=4)) self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if __name__ == "__main__": if __name__ == "__main__":
......
python/paddle/static/sparsity/__init__.py
浏览文件 @ 6aeb60aa
...@@ -17,6 +17,7 @@ from ...fluid.contrib.sparsity import calculate_density #noqa: F401 ...@@ -17,6 +17,7 @@ from ...fluid.contrib.sparsity import calculate_density #noqa: F401
from ...fluid.contrib.sparsity import decorate #noqa: F401 from ...fluid.contrib.sparsity import decorate #noqa: F401
from ...fluid.contrib.sparsity import prune_model #noqa: F401 from ...fluid.contrib.sparsity import prune_model #noqa: F401
from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401 from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401
from ...fluid.contrib.sparsity import add_supported_layer #noqa: F401
from ...fluid.contrib import sparsity #noqa: F401 from ...fluid.contrib import sparsity #noqa: F401
...@@ -27,5 +28,5 @@ def set_excluded_layers(main_program, param_names): ...@@ -27,5 +28,5 @@ def set_excluded_layers(main_program, param_names):
__all__ = [ #noqa __all__ = [ #noqa
'calculate_density', 'decorate', 'prune_model', 'set_excluded_layers', 'calculate_density', 'decorate', 'prune_model', 'set_excluded_layers',
'reset_excluded_layers' 'reset_excluded_layers', 'add_supported_layer'
] ]
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或