Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 1 年 前同步成功

通知 2298
Star 20931
Fork 5422
P
Paddle
未验证 提交 6aeb60aa 编写于 作者: M minghaoBD 提交者： GitHub
浏览文件
操作

[ASP] fix some bugs of asp (#43853)

上级 b34b54db
隐藏空白更改
内联 并排
Showing with 233 addition and 74 deletion
python/paddle/fluid/contrib/sparsity/asp.py
浏览文件 @ 6aeb60aa
......@@ -722,6 +722,10 @@ class ASPHelper(object):
if param_name in supported_layers_and_prune_func_map:
return True
# The parameter's name is neither in *.* format nor added to supported_layers_and_prune_func_map, return False.
if len(param_name_list) == 1:
return False
param_name_no_weight_suffix = param_name_list[0]
param_type_suffix = param_name_list[1]
layer_name = param_name_no_weight_suffix[:param_name_no_weight_suffix.
......
python/paddle/fluid/contrib/sparsity/supported_layer_list.py
浏览文件 @ 6aeb60aa
......@@ -15,14 +15,38 @@
import numpy as np
import paddle
import copy
from paddle.fluid.contrib import sparsity
import threading
import logging
from ...log_helper import get_logger
__all__ = ['add_supported_layer']
_logger = get_logger(__name__,
logging.INFO,
fmt='%(asctime)s-%(levelname)s: %(message)s')
def _default_pruning(weight_nparray, m, n, func_name, param_name):
# if the to-be-pruned dimension's size is smaller than m, we don't prune it. This strong assertion is required by the inference from cuSparseLT.
shape = weight_nparray.shape
weight_pruned_nparray = copy.deepcopy(weight_nparray)
weight_sparse_mask = np.ones_like(weight_pruned_nparray)
exlude_cond_shape2 = len(shape) == 2 and shape[0] < m
exlude_cond_shape4 = len(shape) == 4 and shape[1] < m
if exlude_cond_shape2:
_logger.warning(
'{} is not pruned because the first dimension of {} is smaller than {}'
.format(param_name, shape, m))
return weight_pruned_nparray, weight_sparse_mask
if exlude_cond_shape4:
_logger.warning(
'{} is not pruned because the second dimension of {} is smaller than {}'
.format(param_name, shape, m))
return weight_pruned_nparray, weight_sparse_mask
checked_func_name = sparsity.CheckMethod.get_checking_method(func_name)
# The double transpose ops here make sure pruning direction consistent with cuSparseLt.
......
python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py
浏览文件 @ 6aeb60aa
......@@ -224,12 +224,20 @@ class TestASPStaticCustomerizedPruneFunc(unittest.TestCase):
self.assertLessEqual(
np.sum(mat.flatten() - static_tensor.flatten()), 1e-4)
else:
self.assertTrue(
sparsity.check_sparsity(
mat.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
sparsity.check_sparsity(
mat.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
self.assertEqual(supported_layer_count, self.supported_layer_count_ref)
def test_training_pruning(self):
......@@ -264,18 +272,26 @@ class TestASPStaticCustomerizedPruneFunc(unittest.TestCase):
np.sum(mat_mask.flatten() -
static_tensor_mask.flatten()), 1e-4)
else:
self.assertTrue(
sparsity.check_sparsity(
mat.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
self.assertTrue(
sparsity.check_sparsity(
mat_mask.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
sparsity.check_sparsity(mat.T, n=2, m=4))
self.assertFalse(
sparsity.check_sparsity(mat_mask.T, n=2, m=4))
else:
self.assertTrue(
sparsity.check_sparsity(
mat.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
self.assertTrue(
sparsity.check_sparsity(
mat_mask.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
self.assertEqual(supported_layer_count, self.supported_layer_count_ref)
......
python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py
浏览文件 @ 6aeb60aa
......@@ -131,10 +131,18 @@ class TestASPDynamicOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
def test_asp_training_with_amp(self):
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
......@@ -165,10 +173,19 @@ class TestASPDynamicOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if __name__ == '__main__':
......
python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py
浏览文件 @ 6aeb60aa
......@@ -143,10 +143,18 @@ class TestASPStaticOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
def test_asp_training_with_amp(self):
if core.is_compiled_with_cuda():
......@@ -172,10 +180,18 @@ class TestASPStaticOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
def __get_param_names(self, params):
param_names = []
......
python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py
浏览文件 @ 6aeb60aa
......@@ -85,9 +85,17 @@ class TestASPDynamicPruningBase(unittest.TestCase):
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, func_name=self.mask_check_func, n=2, m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, func_name=self.mask_check_func, n=2, m=4))
class TestASPDynamicPruning1D(TestASPDynamicPruningBase):
......
python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py
浏览文件 @ 6aeb60aa
......@@ -43,7 +43,8 @@ class TestASPStaticPruningBase(unittest.TestCase):
padding=2,
act="relu")
hidden = fluid.layers.fc(input=hidden, size=32, act='softmax')
prediction = fluid.layers.fc(input=hidden, size=10, act='softmax')
hidden = fluid.layers.fc(input=hidden, size=3, act='softmax')
prediction = fluid.layers.fc(input=hidden, size=3, act='softmax')
return img, label, prediction
with fluid.program_guard(self.main_program, self.startup_program):
......@@ -88,9 +89,17 @@ class TestASPStaticPruningBase(unittest.TestCase):
if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, func_name=self.mask_check_func, n=2, m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, func_name=self.mask_check_func, n=2, m=4))
class TestASPStaticPruning1D(TestASPStaticPruningBase):
......
python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py
浏览文件 @ 6aeb60aa
......@@ -103,10 +103,18 @@ class TestASPDynamicOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
class TestASPStaticOptimize(unittest.TestCase):
......@@ -171,10 +179,18 @@ class TestASPStaticOptimize(unittest.TestCase):
if ASPHelper._is_supported_layer(prog, param.name):
mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if __name__ == '__main__':
......
python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py
浏览文件 @ 6aeb60aa
......@@ -90,10 +90,18 @@ class TestFleetWithASPDynamic(unittest.TestCase):
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
class TestFleetWithASPAMPDynamic(unittest.TestCase):
......@@ -146,10 +154,18 @@ class TestFleetWithASPAMPDynamic(unittest.TestCase):
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if __name__ == "__main__":
......
python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py
浏览文件 @ 6aeb60aa
......@@ -110,10 +110,18 @@ class TestFleetWithASPSharding(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if __name__ == "__main__":
......
python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py
浏览文件 @ 6aeb60aa
......@@ -85,10 +85,18 @@ class TestFleetWithASPStatic(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
class TestFleetWithASPAMPStatic(unittest.TestCase):
......@@ -146,10 +154,18 @@ class TestFleetWithASPAMPStatic(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
def test_with_asp_and_pure_fp16(self):
fleet.init(is_collective=True)
......@@ -187,10 +203,18 @@ class TestFleetWithASPAMPStatic(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(
param.name).get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if (len(param.shape) == 4
and param.shape[1] < 4) or (len(param.shape) == 2
and param.shape[0] < 4):
self.assertFalse(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
else:
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(mat.T,
n=2,
m=4))
if __name__ == "__main__":
......
python/paddle/static/sparsity/__init__.py
浏览文件 @ 6aeb60aa
......@@ -17,6 +17,7 @@ from ...fluid.contrib.sparsity import calculate_density #noqa: F401
from ...fluid.contrib.sparsity import decorate #noqa: F401
from ...fluid.contrib.sparsity import prune_model #noqa: F401
from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401
from ...fluid.contrib.sparsity import add_supported_layer #noqa: F401
from ...fluid.contrib import sparsity #noqa: F401
......@@ -27,5 +28,5 @@ def set_excluded_layers(main_program, param_names):
__all__ = [ #noqa
'calculate_density', 'decorate', 'prune_model', 'set_excluded_layers',
'reset_excluded_layers'
'reset_excluded_layers', 'add_supported_layer'
]
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或