未验证 提交 113816d8 编写于 作者: M Ming-Xu Huang 提交者: GitHub

Move the ASP training API to paddle.static.sparsity. (#36525)

上级 f6b4ed22
...@@ -25,8 +25,10 @@ from .utils import create_mask ...@@ -25,8 +25,10 @@ from .utils import create_mask
from .utils import check_sparsity from .utils import check_sparsity
from .utils import MaskAlgo from .utils import MaskAlgo
from .utils import CheckMethod from .utils import CheckMethod
from .asp import decorate, prune_model from .asp import decorate
from .asp import set_excluded_layers, reset_excluded_layers from .asp import prune_model
from .asp import set_excluded_layers
from .asp import reset_excluded_layers
__all__ = [ __all__ = [
'calculate_density', 'check_mask_1d', 'get_mask_1d', 'check_mask_2d', 'calculate_density', 'check_mask_1d', 'get_mask_1d', 'check_mask_2d',
......
...@@ -19,10 +19,9 @@ Functions for Auto SParsity (ASP) training and inference. ...@@ -19,10 +19,9 @@ Functions for Auto SParsity (ASP) training and inference.
import copy import copy
import numpy as np import numpy as np
import paddle import paddle
from paddle.fluid import framework, global_scope, program_guard, layers from paddle.fluid import global_scope, program_guard, layers
from paddle.fluid.initializer import ConstantInitializer from paddle.fluid.initializer import ConstantInitializer
from paddle.fluid.contrib import sparsity from paddle.fluid.contrib import sparsity
from paddle.fluid import core
__all__ = [ __all__ = [
'decorate', 'prune_model', 'set_excluded_layers', 'reset_excluded_layers' 'decorate', 'prune_model', 'set_excluded_layers', 'reset_excluded_layers'
...@@ -36,6 +35,35 @@ def set_excluded_layers(main_program, param_names): ...@@ -36,6 +35,35 @@ def set_excluded_layers(main_program, param_names):
Args: Args:
main_program (Program, optional): Program with model definition and its parameters. main_program (Program, optional): Program with model definition and its parameters.
param_names (list): A list contains names of parameters. param_names (list): A list contains names of parameters.
Examples:
.. code-block:: python
import paddle
from paddle.static import sparsity
paddle.enable_static()
main_program = paddle.static.Program()
startup_program = paddle.static.Program()
with paddle.static.program_guard(main_program, startup_program):
input_data = paddle.static.data(name='data', shape=[None, 128])
label = paddle.static.data(name='label', shape=[None, 10])
hidden = paddle.static.nn.fc(x=input_data, num_flatten_dims=-1, size=32, activation=None, name="need_sparse_fc")
hidden = paddle.static.nn.fc(x=hidden, num_flatten_dims=-1, size=32, activation=None, name="need_dense_fc")
prob = paddle.static.nn.fc(x=hidden, num_flatten_dims=-1, size=10, activation=None)
loss = paddle.mean(paddle.nn.functional.square_error_cost(prob, label))
# Setup exluded layers out from ASP workflow.
# Please note, excluded_layers must be set before calling `optimizer.minimize()`.
sparsity.set_excluded_layers(main_program, ["need_dense_fc"])
optimizer = paddle.optimizer.SGD(learning_rate=0.1)
optimizer = paddle.static.amp.decorate(optimizer )
# Calling sparsity.decorate() to wrap minimize() in optimizer, which
# will insert necessary masking operations for ASP workflow.
optimizer = sparsity.decorate(optimizer)
optimizer.minimize(loss, startup_program)
""" """
ASPHelper.set_excluded_layers( ASPHelper.set_excluded_layers(
main_program=main_program, param_names=param_names) main_program=main_program, param_names=param_names)
...@@ -48,6 +76,33 @@ def reset_excluded_layers(main_program=None): ...@@ -48,6 +76,33 @@ def reset_excluded_layers(main_program=None):
Args: Args:
main_program (Program, optional): Program with model definition and its parameters. main_program (Program, optional): Program with model definition and its parameters.
Examples:
.. code-block:: python
import paddle
from paddle.static import sparsity
paddle.enable_static()
main_program = paddle.static.Program()
startup_program = paddle.static.Program()
with paddle.static.program_guard(main_program, startup_program):
input_data = paddle.static.data(name='data', shape=[None, 128])
label = paddle.static.data(name='label', shape=[None, 10])
hidden = paddle.static.nn.fc(x=input_data, num_flatten_dims=-1, size=32, activation=None, name="my_first_fc")
hidden = paddle.static.nn.fc(x=hidden, num_flatten_dims=-1, size=32, activation=None, name="my_second_fc")
prob = paddle.static.nn.fc(x=hidden, num_flatten_dims=-1, size=10, activation=None)
loss = paddle.mean(paddle.nn.functional.square_error_cost(prob, label))
# Setup exluded layers out from ASP workflow.
# Please note, excluded_layers must be set before calling `optimizer.minimize()`.
sparsity.set_excluded_layers(main_program, ["my_second_fc"])
# Now the weights of "my_second_fc" would not be included in Automatic SParsity's workflow.
# Reset excluded_layers, all FC layers would be included into Automatic SParsity's workflow.
# Please note, reset_excluded_layers also must be called before calling `optimizer.minimize()`.
sparsity.reset_excluded_layers(main_program)
""" """
ASPHelper.reset_excluded_layers(main_program=main_program) ASPHelper.reset_excluded_layers(main_program=main_program)
...@@ -65,22 +120,21 @@ def decorate(optimizer): ...@@ -65,22 +120,21 @@ def decorate(optimizer):
.. code-block:: python .. code-block:: python
import paddle import paddle
import paddle.fluid as fluid from paddle.static import sparsity
from paddle.fluid.contrib import sparsity
main_program = fluid.Program() main_program = paddle.static.Program()
startup_program = fluid.Program() startup_program = paddle.static.Program()
paddle.enable_static() paddle.enable_static()
with fluid.program_guard(main_program, startup_program): with paddle.static.program_guard(main_program, startup_program):
input_data = fluid.layers.data(name='data', shape=[None, 128]) input_data = paddle.static.data(name='data', shape=[None, 128])
label = fluid.layers.data(name='label', shape=[None, 10]) label = paddle.static.data(name='label', shape=[None, 10])
hidden = fluid.layers.fc(input=input_data, num_flatten_dims=-1, size=32, act=None) hidden = paddle.static.nn.fc(x=input_data, num_flatten_dims=-1, size=32, activation=None)
prob = fluid.layers.fc(input=hidden, num_flatten_dims=-1, size=10, act=None) prob = paddle.static.nn.fc(x=hidden, num_flatten_dims=-1, size=10, activation=None)
loss = fluid.layers.mean(fluid.layers.square_error_cost(prob, label)) loss = paddle.mean(paddle.nn.functional.square_error_cost(prob, label))
optimizer = fluid.optimizer.SGD(learning_rate=0.1) optimizer = paddle.optimizer.SGD(learning_rate=0.1)
optimizer = sparsity.decorate(optimizer) optimizer = sparsity.decorate(optimizer)
# if do sparse training with Fleet, please replace above decorate with: # if do sparse training with Fleet, please replace above decorate with:
# strategy = paddle.distributed.fleet.DistributedStrategy() # strategy = paddle.distributed.fleet.DistributedStrategy()
...@@ -92,15 +146,14 @@ def decorate(optimizer): ...@@ -92,15 +146,14 @@ def decorate(optimizer):
return ASPHelper.decorate(optimizer) return ASPHelper.decorate(optimizer)
def prune_model(place, def prune_model(main_program=None,
main_program=None,
n=2, n=2,
m=4, m=4,
func_name=sparsity.MaskAlgo.MASK_1D, mask_algo='mask_1d',
with_mask=True): with_mask=True):
r""" r"""
Pruning parameters of supported layers in :attr:`main_program` via Pruning parameters of supported layers in :attr:`main_program` via
specified mask generation function given by :attr:`func_name`. This specified mask generation function given by :attr:`mask_algo`. This
function supports both training and inference controlled by :attr:`with_mask`. function supports both training and inference controlled by :attr:`with_mask`.
If :attr:`with_mask` is True, it would also prune parameter related ASP mask Variables, If :attr:`with_mask` is True, it would also prune parameter related ASP mask Variables,
else only prunes parameters. else only prunes parameters.
...@@ -114,11 +167,11 @@ def prune_model(place, ...@@ -114,11 +167,11 @@ def prune_model(place,
inference only. To obtain OptimizerWithSparsityGuarantee, please see `sparsity.decoreate()`. inference only. To obtain OptimizerWithSparsityGuarantee, please see `sparsity.decoreate()`.
Args: Args:
place (fluid.CPUPlace()|fluid.CUDAPlace(N)): Device place for pruned parameter and mask Variables, and N means the GPU's id. It should be the same as created instance of Executor.
main_program (Program, optional): Program with model definition and its parameters. Default is `paddle.static.default_main_program() main_program (Program, optional): Program with model definition and its parameters. Default is `paddle.static.default_main_program()
n (int): n of `n:m` sparse pattern. n (int): n of `n:m` sparse pattern.
m (int): m of `n:m` sparse pattern. m (int): m of `n:m` sparse pattern.
func_name (MaskAlgo, optional): The function name to generate spase mask. Default is `MaskAlgo.MASK_1D`. All options please refer to `MaskAlgo`. mask_algo (string, optional): The function name to generate spase mask. Default is `mask_1d`.
The vaild inputs should be one of 'mask_1d', 'mask_2d_greedy' and 'mask_2d_best'.
with_mask (bool, optional): To prune mask Variables related to parameters or not. Ture is purning also, False is not. Defalut is True. with_mask (bool, optional): To prune mask Variables related to parameters or not. Ture is purning also, False is not. Defalut is True.
Returns: Returns:
dictionary: A dictionary with key: `parameter name` (string) and value: its corresponding mask Variable. dictionary: A dictionary with key: `parameter name` (string) and value: its corresponding mask Variable.
...@@ -126,50 +179,58 @@ def prune_model(place, ...@@ -126,50 +179,58 @@ def prune_model(place,
.. code-block:: python .. code-block:: python
import paddle import paddle
import paddle.fluid as fluid from paddle.static import sparsity
import paddle.fluid.core as core
from paddle.fluid.contrib import sparsity
paddle.enable_static() paddle.enable_static()
main_program = fluid.Program() main_program = paddle.static.Program()
startup_program = fluid.Program() startup_program = paddle.static.Program()
place = paddle.CPUPlace() with paddle.static.program_guard(main_program, startup_program):
if core.is_compiled_with_cuda(): input_data = paddle.static.data(name='data', shape=[None, 128])
place = paddle.CUDAPlace(0) label = paddle.static.data(name='label', shape=[None, 10])
hidden = paddle.static.nn.fc(x=input_data, num_flatten_dims=-1, size=32, activation=None, name="need_sparse_fc")
with fluid.program_guard(main_program, startup_program): hidden = paddle.static.nn.fc(x=hidden, num_flatten_dims=-1, size=32, activation=None, name="need_dense_fc")
input_data = fluid.layers.data(name='data', shape=[None, 128]) prob = paddle.static.nn.fc(x=hidden, num_flatten_dims=-1, size=10, activation=None)
label = fluid.layers.data(name='label', shape=[None, 10]) loss = paddle.mean(paddle.nn.functional.square_error_cost(prob, label))
hidden = fluid.layers.fc(input=input_data, num_flatten_dims=-1, size=32, act=None, name="need_sparse")
hidden = fluid.layers.fc(input=hidden, num_flatten_dims=-1, size=32, act=None, name="need_dense")
prob = fluid.layers.fc(input=hidden, num_flatten_dims=-1, size=10, act=None)
loss = fluid.layers.mean(fluid.layers.square_error_cost(prob, label))
# Setup exluded layers out from ASP workflow. # Setup exluded layers out from ASP workflow.
# Please note, excluded_layers must be set before calling `optimizer.minimize()`. # Please note, excluded_layers must be set before calling `optimizer.minimize()`.
sparsity.set_excluded_layers(main_program, ["need_dense"]) sparsity.set_excluded_layers(main_program, ["need_dense_fc"])
optimizer = fluid.optimizer.SGD(learning_rate=0.1) optimizer = paddle.optimizer.SGD(learning_rate=0.1)
optimizer = fluid.contrib.mixed_precision.decorator.decorate(optimizer ) optimizer = paddle.static.amp.decorate(optimizer )
# Calling sparsity.decorate() to wrap minimize() in optimizer, which # Calling sparsity.decorate() to wrap minimize() in optimizer, which
# will insert necessary masking operations for ASP workflow. # will insert necessary masking operations for ASP workflow.
optimizer = sparsity.decorate(optimizer) optimizer = sparsity.decorate(optimizer)
optimizer.minimize(loss, startup_program) optimizer.minimize(loss, startup_program)
exe = fluid.Executor(place) device = paddle.device.get_device()
place = paddle.set_device(device)
exe = paddle.static.Executor(place)
exe.run(startup_program) exe.run(startup_program)
# Must call `exe.run(startup_program)` first before calling `sparsity.prune_model` # Must call `exe.run(startup_program)` first before calling `sparsity.prune_model`
sparsity.prune_model(place, main_program, func_name=sparsity.MaskAlgo.MASK_2D_BEST) sparsity.prune_model(main_program, mask_algo='mask_2d_best')
""" """
device = paddle.device.get_device()
place = paddle.set_device(device)
MaskAlgo_mapping = {
'mask_1d': sparsity.MaskAlgo.MASK_1D,
'mask_2d_greedy': sparsity.MaskAlgo.MASK_2D_GREEDY,
'mask_2d_best': sparsity.MaskAlgo.MASK_2D_BEST
}
assert (mask_algo in MaskAlgo_mapping), \
'The "mask_algo" should be one of ["mask_1d", "mask_2d_greedy", "mask_2d_best"]'
return ASPHelper.prune_model( return ASPHelper.prune_model(
place=place, place=place,
main_program=main_program, main_program=main_program,
n=n, n=n,
m=m, m=m,
func_name=func_name, mask_algo=MaskAlgo_mapping[mask_algo],
with_mask=with_mask) with_mask=with_mask)
...@@ -256,12 +317,12 @@ class ASPHelper(object): ...@@ -256,12 +317,12 @@ class ASPHelper(object):
main_program=None, main_program=None,
n=2, n=2,
m=4, m=4,
func_name=sparsity.MaskAlgo.MASK_1D, mask_algo=sparsity.MaskAlgo.MASK_1D,
with_mask=True): with_mask=True):
r""" r"""
This is the implementation of `sparsity.prune_model`, for details please see explanation in `sparsity.prune_model`. This is the implementation of `sparsity.prune_model`, for details please see explanation in `sparsity.prune_model`.
""" """
checked_func_name = sparsity.CheckMethod.get_checking_method(func_name) checked_func_name = sparsity.CheckMethod.get_checking_method(mask_algo)
if main_program is None: if main_program is None:
main_program = paddle.static.default_main_program() main_program = paddle.static.default_main_program()
...@@ -284,7 +345,7 @@ class ASPHelper(object): ...@@ -284,7 +345,7 @@ class ASPHelper(object):
# matrices beforce invoking create_mask. Then we transpose the result maks to make # matrices beforce invoking create_mask. Then we transpose the result maks to make
# sure its shape to be the same as the input weight. # sure its shape to be the same as the input weight.
weight_sparse_mask = sparsity.create_mask( weight_sparse_mask = sparsity.create_mask(
weight_nparray.T, func_name=func_name, n=n, m=m).T weight_nparray.T, func_name=mask_algo, n=n, m=m).T
weight_pruned_nparray = np.multiply(weight_nparray, weight_pruned_nparray = np.multiply(weight_nparray,
weight_sparse_mask) weight_sparse_mask)
weight_tensor.set(weight_pruned_nparray, place) weight_tensor.set(weight_pruned_nparray, place)
...@@ -347,15 +408,14 @@ class ASPHelper(object): ...@@ -347,15 +408,14 @@ class ASPHelper(object):
Examples: Examples:
.. code-block:: python .. code-block:: python
import paddle.fluid as fluid from paddle.static.sparsity.asp import ASPHelper
from paddle.fluid.contrib.sparsity.asp import ASPHelper
main_program = fluid.Program() main_program = paddle.static.Program()
startup_program = fluid.Program() startup_program = paddle.static.Program()
with fluid.program_guard(main_program, startup_program): with paddle.static.program_guard(main_program, startup_program):
input_data = fluid.layers.data(name='data', shape=[None, 128]) input_data = paddle.static.data(name='data', shape=[None, 128])
fc = fluid.layers.fc(input=input_data, num_flatten_dims=-1, size=32, act=None) fc = paddle.static.nn.fc(x=input_data, num_flatten_dims=-1, size=32, activation=None)
for param in main_program.global_block().all_parameters(): for param in main_program.global_block().all_parameters():
ASPHelper._is_supported_layer(main_program, param.name) ASPHelper._is_supported_layer(main_program, param.name)
......
...@@ -64,7 +64,8 @@ class CheckMethod(Enum): ...@@ -64,7 +64,8 @@ class CheckMethod(Enum):
.. code-block:: python .. code-block:: python
import numpy as np import numpy as np
from paddle.fluid.contrib.sparsity import MaskAlgo, CheckMethod from paddle.static.sparsity import MaskAlgo
from paddle.fluid.contrib.sparsity import CheckMethod
CheckMethod.get_checking_method(MaskAlgo.MASK_1D) CheckMethod.get_checking_method(MaskAlgo.MASK_1D)
# CheckMethod.CHECK_1D # CheckMethod.CHECK_1D
...@@ -95,7 +96,7 @@ def calculate_density(x): ...@@ -95,7 +96,7 @@ def calculate_density(x):
.. code-block:: python .. code-block:: python
import numpy as np import numpy as np
import paddle.fluid.contrib.sparsity as sparsity import paddle.static.sparsity as sparsity
x = np.array([[0, 1, 3, 0], x = np.array([[0, 1, 3, 0],
[1, 1, 0, 1]]) [1, 1, 0, 1]])
...@@ -446,7 +447,7 @@ def get_mask_2d_best(mat, n, m): ...@@ -446,7 +447,7 @@ def get_mask_2d_best(mat, n, m):
[5, 6, 3, 9], [5, 6, 3, 9],
[2, 4, 6, 9]]) [2, 4, 6, 9]])
mask_greedy = sparsity.get_mask_2d_greedy(mat, 2, 4) mask_greedy = sparsity.get_mask_2d_greedy(mat, 2, 4)
mask_greedy = sparsity.get_mask_2d_best(mat, 2, 4) mask_best = sparsity.get_mask_2d_best(mat, 2, 4)
print("L1 norm of `greedy` sparse matrix", np.multiply(mat, mask_greedy).sum()) # 56 print("L1 norm of `greedy` sparse matrix", np.multiply(mat, mask_greedy).sum()) # 56
print("L1 norm of `best` sparse matrix", np.multiply(mat, mask_best).sum()) # 61 print("L1 norm of `best` sparse matrix", np.multiply(mat, mask_best).sum()) # 61
""" """
......
...@@ -20,7 +20,7 @@ import threading, time ...@@ -20,7 +20,7 @@ import threading, time
import paddle import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.core as core import paddle.fluid.core as core
from paddle.fluid.contrib import sparsity from paddle.static import sparsity
from paddle.fluid.contrib.sparsity.asp import ASPHelper from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np import numpy as np
...@@ -76,14 +76,11 @@ class TestASPHelperPruningBase(unittest.TestCase): ...@@ -76,14 +76,11 @@ class TestASPHelperPruningBase(unittest.TestCase):
check_func_name, with_mask): check_func_name, with_mask):
exe.run(self.startup_program) exe.run(self.startup_program)
sparsity.prune_model( sparsity.prune_model(
place, self.main_program, mask_algo=mask_func_name, with_mask=with_mask)
self.main_program,
func_name=mask_func_name,
with_mask=with_mask)
for param in self.main_program.global_block().all_parameters(): for param in self.main_program.global_block().all_parameters():
if ASPHelper._is_supported_layer(self.main_program, param.name): if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var(param.name) mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor()) .get_tensor())
self.assertTrue( self.assertTrue(
sparsity.check_sparsity( paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, func_name=check_func_name, n=2, m=4)) mat.T, func_name=check_func_name, n=2, m=4))
...@@ -20,7 +20,7 @@ import threading, time ...@@ -20,7 +20,7 @@ import threading, time
import paddle import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.core as core import paddle.fluid.core as core
from paddle.fluid.contrib import sparsity from paddle.static import sparsity
from paddle.fluid.contrib.sparsity.asp import ASPHelper from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np import numpy as np
...@@ -129,7 +129,7 @@ class TestASPHelper(unittest.TestCase): ...@@ -129,7 +129,7 @@ class TestASPHelper(unittest.TestCase):
feeder = fluid.DataFeeder(feed_list=[self.img, self.label], place=place) feeder = fluid.DataFeeder(feed_list=[self.img, self.label], place=place)
exe.run(self.startup_program) exe.run(self.startup_program)
sparsity.prune_model(place, self.main_program) sparsity.prune_model(self.main_program)
data = (np.random.randn(64, 3, 32, 32), np.random.randint( data = (np.random.randn(64, 3, 32, 32), np.random.randint(
10, size=(64, 1))) 10, size=(64, 1)))
...@@ -139,7 +139,9 @@ class TestASPHelper(unittest.TestCase): ...@@ -139,7 +139,9 @@ class TestASPHelper(unittest.TestCase):
if ASPHelper._is_supported_layer(self.main_program, param.name): if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var(param.name) mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor()) .get_tensor())
self.assertTrue(sparsity.check_sparsity(mat.T, n=2, m=4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
def test_asp_training_with_amp(self): def test_asp_training_with_amp(self):
if core.is_compiled_with_cuda(): if core.is_compiled_with_cuda():
...@@ -155,7 +157,7 @@ class TestASPHelper(unittest.TestCase): ...@@ -155,7 +157,7 @@ class TestASPHelper(unittest.TestCase):
feed_list=[self.img, self.label], place=place) feed_list=[self.img, self.label], place=place)
exe.run(self.startup_program) exe.run(self.startup_program)
sparsity.prune_model(place, self.main_program) sparsity.prune_model(self.main_program)
data = (np.random.randn(64, 3, 32, 32), np.random.randint( data = (np.random.randn(64, 3, 32, 32), np.random.randint(
10, size=(64, 1))) 10, size=(64, 1)))
...@@ -165,7 +167,9 @@ class TestASPHelper(unittest.TestCase): ...@@ -165,7 +167,9 @@ class TestASPHelper(unittest.TestCase):
if ASPHelper._is_supported_layer(self.main_program, param.name): if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var(param.name) mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor()) .get_tensor())
self.assertTrue(sparsity.check_sparsity(mat.T, n=2, m=4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
def __get_param_names(self, params): def __get_param_names(self, params):
param_names = [] param_names = []
......
...@@ -17,7 +17,7 @@ from __future__ import print_function ...@@ -17,7 +17,7 @@ from __future__ import print_function
import unittest import unittest
import paddle import paddle
from paddle.fluid.contrib import sparsity from paddle.static import sparsity
from paddle.fluid.tests.unittests.asp.asp_pruning_base import TestASPHelperPruningBase from paddle.fluid.tests.unittests.asp.asp_pruning_base import TestASPHelperPruningBase
paddle.enable_static() paddle.enable_static()
...@@ -25,12 +25,12 @@ paddle.enable_static() ...@@ -25,12 +25,12 @@ paddle.enable_static()
class TestASPHelperPruning1D(TestASPHelperPruningBase): class TestASPHelperPruning1D(TestASPHelperPruningBase):
def test_1D_inference_pruning(self): def test_1D_inference_pruning(self):
self.run_inference_pruning_test(sparsity.MaskAlgo.MASK_1D, self.run_inference_pruning_test(
sparsity.CheckMethod.CHECK_1D) 'mask_1d', paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D)
def test_1D_training_pruning(self): def test_1D_training_pruning(self):
self.run_training_pruning_test(sparsity.MaskAlgo.MASK_1D, self.run_training_pruning_test(
sparsity.CheckMethod.CHECK_1D) 'mask_1d', paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D)
if __name__ == '__main__': if __name__ == '__main__':
......
...@@ -17,7 +17,7 @@ from __future__ import print_function ...@@ -17,7 +17,7 @@ from __future__ import print_function
import paddle import paddle
import unittest import unittest
from paddle.fluid.contrib import sparsity from paddle.static import sparsity
from paddle.fluid.tests.unittests.asp.asp_pruning_base import TestASPHelperPruningBase from paddle.fluid.tests.unittests.asp.asp_pruning_base import TestASPHelperPruningBase
paddle.enable_static() paddle.enable_static()
...@@ -25,12 +25,12 @@ paddle.enable_static() ...@@ -25,12 +25,12 @@ paddle.enable_static()
class TestASPHelperPruning2DBest(TestASPHelperPruningBase): class TestASPHelperPruning2DBest(TestASPHelperPruningBase):
def test_2D_best_inference_pruning(self): def test_2D_best_inference_pruning(self):
self.run_inference_pruning_test(sparsity.MaskAlgo.MASK_2D_BEST, self.run_inference_pruning_test(
sparsity.CheckMethod.CHECK_2D) 'mask_2d_best', paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D)
def test_2D_best_training_pruning(self): def test_2D_best_training_pruning(self):
self.run_training_pruning_test(sparsity.MaskAlgo.MASK_2D_BEST, self.run_training_pruning_test(
sparsity.CheckMethod.CHECK_2D) 'mask_2d_best', paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D)
if __name__ == '__main__': if __name__ == '__main__':
......
...@@ -17,7 +17,7 @@ from __future__ import print_function ...@@ -17,7 +17,7 @@ from __future__ import print_function
import unittest import unittest
import paddle import paddle
from paddle.fluid.contrib import sparsity from paddle.static import sparsity
from paddle.fluid.tests.unittests.asp.asp_pruning_base import TestASPHelperPruningBase from paddle.fluid.tests.unittests.asp.asp_pruning_base import TestASPHelperPruningBase
paddle.enable_static() paddle.enable_static()
...@@ -25,12 +25,14 @@ paddle.enable_static() ...@@ -25,12 +25,14 @@ paddle.enable_static()
class TestASPHelperPruning2DGreedy(TestASPHelperPruningBase): class TestASPHelperPruning2DGreedy(TestASPHelperPruningBase):
def test_2D_greedy_inference_pruning(self): def test_2D_greedy_inference_pruning(self):
self.run_inference_pruning_test(sparsity.MaskAlgo.MASK_2D_GREEDY, self.run_inference_pruning_test(
sparsity.CheckMethod.CHECK_2D) 'mask_2d_greedy',
paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D)
def test_2D_greedy_training_pruning(self): def test_2D_greedy_training_pruning(self):
self.run_training_pruning_test(sparsity.MaskAlgo.MASK_2D_GREEDY, self.run_training_pruning_test(
sparsity.CheckMethod.CHECK_2D) 'mask_2d_greedy',
paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D)
if __name__ == '__main__': if __name__ == '__main__':
......
...@@ -18,22 +18,24 @@ from __future__ import print_function ...@@ -18,22 +18,24 @@ from __future__ import print_function
import unittest import unittest
import threading, time import threading, time
import paddle import paddle
from paddle.fluid.contrib import sparsity from paddle.static import sparsity
import numpy as np import numpy as np
class TestASPUtils(unittest.TestCase): class TestASPUtils(unittest.TestCase):
def test_get_check_method(self): def test_get_check_method(self):
self.assertEqual( self.assertEqual(
sparsity.CheckMethod.get_checking_method(sparsity.MaskAlgo.MASK_1D), paddle.fluid.contrib.sparsity.CheckMethod.get_checking_method(
sparsity.CheckMethod.CHECK_1D) paddle.fluid.contrib.sparsity.MaskAlgo.MASK_1D),
paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D)
self.assertEqual( self.assertEqual(
sparsity.CheckMethod.get_checking_method( paddle.fluid.contrib.sparsity.CheckMethod.get_checking_method(
sparsity.MaskAlgo.MASK_2D_GREEDY), paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_GREEDY),
sparsity.CheckMethod.CHECK_2D) paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D)
self.assertEqual( self.assertEqual(
sparsity.CheckMethod.get_checking_method( paddle.fluid.contrib.sparsity.CheckMethod.get_checking_method(
sparsity.MaskAlgo.MASK_2D_BEST), sparsity.CheckMethod.CHECK_2D) paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_BEST),
paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D)
def test_density(self): def test_density(self):
x = np.array([[1.0, 1.0, 1.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], x = np.array([[1.0, 1.0, 1.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0],
...@@ -47,53 +49,59 @@ class TestASPUtils(unittest.TestCase): ...@@ -47,53 +49,59 @@ class TestASPUtils(unittest.TestCase):
x = np.array([[1.0, 0.0, 0.0, 1.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], x = np.array([[1.0, 0.0, 0.0, 1.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0],
[1.0, 1.0, 0.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0],
[0.0, 1.0, 0.0, 0.0, 1.0]]) [0.0, 1.0, 0.0, 0.0, 1.0]])
self.assertTrue(sparsity.check_mask_1d(x, 2, 4)) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 4))
self.assertFalse(sparsity.check_mask_1d(x, 3, 4)) self.assertFalse(paddle.fluid.contrib.sparsity.check_mask_1d(x, 3, 4))
self.assertTrue(sparsity.check_mask_1d(x, 2, 5)) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 5))
self.assertFalse(sparsity.check_mask_1d(x, 3, 5)) self.assertFalse(paddle.fluid.contrib.sparsity.check_mask_1d(x, 3, 5))
self.assertTrue(sparsity.check_mask_1d(x, 3, 6)) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d(x, 3, 6))
self.assertFalse(sparsity.check_mask_1d(x, 4, 6)) self.assertFalse(paddle.fluid.contrib.sparsity.check_mask_1d(x, 4, 6))
def test_get_mask_1d(self): def test_get_mask_1d(self):
for _ in range(10): for _ in range(10):
x = np.random.randint(10, size=(5, 5)) x = np.random.randint(10, size=(5, 5))
x = sparsity.get_mask_1d(x, 2, 4) x = paddle.fluid.contrib.sparsity.get_mask_1d(x, 2, 4)
self.assertTrue(sparsity.check_mask_1d(x, 2, 4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 4))
x = np.random.randn(5, 4) x = np.random.randn(5, 4)
x = sparsity.get_mask_1d(x, 2, 4) x = paddle.fluid.contrib.sparsity.get_mask_1d(x, 2, 4)
self.assertTrue(sparsity.check_mask_1d(x, 2, 4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 4))
def test_check_mask_2d(self): def test_check_mask_2d(self):
x = np.array([[1.0, 0.0, 0.0, 1.0, 1.0], [0.0, 1.0, 0.0, 0.0, 0.0], x = np.array([[1.0, 0.0, 0.0, 1.0, 1.0], [0.0, 1.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 1.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.0, 0.0, 1.0]]) [0.0, 1.0, 0.0, 0.0, 1.0]])
self.assertTrue(sparsity.check_mask_2d(x, 2, 4)) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4))
self.assertFalse(sparsity.check_mask_2d(x, 3, 4)) self.assertFalse(paddle.fluid.contrib.sparsity.check_mask_2d(x, 3, 4))
self.assertTrue(sparsity.check_mask_2d(x, 2, 5)) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 5))
self.assertFalse(sparsity.check_mask_2d(x, 3, 5)) self.assertFalse(paddle.fluid.contrib.sparsity.check_mask_2d(x, 3, 5))
self.assertTrue(sparsity.check_mask_2d(x, 3, 6)) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d(x, 3, 6))
self.assertFalse(sparsity.check_mask_2d(x, 4, 6)) self.assertFalse(paddle.fluid.contrib.sparsity.check_mask_2d(x, 4, 6))
def test_get_mask_2d_greedy(self): def test_get_mask_2d_greedy(self):
for _ in range(10): for _ in range(10):
x = np.random.randint(10, size=(5, 5)) x = np.random.randint(10, size=(5, 5))
x = sparsity.get_mask_2d_greedy(x, 2, 4) x = paddle.fluid.contrib.sparsity.get_mask_2d_greedy(x, 2, 4)
self.assertTrue(sparsity.check_mask_2d(x, 2, 4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4))
x = np.random.randn(5, 4) x = np.random.randn(5, 4)
x = sparsity.get_mask_2d_greedy(x, 2, 4) x = paddle.fluid.contrib.sparsity.get_mask_2d_greedy(x, 2, 4)
self.assertTrue(sparsity.check_mask_2d(x, 2, 4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4))
def test_get_mask_2d_best(self): def test_get_mask_2d_best(self):
for _ in range(10): for _ in range(10):
x = np.random.randint(10, size=(5, 5)) x = np.random.randint(10, size=(5, 5))
x = sparsity.get_mask_2d_best(x, 2, 4) x = paddle.fluid.contrib.sparsity.get_mask_2d_best(x, 2, 4)
self.assertTrue(sparsity.check_mask_2d(x, 2, 4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4))
x = np.random.randn(5, 4) x = np.random.randn(5, 4)
x = sparsity.get_mask_2d_best(x, 2, 4) x = paddle.fluid.contrib.sparsity.get_mask_2d_best(x, 2, 4)
self.assertTrue(sparsity.check_mask_2d(x, 2, 4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4))
def test_threadsafe_valid_2d_patterns(self): def test_threadsafe_valid_2d_patterns(self):
def get_reference(m=4, n=2): def get_reference(m=4, n=2):
...@@ -160,30 +168,54 @@ class TestASPUtils(unittest.TestCase): ...@@ -160,30 +168,54 @@ class TestASPUtils(unittest.TestCase):
self.__test_1D_2D_sparse_mask_generation_methods(x) self.__test_1D_2D_sparse_mask_generation_methods(x)
def __test_1D_2D_sparsity_checking_methods(self, x_2d): def __test_1D_2D_sparsity_checking_methods(self, x_2d):
mask = sparsity.get_mask_1d(x_2d, 2, 4) mask = paddle.fluid.contrib.sparsity.get_mask_1d(x_2d, 2, 4)
self.assertEqual( self.assertEqual(
sparsity.check_sparsity( paddle.fluid.contrib.sparsity.check_sparsity(
mask, func_name=sparsity.CheckMethod.CHECK_1D, n=2, m=4), mask,
sparsity.check_mask_1d(mask, 2, 4)) func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D,
mask = sparsity.get_mask_2d_best(x_2d, 2, 4) n=2,
m=4),
paddle.fluid.contrib.sparsity.check_mask_1d(mask, 2, 4))
mask = paddle.fluid.contrib.sparsity.get_mask_2d_best(x_2d, 2, 4)
self.assertEqual( self.assertEqual(
sparsity.check_sparsity( paddle.fluid.contrib.sparsity.check_sparsity(
mask, func_name=sparsity.CheckMethod.CHECK_2D, n=2, m=4), mask,
sparsity.check_mask_2d(mask, 2, 4)) func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D,
n=2,
m=4),
paddle.fluid.contrib.sparsity.check_mask_2d(mask, 2, 4))
def __test_1D_2D_sparse_mask_generation_methods(self, x): def __test_1D_2D_sparse_mask_generation_methods(self, x):
mask = sparsity.create_mask( mask = paddle.fluid.contrib.sparsity.create_mask(
x, func_name=sparsity.MaskAlgo.MASK_1D, n=2, m=4) x,
func_name=paddle.fluid.contrib.sparsity.MaskAlgo.MASK_1D,
n=2,
m=4)
self.assertTrue( self.assertTrue(
sparsity.check_sparsity( paddle.fluid.contrib.sparsity.check_sparsity(
mask, func_name=sparsity.CheckMethod.CHECK_1D, n=2, m=4)) mask,
mask = sparsity.create_mask( func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D,
x, func_name=sparsity.MaskAlgo.MASK_2D_GREEDY, n=2, m=4) n=2,
m=4))
mask = paddle.fluid.contrib.sparsity.create_mask(
x,
func_name=paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_GREEDY,
n=2,
m=4)
self.assertTrue( self.assertTrue(
sparsity.check_sparsity( paddle.fluid.contrib.sparsity.check_sparsity(
mask, func_name=sparsity.CheckMethod.CHECK_2D, n=2, m=4)) mask,
mask = sparsity.create_mask( func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D,
x, func_name=sparsity.MaskAlgo.MASK_2D_BEST, n=2, m=4) n=2,
m=4))
mask = paddle.fluid.contrib.sparsity.create_mask(
x,
func_name=paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_BEST,
n=2,
m=4)
self.assertTrue( self.assertTrue(
sparsity.check_sparsity( paddle.fluid.contrib.sparsity.check_sparsity(
mask, func_name=sparsity.CheckMethod.CHECK_2D, n=2, m=4)) mask,
func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D,
n=2,
m=4))
...@@ -20,7 +20,7 @@ import paddle ...@@ -20,7 +20,7 @@ import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.core as core import paddle.fluid.core as core
import os import os
from paddle.fluid.contrib import sparsity from paddle.static import sparsity
from paddle.fluid.contrib.sparsity.asp import ASPHelper from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np import numpy as np
cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES') cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES')
...@@ -73,7 +73,7 @@ class TestFleetWithASP(unittest.TestCase): ...@@ -73,7 +73,7 @@ class TestFleetWithASP(unittest.TestCase):
feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place) feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place)
exe.run(startup_prog) exe.run(startup_prog)
sparsity.prune_model(place, train_prog) sparsity.prune_model(train_prog)
data = (np.random.randn(64, 32), np.random.randint(2, size=(64, 1))) data = (np.random.randn(64, 32), np.random.randint(2, size=(64, 1)))
exe.run(train_prog, feed=feeder.feed([data])) exe.run(train_prog, feed=feeder.feed([data]))
...@@ -82,7 +82,9 @@ class TestFleetWithASP(unittest.TestCase): ...@@ -82,7 +82,9 @@ class TestFleetWithASP(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name): if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(param.name) mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor()) .get_tensor())
self.assertTrue(sparsity.check_sparsity(mat.T, n=2, m=4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -20,7 +20,7 @@ import paddle ...@@ -20,7 +20,7 @@ import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.core as core import paddle.fluid.core as core
import os import os
from paddle.fluid.contrib import sparsity from paddle.static import sparsity
from paddle.fluid.contrib.sparsity.asp import ASPHelper from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np import numpy as np
cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES') cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES')
...@@ -76,7 +76,7 @@ class TestFleetWithASP(unittest.TestCase): ...@@ -76,7 +76,7 @@ class TestFleetWithASP(unittest.TestCase):
optimizer.amp_init(place) optimizer.amp_init(place)
sparsity.prune_model(place, train_prog) sparsity.prune_model(train_prog)
data = (np.random.randn(64, 32), np.random.randint(2, size=(64, 1))) data = (np.random.randn(64, 32), np.random.randint(2, size=(64, 1)))
exe.run(train_prog, feed=feeder.feed([data])) exe.run(train_prog, feed=feeder.feed([data]))
...@@ -85,7 +85,9 @@ class TestFleetWithASP(unittest.TestCase): ...@@ -85,7 +85,9 @@ class TestFleetWithASP(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name): if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(param.name) mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor()) .get_tensor())
self.assertTrue(sparsity.check_sparsity(mat.T, n=2, m=4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
def test_with_asp_and_pure_fp16(self): def test_with_asp_and_pure_fp16(self):
fleet.init(is_collective=True) fleet.init(is_collective=True)
...@@ -114,7 +116,7 @@ class TestFleetWithASP(unittest.TestCase): ...@@ -114,7 +116,7 @@ class TestFleetWithASP(unittest.TestCase):
optimizer.amp_init(place) optimizer.amp_init(place)
sparsity.prune_model(place, train_prog) sparsity.prune_model(train_prog)
data = (np.random.randn(64, 32), np.random.randint(2, size=(64, 1))) data = (np.random.randn(64, 32), np.random.randint(2, size=(64, 1)))
exe.run(train_prog, feed=feeder.feed([data])) exe.run(train_prog, feed=feeder.feed([data]))
...@@ -123,7 +125,9 @@ class TestFleetWithASP(unittest.TestCase): ...@@ -123,7 +125,9 @@ class TestFleetWithASP(unittest.TestCase):
if ASPHelper._is_supported_layer(train_prog, param.name): if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(param.name) mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor()) .get_tensor())
self.assertTrue(sparsity.check_sparsity(mat.T, n=2, m=4)) self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
if __name__ == "__main__": if __name__ == "__main__":
......
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2021 NVIDIA Corporation. All rights reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
...@@ -13,6 +14,7 @@ ...@@ -13,6 +14,7 @@
# limitations under the License. # limitations under the License.
from . import amp # noqa: F401 from . import amp # noqa: F401
from . import sparsity # noqa: F401
from . import nn # noqa: F401 from . import nn # noqa: F401
from .io import save_inference_model # noqa: F401 from .io import save_inference_model # noqa: F401
from .io import load_inference_model # noqa: F401 from .io import load_inference_model # noqa: F401
......
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2021 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ...fluid.contrib.sparsity import calculate_density #noqa: F401
from ...fluid.contrib.sparsity import decorate #noqa: F401
from ...fluid.contrib.sparsity import prune_model #noqa: F401
from ...fluid.contrib.sparsity import set_excluded_layers #noqa: F401
from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401
__all__ = [ #noqa
'calculate_density',
'decorate',
'prune_model',
'set_excluded_layers',
'reset_excluded_layers'
]
...@@ -357,6 +357,7 @@ packages=['paddle', ...@@ -357,6 +357,7 @@ packages=['paddle',
'paddle.static', 'paddle.static',
'paddle.static.nn', 'paddle.static.nn',
'paddle.static.amp', 'paddle.static.amp',
'paddle.static.sparsity',
'paddle.tensor', 'paddle.tensor',
'paddle.onnx', 'paddle.onnx',
'paddle.autograd', 'paddle.autograd',
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册