未验证 提交 e5fc68b2 编写于 作者: M Ming-Xu Huang 提交者: GitHub

Dynamic graph support to Automatic SParsity. (#41177)

* Dynamic graph support to Automatic SParsity.

1. Added dynamic support to ASP module (paddle.fluid.contrib.sparsity).
2. Added ASP related unit-tests regards to above changes.
3. Put ASP module under paddle.static for now, waiting for APIs confirmation from Paddle.

* Modified documents of functions to have correct examples.

* Update in_dygraph_mode to paddle.in_dynamic_mode()

* Modified documents of functions and added comments

* Minor changes.

* Fix example errors in asp API.

* Code Change for Review

1. Added more examples in documents.
2. Chaged test_asp_pruning_static.

* Minor changes

* Update ASP function documents.

* Update ASP function documents.

* Reduce test case size of asp pruning due CI time limit.

* Update time limitation to some asp UTs.

* Fix sample code errors.

* Fix sample code errors.

* Fix sample code errors.

* Update time limitation to parts of ASP UTs.

* Update UTs to fit with CI.

* Reduce problem size in python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py

* Added paddle.asp

* Fixed type casting error of OpRole.Optimize in new dygraph mode.

* Made set_excluded_layers be compatible with 2.2

* Fix example code of calculate_density.

* Update code examples.

* Move paddle.asp to paddle.incubate.asp

* Fixed an example error of calculate_density
上级 4218957b
......@@ -94,13 +94,12 @@ def calculate_density(x):
float: The density of :attr:`x`.
Examples:
.. code-block:: python
import paddle
import numpy as np
import paddle.static.sparsity as sparsity
x = np.array([[0, 1, 3, 0],
[1, 1, 0, 1]])
sparsity.calculate_density(x) # 0.625
paddle.incubate.asp.calculate_density(x) # 0.625
"""
x_flattened = x.flatten()
return float(np.nonzero(x_flattened)[0].size) / x_flattened.size
......
file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
list(REMOVE_ITEM TEST_OPS "test_fleet_with_asp")
list(REMOVE_ITEM TEST_OPS "test_fleet_with_asp_amp")
list(REMOVE_ITEM TEST_OPS "test_fleet_with_asp_static")
list(REMOVE_ITEM TEST_OPS "test_fleet_with_asp_dynamic")
list(REMOVE_ITEM TEST_OPS "test_fleet_with_asp_sharding")
foreach(TEST_OP ${TEST_OPS})
......@@ -10,9 +10,9 @@ foreach(TEST_OP ${TEST_OPS})
endforeach(TEST_OP)
if(WITH_DISTRIBUTE)
py_test_modules(test_fleet_with_asp MODULES test_fleet_with_asp ENVS ${dist_ENVS})
if (WITH_GPU OR WITH_XPU OR WITH_ASCEND OR WITH_ASCEND_CL)
py_test_modules(test_fleet_with_asp_amp MODULES test_fleet_with_asp_amp ENVS ${dist_ENVS})
py_test_modules(test_fleet_with_asp_dynamic MODULES test_fleet_with_asp_dynamic ENVS ${dist_ENVS})
py_test_modules(test_fleet_with_asp_static MODULES test_fleet_with_asp_static ENVS ${dist_ENVS})
endif()
endif()
......@@ -21,3 +21,8 @@ if((WITH_DISTRIBUTE) AND (NOT WIN32) AND (NOT APPLE))
py_test_modules(test_fleet_with_asp_sharding MODULES test_fleet_with_asp_sharding ENVS ${dist_ENVS})
endif()
endif()
set_tests_properties(test_asp_pruning_dynamic PROPERTIES TIMEOUT 30)
set_tests_properties(test_asp_pruning_static PROPERTIES TIMEOUT 30)
set_tests_properties(test_asp_optimize_dynamic PROPERTIES TIMEOUT 30)
set_tests_properties(test_asp_optimize_static PROPERTIES TIMEOUT 30)
......@@ -20,7 +20,6 @@ import threading, time
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
from paddle.static import sparsity
from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np
......@@ -60,7 +59,7 @@ class TestASPHelperPruningBase(unittest.TestCase):
loss = fluid.layers.mean(
fluid.layers.cross_entropy(
input=self.predict, label=self.label))
optimizer = sparsity.decorate(
optimizer = paddle.incubate.asp.decorate(
fluid.optimizer.SGD(learning_rate=0.01))
optimizer.minimize(loss, self.startup_program)
......@@ -75,7 +74,7 @@ class TestASPHelperPruningBase(unittest.TestCase):
def __pruning_and_checking(self, exe, place, mask_func_name,
check_func_name, with_mask):
exe.run(self.startup_program)
sparsity.prune_model(
paddle.incubate.asp.prune_model(
self.main_program, mask_algo=mask_func_name, with_mask=with_mask)
for param in self.main_program.global_block().all_parameters():
if ASPHelper._is_supported_layer(self.main_program, param.name):
......
......@@ -66,6 +66,97 @@ class TestASPAddSupportedLayer(unittest.TestCase):
my_own_layer_name in supported_layers_and_prune_func_map)
class TestASPDynamicCustomerizedPruneFunc(unittest.TestCase):
def setUp(self):
paddle.disable_static()
class CustomerLayer(paddle.nn.Layer):
def __init__(self):
super(CustomerLayer, self).__init__()
self.weight = self.create_parameter(
shape=[32, 32], attr=None, dtype='float32', is_bias=False)
self.linear1 = paddle.nn.Linear(32, 32)
self.linear2 = paddle.nn.Linear(32, 10)
def forward(self, input_):
hidden = paddle.nn.functional.linear(
x=input_, weight=self.weight)
hidden = self.linear1(hidden)
out = self.linear2(hidden)
return out
sparsity.add_supported_layer(CustomerLayer, my_own_pruning)
self.layer = CustomerLayer()
self.customer_prefix = paddle.fluid.dygraph.layers._convert_camel_to_snake(
CustomerLayer.__name__)
self.supported_layer_count_ref = 3
def test_inference_pruning(self):
sparsity.prune_model(self.layer, mask_algo="mask_1d", with_mask=False)
supported_layer_count = 0
for param in self.layer.parameters():
mat = param.numpy()
if sparsity.asp.ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
supported_layer_count += 1
if (self.customer_prefix in param.name):
self.assertLessEqual(
np.sum(mat.flatten() - static_tensor.flatten()), 1e-4)
else:
self.assertTrue(
sparsity.check_sparsity(
mat.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
self.assertEqual(supported_layer_count, self.supported_layer_count_ref)
def test_training_pruning(self):
optimizer = paddle.optimizer.SGD(learning_rate=0.01,
parameters=self.layer.parameters())
optimizer = sparsity.decorate(optimizer)
sparsity.prune_model(self.layer, mask_algo="mask_1d", with_mask=True)
supported_layer_count = 0
for param in self.layer.parameters():
mat = param.numpy()
if sparsity.asp.ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat_mask = sparsity.asp.ASPHelper._get_program_asp_info(
paddle.static.default_main_program()).mask_vars[
param.name].numpy()
supported_layer_count += 1
if (self.customer_prefix in param.name):
self.assertLessEqual(
np.sum(mat.flatten() - static_tensor.flatten()), 1e-4)
self.assertLessEqual(
np.sum(mat_mask.flatten() - static_tensor_mask.flatten(
)), 1e-4)
else:
self.assertTrue(
sparsity.check_sparsity(
mat.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
self.assertTrue(
sparsity.check_sparsity(
mat_mask.T,
func_name=sparsity.CheckMethod.CHECK_1D,
n=2,
m=4))
self.assertEqual(supported_layer_count, self.supported_layer_count_ref)
class TestASPStaticCustomerizedPruneFunc(unittest.TestCase):
def setUp(self):
paddle.enable_static()
......
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import unittest
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np
class MyLayer(paddle.nn.Layer):
def __init__(self):
super(MyLayer, self).__init__()
self.conv1 = paddle.nn.Conv2D(
in_channels=3, out_channels=2, kernel_size=3, padding=2)
self.linear1 = paddle.nn.Linear(1352, 32)
self.linear2 = paddle.nn.Linear(32, 32)
self.linear3 = paddle.nn.Linear(32, 10)
def forward(self, img):
hidden = self.conv1(img)
hidden = paddle.flatten(hidden, start_axis=1)
hidden = self.linear1(hidden)
hidden = self.linear2(hidden)
prediction = self.linear3(hidden)
return prediction
class TestASPDynamicOptimize(unittest.TestCase):
def setUp(self):
self.layer = MyLayer()
self.place = paddle.CPUPlace()
if core.is_compiled_with_cuda():
self.place = paddle.CUDAPlace(0)
self.optimizer = paddle.optimizer.SGD(
learning_rate=0.01, parameters=self.layer.parameters())
def test_is_supported_layers(self):
program = paddle.static.default_main_program()
names = [
'embedding_0.w_0', 'fack_layer_0.w_0', 'conv2d_0.w_0',
'conv2d_0.b_0', 'conv2d_1.w_0', 'conv2d_1.b_0', 'fc_0.w_0',
'fc_0.b_0', 'fc_1.w_0', 'fc_1.b_0', 'linear_2.w_0', 'linear_2.b_0'
]
ref = [
False, False, True, False, True, False, True, False, True, False,
True, False
]
for i, name in enumerate(names):
self.assertTrue(
ref[i] == ASPHelper._is_supported_layer(program, name))
paddle.incubate.asp.set_excluded_layers(['fc_1', 'conv2d_0'])
ref = [
False, False, False, False, True, False, True, False, False, False,
True, False
]
for i, name in enumerate(names):
self.assertTrue(
ref[i] == ASPHelper._is_supported_layer(program, name))
paddle.incubate.asp.reset_excluded_layers()
ref = [
False, False, True, False, True, False, True, False, True, False,
True, False
]
for i, name in enumerate(names):
self.assertTrue(
ref[i] == ASPHelper._is_supported_layer(program, name))
def test_decorate(self):
param_names = [param.name for param in self.layer.parameters()]
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
program = paddle.static.default_main_program()
for name in param_names:
mask_var = ASPHelper._get_program_asp_info(program).mask_vars.get(
name, None)
if ASPHelper._is_supported_layer(program, name):
self.assertTrue(mask_var is not None)
else:
self.assertTrue(mask_var is None)
def test_asp_training(self):
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
paddle.incubate.asp.prune_model(self.layer)
imgs = paddle.to_tensor(
np.random.randn(32, 3, 24, 24),
dtype='float32',
place=self.place,
stop_gradient=False)
labels = paddle.to_tensor(
np.random.randint(
10, size=(32, 1)),
dtype='float32',
place=self.place,
stop_gradient=False)
loss_fn = paddle.nn.MSELoss(reduction='mean')
output = self.layer(imgs)
loss = loss_fn(output, labels)
loss.backward()
self.optimizer.step()
self.optimizer.clear_grad()
for param in self.layer.parameters():
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
def test_asp_training_with_amp(self):
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
paddle.incubate.asp.prune_model(self.layer)
imgs = paddle.to_tensor(
np.random.randn(32, 3, 24, 24),
dtype='float32',
place=self.place,
stop_gradient=False)
labels = paddle.to_tensor(
np.random.randint(
10, size=(32, 1)),
dtype='float32',
place=self.place,
stop_gradient=False)
loss_fn = paddle.nn.MSELoss(reduction='mean')
scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
with paddle.amp.auto_cast(enable=True):
output = self.layer(imgs)
loss = loss_fn(output, labels)
scaled = scaler.scale(loss)
scaled.backward()
scaler.minimize(self.optimizer, scaled)
self.optimizer.clear_grad()
for param in self.layer.parameters():
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
if __name__ == '__main__':
unittest.main()
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2021 NVIDIA Corporation. All rights reserved.
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -20,21 +20,20 @@ import threading, time
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
from paddle.static import sparsity
from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np
paddle.enable_static()
class TestASPHelper(unittest.TestCase):
class TestASPStaticOptimize(unittest.TestCase):
def setUp(self):
self.main_program = fluid.Program()
self.startup_program = fluid.Program()
def build_model():
img = fluid.data(
name='img', shape=[None, 3, 32, 32], dtype='float32')
name='img', shape=[None, 3, 24, 24], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
hidden = fluid.layers.conv2d(
input=img, num_filters=4, filter_size=3, padding=2, act="relu")
......@@ -87,7 +86,7 @@ class TestASPHelper(unittest.TestCase):
self.assertTrue(
ref[i] == ASPHelper._is_supported_layer(program, name))
sparsity.set_excluded_layers(program, ['fc_1', 'conv2d_0'])
paddle.incubate.asp.set_excluded_layers(['fc_1', 'conv2d_0'], program)
ref = [
False, False, False, False, True, False, True, False, False, False,
True, False
......@@ -96,7 +95,7 @@ class TestASPHelper(unittest.TestCase):
self.assertTrue(
ref[i] == ASPHelper._is_supported_layer(program, name))
sparsity.reset_excluded_layers(program)
paddle.incubate.asp.reset_excluded_layers(program)
ref = [
False, False, True, False, True, False, True, False, True, False,
True, False
......@@ -109,7 +108,7 @@ class TestASPHelper(unittest.TestCase):
param_names = self.__get_param_names(self.main_program.global_block()
.all_parameters())
with fluid.program_guard(self.main_program, self.startup_program):
self.optimizer = sparsity.decorate(self.optimizer)
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
self.optimizer.minimize(self.loss, self.startup_program)
param_names_after_minimize = self.__get_param_names(
self.main_program.global_block().all_parameters())
......@@ -119,7 +118,7 @@ class TestASPHelper(unittest.TestCase):
def test_asp_training(self):
with fluid.program_guard(self.main_program, self.startup_program):
self.optimizer = sparsity.decorate(self.optimizer)
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
self.optimizer.minimize(self.loss, self.startup_program)
place = paddle.CPUPlace()
......@@ -129,10 +128,10 @@ class TestASPHelper(unittest.TestCase):
feeder = fluid.DataFeeder(feed_list=[self.img, self.label], place=place)
exe.run(self.startup_program)
sparsity.prune_model(self.main_program)
paddle.incubate.asp.prune_model(self.main_program)
data = (np.random.randn(64, 3, 32, 32), np.random.randint(
10, size=(64, 1)))
data = (np.random.randn(32, 3, 24, 24), np.random.randint(
10, size=(32, 1)))
exe.run(self.main_program, feed=feeder.feed([data]))
for param in self.main_program.global_block().all_parameters():
......@@ -149,7 +148,7 @@ class TestASPHelper(unittest.TestCase):
with fluid.program_guard(self.main_program, self.startup_program):
self.optimizer = fluid.contrib.mixed_precision.decorator.decorate(
self.optimizer)
self.optimizer = sparsity.decorate(self.optimizer)
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
self.optimizer.minimize(self.loss, self.startup_program)
exe = fluid.Executor(place)
......@@ -157,10 +156,10 @@ class TestASPHelper(unittest.TestCase):
feed_list=[self.img, self.label], place=place)
exe.run(self.startup_program)
sparsity.prune_model(self.main_program)
paddle.incubate.asp.prune_model(self.main_program)
data = (np.random.randn(64, 3, 32, 32), np.random.randint(
10, size=(64, 1)))
data = (np.random.randn(32, 3, 24, 24), np.random.randint(
10, size=(32, 1)))
exe.run(self.main_program, feed=feeder.feed([data]))
for param in self.main_program.global_block().all_parameters():
......
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2021 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import paddle
import unittest
from paddle.static import sparsity
from paddle.fluid.tests.unittests.asp.asp_pruning_base import TestASPHelperPruningBase
paddle.enable_static()
class TestASPHelperPruning2DBest(TestASPHelperPruningBase):
def test_2D_best_inference_pruning(self):
self.run_inference_pruning_test(
'mask_2d_best', paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D)
def test_2D_best_training_pruning(self):
self.run_training_pruning_test(
'mask_2d_best', paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D)
if __name__ == '__main__':
unittest.main()
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2021 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import unittest
import paddle
from paddle.static import sparsity
from paddle.fluid.tests.unittests.asp.asp_pruning_base import TestASPHelperPruningBase
paddle.enable_static()
class TestASPHelperPruning2DGreedy(TestASPHelperPruningBase):
def test_2D_greedy_inference_pruning(self):
self.run_inference_pruning_test(
'mask_2d_greedy',
paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D)
def test_2D_greedy_training_pruning(self):
self.run_training_pruning_test(
'mask_2d_greedy',
paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D)
if __name__ == '__main__':
unittest.main()
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import unittest
import numpy as np
import paddle
from paddle.fluid import core
from paddle.fluid.contrib.sparsity.asp import ASPHelper
class MyLayer(paddle.nn.Layer):
def __init__(self):
super(MyLayer, self).__init__()
self.conv1 = paddle.nn.Conv2D(
in_channels=3, out_channels=2, kernel_size=3, padding=2)
self.linear1 = paddle.nn.Linear(1352, 32)
self.linear2 = paddle.nn.Linear(32, 10)
def forward(self, img):
hidden = self.conv1(img)
hidden = paddle.flatten(hidden, start_axis=1)
hidden = self.linear1(hidden)
prediction = self.linear2(hidden)
return prediction
class TestASPDynamicPruningBase(unittest.TestCase):
def setUp(self):
self.layer = MyLayer()
place = paddle.CPUPlace()
if core.is_compiled_with_cuda():
place = paddle.CUDAPlace(0)
self.img = paddle.to_tensor(
np.random.uniform(
low=-0.5, high=0.5, size=(32, 3, 24, 24)),
dtype=np.float32,
place=place,
stop_gradient=False)
self.set_config()
def set_config(self):
self.mask_gen_func = 'mask_1d'
self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D
def test_inference_pruning(self):
self.__pruning_and_checking(False)
def test_training_pruning(self):
optimizer = paddle.optimizer.SGD(learning_rate=0.01,
parameters=self.layer.parameters())
optimizer = paddle.incubate.asp.decorate(optimizer)
self.__pruning_and_checking(True)
def __pruning_and_checking(self, with_mask):
paddle.incubate.asp.prune_model(
self.layer, mask_algo=self.mask_gen_func, with_mask=with_mask)
for param in self.layer.parameters():
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, func_name=self.mask_check_func, n=2, m=4))
class TestASPDynamicPruning1D(TestASPDynamicPruningBase):
def set_config(self):
self.mask_gen_func = 'mask_1d'
self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D
class TestASPDynamicPruning2DBest(TestASPDynamicPruningBase):
def set_config(self):
self.mask_gen_func = 'mask_2d_best'
self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D
class TestASPDynamicPruning2DGreedy(TestASPDynamicPruningBase):
def set_config(self):
self.mask_gen_func = 'mask_2d_greedy'
self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D
if __name__ == '__main__':
unittest.main()
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import unittest
import threading, time
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np
paddle.enable_static()
class TestASPStaticPruningBase(unittest.TestCase):
def setUp(self):
self.main_program = fluid.Program()
self.startup_program = fluid.Program()
def build_model():
img = fluid.data(
name='img', shape=[None, 3, 24, 24], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
hidden = fluid.layers.conv2d(
input=img, num_filters=2, filter_size=3, padding=2, act="relu")
hidden = fluid.layers.fc(input=hidden, size=32, act='softmax')
prediction = fluid.layers.fc(input=hidden, size=10, act='softmax')
return img, label, prediction
with fluid.program_guard(self.main_program, self.startup_program):
self.img, self.label, self.predict = build_model()
self.set_config()
def set_config(self):
self.mask_gen_func = 'mask_1d'
self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D
def test_inference_pruning(self):
place = paddle.CPUPlace()
if core.is_compiled_with_cuda():
place = paddle.CUDAPlace(0)
exe = fluid.Executor(place)
self.__pruning_and_checking(exe, place, False)
def test_training_pruning(self):
with fluid.program_guard(self.main_program, self.startup_program):
loss = fluid.layers.mean(
fluid.layers.cross_entropy(
input=self.predict, label=self.label))
optimizer = paddle.incubate.asp.decorate(
fluid.optimizer.SGD(learning_rate=0.01))
optimizer.minimize(loss, self.startup_program)
place = paddle.CPUPlace()
if core.is_compiled_with_cuda():
place = paddle.CUDAPlace(0)
exe = fluid.Executor(place)
self.__pruning_and_checking(exe, place, True)
def __pruning_and_checking(self, exe, place, with_mask):
exe.run(self.startup_program)
paddle.incubate.asp.prune_model(
self.main_program,
mask_algo=self.mask_gen_func,
with_mask=with_mask)
for param in self.main_program.global_block().all_parameters():
if ASPHelper._is_supported_layer(self.main_program, param.name):
mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, func_name=self.mask_check_func, n=2, m=4))
class TestASPStaticPruning1D(TestASPStaticPruningBase):
def set_config(self):
self.mask_gen_func = 'mask_1d'
self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D
class TestASPStaticPruning2DBest(TestASPStaticPruningBase):
def set_config(self):
self.mask_gen_func = 'mask_2d_best'
self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D
class TestASPStaticPruning2DGreedy(TestASPStaticPruningBase):
def set_config(self):
self.mask_gen_func = 'mask_2d_greedy'
self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D
if __name__ == '__main__':
unittest.main()
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import unittest
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np
class MyLayer(paddle.nn.Layer):
def __init__(self):
super(MyLayer, self).__init__()
self.conv1 = paddle.nn.Conv2D(
in_channels=3, out_channels=4, kernel_size=3, padding=2)
self.linear1 = paddle.nn.Linear(4624, 32)
self.linear2 = paddle.nn.Linear(32, 32)
self.linear3 = paddle.nn.Linear(32, 10)
def forward(self, img):
hidden = self.conv1(img)
hidden = paddle.flatten(hidden, start_axis=1)
hidden = self.linear1(hidden)
hidden = self.linear2(hidden)
prediction = self.linear3(hidden)
return prediction
class TestASPDynamicOptimize(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.layer = MyLayer()
self.place = paddle.CPUPlace()
if core.is_compiled_with_cuda():
self.place = paddle.CUDAPlace(0)
self.optimizer = paddle.optimizer.SGD(
learning_rate=0.01, parameters=self.layer.parameters())
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
paddle.incubate.asp.prune_model(self.layer)
def test_save_and_load(self):
path = "/tmp/paddle_asp_save_dy/"
net_path = path + "asp_net.pdparams"
opt_path = path + "asp_opt.pdopt"
paddle.save(self.layer.state_dict(), net_path)
paddle.save(self.optimizer.state_dict(), opt_path)
asp_info = ASPHelper._get_program_asp_info(
paddle.static.default_main_program())
for param_name in asp_info.mask_vars:
mask = asp_info.mask_vars[param_name]
asp_info.update_mask_vars(
param_name, paddle.ones(
shape=mask.shape, dtype=mask.dtype))
asp_info.update_masks(param_name, np.ones(shape=mask.shape))
net_state_dict = paddle.load(net_path)
opt_state_dict = paddle.load(opt_path)
self.layer.set_state_dict(net_state_dict)
self.optimizer.set_state_dict(opt_state_dict)
imgs = paddle.to_tensor(
np.random.randn(64, 3, 32, 32),
dtype='float32',
place=self.place,
stop_gradient=False)
labels = paddle.to_tensor(
np.random.randint(
10, size=(64, 1)),
dtype='float32',
place=self.place,
stop_gradient=False)
loss_fn = paddle.nn.MSELoss(reduction='mean')
output = self.layer(imgs)
loss = loss_fn(output, labels)
loss.backward()
self.optimizer.step()
self.optimizer.clear_grad()
for param in self.layer.parameters():
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
class TestASPStaticOptimize(unittest.TestCase):
def setUp(self):
paddle.enable_static()
self.main_program = fluid.Program()
self.startup_program = fluid.Program()
def build_model():
img = fluid.data(
name='img', shape=[None, 3, 32, 32], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
hidden = fluid.layers.conv2d(
input=img, num_filters=4, filter_size=3, padding=2, act="relu")
hidden = fluid.layers.fc(input=hidden, size=32, act='relu')
prediction = fluid.layers.fc(input=hidden, size=10, act='softmax')
return img, label, prediction
with fluid.program_guard(self.main_program, self.startup_program):
self.img, self.label, predict = build_model()
self.loss = fluid.layers.mean(
fluid.layers.cross_entropy(
input=predict, label=self.label))
self.optimizer = fluid.optimizer.SGD(learning_rate=0.01)
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
self.optimizer.minimize(self.loss, self.startup_program)
self.place = paddle.CPUPlace()
if core.is_compiled_with_cuda():
self.place = paddle.CUDAPlace(0)
self.exe = fluid.Executor(self.place)
self.exe.run(self.startup_program)
paddle.incubate.asp.prune_model(self.main_program)
def test_save_and_load(self):
path = "/tmp/paddle_asp_save_st/"
param_path = path + "asp.pdparams"
model_path = path + "asp.pdmodel"
paddle.save(self.main_program.state_dict(), param_path)
paddle.save(self.main_program, model_path)
prog = paddle.load(model_path)
state_dict = paddle.load(param_path)
prog.set_state_dict(state_dict)
feeder = fluid.DataFeeder(
feed_list=[self.img, self.label], place=self.place)
data = (np.random.randn(64, 3, 32, 32), np.random.randint(
10, size=(64, 1)))
self.exe.run(prog, feed=feeder.feed([data]))
for param in prog.global_block().all_parameters():
if ASPHelper._is_supported_layer(prog, param.name):
mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
if __name__ == '__main__':
unittest.main()
......@@ -18,7 +18,6 @@ from __future__ import print_function
import unittest
import threading, time
import paddle
from paddle.static import sparsity
import numpy as np
......@@ -41,9 +40,9 @@ class TestASPUtils(unittest.TestCase):
x = np.array([[1.0, 1.0, 1.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0],
[1.0, 0.0, 0.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0],
[0.0, 1.0, 0.0, 0.0, 1.0]])
self.assertEqual(sparsity.calculate_density(x), 0.56)
self.assertEqual(paddle.incubate.asp.calculate_density(x), 0.56)
x[:, 0] = 0.0
self.assertEqual(sparsity.calculate_density(x), 0.4)
self.assertEqual(paddle.incubate.asp.calculate_density(x), 0.4)
def test_check_mask_1d(self):
x = np.array([[1.0, 0.0, 0.0, 1.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0],
......@@ -219,3 +218,7 @@ class TestASPUtils(unittest.TestCase):
func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D,
n=2,
m=4))
if __name__ == '__main__':
unittest.main()
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2021 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle.distributed.fleet as fleet
import paddle.distributed.fleet.base.role_maker as role_maker
import unittest
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
import os
from paddle.static import sparsity
from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np
cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES')
if cuda_visible_devices is None or cuda_visible_devices == "":
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
else:
os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devices.split(',')[0]
paddle.enable_static()
class TestFleetWithASP(unittest.TestCase):
def setUp(self):
os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213"
os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213"
os.environ["PADDLE_TRAINERS_NUM"] = "1"
os.environ["PADDLE_TRAINER_ID"] = "0"
def net(self, main_prog, startup_prog):
with fluid.program_guard(main_prog, startup_prog):
input_x = paddle.static.data(
name="x", shape=[-1, 32], dtype='float32')
input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64')
fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh')
prediction = fluid.layers.fc(input=fc_1, size=2, act='softmax')
cost = fluid.layers.cross_entropy(input=prediction, label=input_y)
avg_cost = paddle.mean(x=cost)
strategy = paddle.distributed.fleet.DistributedStrategy()
strategy.asp = True
return avg_cost, strategy, input_x, input_y
def test_with_asp(self):
fleet.init(is_collective=True)
train_prog, startup_prog = fluid.Program(), fluid.Program()
avg_cost, strategy, input_x, input_y = self.net(train_prog,
startup_prog)
with fluid.program_guard(train_prog, startup_prog):
optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01)
optimizer = fleet.distributed_optimizer(
optimizer, strategy=strategy)
optimizer.minimize(avg_cost)
place = fluid.CUDAPlace(0) if paddle.fluid.is_compiled_with_cuda(
) else fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place)
exe.run(startup_prog)
sparsity.prune_model(train_prog)
data = (np.random.randn(64, 32), np.random.randint(2, size=(64, 1)))
exe.run(train_prog, feed=feeder.feed([data]))
for param in train_prog.global_block().all_parameters():
if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
if __name__ == "__main__":
unittest.main()
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle.distributed.fleet as fleet
import paddle.distributed.fleet.base.role_maker as role_maker
import unittest
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
import os
from paddle.fluid.contrib.sparsity.asp import ASPHelper
import numpy as np
cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES')
if cuda_visible_devices is None or cuda_visible_devices == "":
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
else:
os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devices.split(',')[0]
class MyLayer(paddle.nn.Layer):
def __init__(self):
super(MyLayer, self).__init__()
self.linear1 = paddle.nn.Linear(32, 32)
self.linear2 = paddle.nn.Linear(32, 10)
def forward(self, x):
hidden = self.linear1(x)
prediction = self.linear2(hidden)
return prediction
class TestFleetWithASPDynamic(unittest.TestCase):
def setUp(self):
os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213"
os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213"
os.environ["PADDLE_TRAINERS_NUM"] = "1"
os.environ["PADDLE_TRAINER_ID"] = "0"
self.layer = MyLayer()
self.place = paddle.CPUPlace()
if core.is_compiled_with_cuda():
self.place = paddle.CUDAPlace(0)
self.optimizer = paddle.optimizer.SGD(
learning_rate=0.01, parameters=self.layer.parameters())
def test_with_asp(self):
fleet.init(is_collective=True)
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
paddle.incubate.asp.prune_model(self.layer)
self.optimizer = fleet.distributed_optimizer(self.optimizer)
self.layer = fleet.distributed_model(self.layer)
imgs = paddle.to_tensor(
np.random.randn(64, 32),
dtype='float32',
place=self.place,
stop_gradient=False)
labels = paddle.to_tensor(
np.random.randint(
10, size=(64, 1)),
dtype='float32',
place=self.place,
stop_gradient=False)
loss_fn = paddle.nn.MSELoss(reduction='mean')
output = self.layer(imgs)
loss = loss_fn(output, labels)
loss.backward()
self.optimizer.step()
self.optimizer.clear_grad()
for param in self.layer.parameters():
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
class TestFleetWithASPAMPDynamic(unittest.TestCase):
def setUp(self):
os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213"
os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213"
os.environ["PADDLE_TRAINERS_NUM"] = "1"
os.environ["PADDLE_TRAINER_ID"] = "0"
self.layer = MyLayer()
self.place = paddle.CPUPlace()
if core.is_compiled_with_cuda():
self.place = paddle.CUDAPlace(0)
self.optimizer = paddle.optimizer.SGD(
learning_rate=0.01, parameters=self.layer.parameters())
def test_with_asp(self):
fleet.init(is_collective=True)
self.optimizer = paddle.incubate.asp.decorate(self.optimizer)
paddle.incubate.asp.prune_model(self.layer)
self.optimizer = fleet.distributed_optimizer(self.optimizer)
self.layer = fleet.distributed_model(self.layer)
imgs = paddle.to_tensor(
np.random.randn(64, 32),
dtype='float32',
place=self.place,
stop_gradient=False)
labels = paddle.to_tensor(
np.random.randint(
10, size=(64, 1)),
dtype='float32',
place=self.place,
stop_gradient=False)
loss_fn = paddle.nn.MSELoss(reduction='mean')
scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
with paddle.amp.auto_cast(enable=True):
output = self.layer(imgs)
loss = loss_fn(output, labels)
scaled = scaler.scale(loss)
scaled.backward()
scaler.minimize(self.optimizer, scaled)
self.optimizer.clear_grad()
for param in self.layer.parameters():
if ASPHelper._is_supported_layer(
paddle.static.default_main_program(), param.name):
mat = param.numpy()
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
if __name__ == "__main__":
unittest.main()
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2021 NVIDIA Corporation. All rights reserved.
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -32,7 +32,62 @@ else:
paddle.enable_static()
class TestFleetWithASP(unittest.TestCase):
class TestFleetWithASPStatic(unittest.TestCase):
def setUp(self):
os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213"
os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213"
os.environ["PADDLE_TRAINERS_NUM"] = "1"
os.environ["PADDLE_TRAINER_ID"] = "0"
def net(self, main_prog, startup_prog):
with fluid.program_guard(main_prog, startup_prog):
input_x = paddle.static.data(
name="x", shape=[-1, 32], dtype='float32')
input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64')
fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh')
prediction = fluid.layers.fc(input=fc_1, size=2, act='softmax')
cost = fluid.layers.cross_entropy(input=prediction, label=input_y)
avg_cost = paddle.mean(x=cost)
strategy = paddle.distributed.fleet.DistributedStrategy()
strategy.asp = True
return avg_cost, strategy, input_x, input_y
def test_with_asp(self):
fleet.init(is_collective=True)
train_prog, startup_prog = fluid.Program(), fluid.Program()
avg_cost, strategy, input_x, input_y = self.net(train_prog,
startup_prog)
with fluid.program_guard(train_prog, startup_prog):
optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01)
optimizer = fleet.distributed_optimizer(
optimizer, strategy=strategy)
optimizer.minimize(avg_cost)
place = fluid.CUDAPlace(0) if paddle.fluid.is_compiled_with_cuda(
) else fluid.CPUPlace()
exe = fluid.Executor(place)
feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place)
exe.run(startup_prog)
sparsity.prune_model(train_prog)
data = (np.random.randn(64, 32), np.random.randint(2, size=(64, 1)))
exe.run(train_prog, feed=feeder.feed([data]))
for param in train_prog.global_block().all_parameters():
if ASPHelper._is_supported_layer(train_prog, param.name):
mat = np.array(fluid.global_scope().find_var(param.name)
.get_tensor())
self.assertTrue(
paddle.fluid.contrib.sparsity.check_sparsity(
mat.T, n=2, m=4))
class TestFleetWithASPAMPStatic(unittest.TestCase):
def setUp(self):
os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213"
os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213"
......
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2022 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......
......@@ -32,6 +32,7 @@ import paddle.incubate.autograd
import paddle.incubate.autotune
from . import nn #noqa: F401
from . import asp #noqa: F401
__all__ = [
'LookAhead',
......
......@@ -13,25 +13,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import unittest
import paddle
from paddle.static import sparsity
from paddle.fluid.tests.unittests.asp.asp_pruning_base import TestASPHelperPruningBase
paddle.enable_static()
class TestASPHelperPruning1D(TestASPHelperPruningBase):
def test_1D_inference_pruning(self):
self.run_inference_pruning_test(
'mask_1d', paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D)
def test_1D_training_pruning(self):
self.run_training_pruning_test(
'mask_1d', paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D)
if __name__ == '__main__':
unittest.main()
from ...fluid.contrib.sparsity import calculate_density #noqa: F401
from ...fluid.contrib.sparsity import decorate #noqa: F401
from ...fluid.contrib.sparsity import prune_model #noqa: F401
from ...fluid.contrib.sparsity import set_excluded_layers #noqa: F401
from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401
__all__ = [ #noqa
'calculate_density',
'decorate',
'prune_model',
'set_excluded_layers',
'reset_excluded_layers'
]
......@@ -16,8 +16,14 @@
from ...fluid.contrib.sparsity import calculate_density #noqa: F401
from ...fluid.contrib.sparsity import decorate #noqa: F401
from ...fluid.contrib.sparsity import prune_model #noqa: F401
from ...fluid.contrib.sparsity import set_excluded_layers #noqa: F401
from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401
from ...fluid.contrib import sparsity #noqa: F401
def set_excluded_layers(main_program, param_names):
sparsity.set_excluded_layers(
param_names=param_names, main_program=main_program)
__all__ = [ #noqa
'calculate_density',
......
......@@ -281,6 +281,7 @@ packages=['paddle',
'paddle.incubate.tensor',
'paddle.incubate.multiprocessing',
'paddle.incubate.nn',
'paddle.incubate.asp',
'paddle.incubate.passes',
'paddle.distribution',
'paddle.distributed.sharding',
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册