Skip to content

P
Paddle

BaiXuePrincess / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
未验证 提交 4b803a4a 编写于 作者: W wanghuancoder 提交者： GitHub
浏览文件
操作

delete FLAGS_enable_eager_mode (#49036) 

* delete FLAGS_enable_eager_mode
上级 2afa7b93
隐藏空白更改
内联 并排
Showing with 5 addition and 1739 deletion
python/paddle/autograd/py_layer.py
浏览文件 @ 4b803a4a
...@@ -462,8 +462,6 @@ class EagerPyLayerContext: ...@@ -462,8 +462,6 @@ class EagerPyLayerContext:
Examples: Examples:
.. code-block:: python .. code-block:: python
import os
os.environ['FLAGS_enable_eager_mode'] = '1'
import paddle import paddle
from paddle.autograd import PyLayer from paddle.autograd import PyLayer
import numpy as np import numpy as np
...@@ -503,8 +501,6 @@ class EagerPyLayerContext: ...@@ -503,8 +501,6 @@ class EagerPyLayerContext:
Examples: Examples:
.. code-block:: python .. code-block:: python
import os
os.environ['FLAGS_enable_eager_mode'] = '1'
import paddle import paddle
from paddle.autograd import PyLayer from paddle.autograd import PyLayer
import numpy as np import numpy as np
......
python/paddle/fluid/framework.py
浏览文件 @ 4b803a4a
...@@ -71,7 +71,7 @@ ZERO_VAR_SUFFIX = core.kZeroVarSuffix() ...@@ -71,7 +71,7 @@ ZERO_VAR_SUFFIX = core.kZeroVarSuffix()
CONTROL_DEP_VAR_PREFIX = core.kControlDepVarName() CONTROL_DEP_VAR_PREFIX = core.kControlDepVarName()
_dygraph_tracer_ = None _dygraph_tracer_ = None
_in_eager_mode_ = os.environ.get('FLAGS_enable_eager_mode', '1') == '1' _in_eager_mode_ = True
_global_expected_place_ = None _global_expected_place_ = None
_current_device = None _current_device = None
global_prog_seed = 0 global_prog_seed = 0
......
python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py
浏览文件 @ 4b803a4a
...@@ -57,9 +57,6 @@ def start_local_trainers( ...@@ -57,9 +57,6 @@ def start_local_trainers(
"PADDLE_DISTRI_CUSTOM_DEVICE_TYPE": "custom_cpu", "PADDLE_DISTRI_CUSTOM_DEVICE_TYPE": "custom_cpu",
} }
if not eager_mode:
proc_env["FLAGS_enable_eager_mode"] = "%d" % 0
current_env.update(proc_env) current_env.update(proc_env)
print("trainer proc env:{}".format(current_env)) print("trainer proc env:{}".format(current_env))
......
python/paddle/fluid/tests/unittests/CMakeLists.txt
浏览文件 @ 4b803a4a
...@@ -517,7 +517,7 @@ endforeach() ...@@ -517,7 +517,7 @@ endforeach()
set(TEST_EAGER_OPS test_jit_save_load test_translated_layer) set(TEST_EAGER_OPS test_jit_save_load test_translated_layer)
foreach(TEST_OP ${TEST_EAGER_OPS}) foreach(TEST_OP ${TEST_EAGER_OPS})
list(REMOVE_ITEM TEST_OPS ${TEST_OP}) list(REMOVE_ITEM TEST_OPS ${TEST_OP})
py_test_modules(${TEST_OP} MODULES ${TEST_OP} ENVS FLAGS_enable_eager_mode=1) py_test_modules(${TEST_OP} MODULES ${TEST_OP})
endforeach() endforeach()
if((NOT WITH_GPU) if((NOT WITH_GPU)
...@@ -1256,10 +1256,6 @@ py_test_modules( ...@@ -1256,10 +1256,6 @@ py_test_modules(
set_tests_properties(test_add_reader_dependency_for_interpretercore set_tests_properties(test_add_reader_dependency_for_interpretercore
PROPERTIES TIMEOUT 120) PROPERTIES TIMEOUT 120)
py_test_modules(test_renorm_op_without_eager MODULES test_renorm_op ENVS
FLAGS_enable_eager_mode=0)
set_tests_properties(test_renorm_op_without_eager PROPERTIES TIMEOUT 120)
py_test_modules( py_test_modules(
test_eager_deletion_padding_rnn_for_interpretercore MODULES test_eager_deletion_padding_rnn_for_interpretercore MODULES
test_eager_deletion_padding_rnn ENVS FLAGS_CONVERT_GRAPH_TO_PROGRAM=true) test_eager_deletion_padding_rnn ENVS FLAGS_CONVERT_GRAPH_TO_PROGRAM=true)
......
python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
浏览文件 @ 4b803a4a
...@@ -51,20 +51,6 @@ if(WITH_NCCL) ...@@ -51,20 +51,6 @@ if(WITH_NCCL)
"120") "120")
endif() endif()
endif() endif()
if(WITH_NCCL)
if((WITH_GPU) AND LOCAL_ALL_PLAT)
bash_test_modules(
test_dygraph_sharding_stage3
START_BASH
../../dist_test.sh
LABELS
"RUN_TYPE=DIST"
ENVS
"PADDLE_DIST_UT_PORT=21202;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
)
set_tests_properties(test_dygraph_sharding_stage3 PROPERTIES TIMEOUT "350")
endif()
endif()
if(WITH_NCCL) if(WITH_NCCL)
if(${NCCL_VERSION} VERSION_GREATER_EQUAL 2212) if(${NCCL_VERSION} VERSION_GREATER_EQUAL 2212)
if((WITH_GPU) AND LOCAL_ALL_PLAT) if((WITH_GPU) AND LOCAL_ALL_PLAT)
...@@ -401,19 +387,6 @@ if(LOCAL_ALL_ARCH AND (LINUX OR WIN32)) ...@@ -401,19 +387,6 @@ if(LOCAL_ALL_ARCH AND (LINUX OR WIN32))
test_fleet_amp_init MODULES test_fleet_amp_init ENVS test_fleet_amp_init MODULES test_fleet_amp_init ENVS
"http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python")
endif() endif()
if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
bash_test_modules(
test_dygraph_sharding_optimizer_stage2
START_BASH
../../dist_test.sh
LABELS
"RUN_TYPE=DIST"
ENVS
"PADDLE_DIST_UT_PORT=21238;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
)
set_tests_properties(test_dygraph_sharding_optimizer_stage2 PROPERTIES TIMEOUT
"120")
endif()
if(LOCAL_ALL_ARCH AND (LINUX OR WIN32)) if(LOCAL_ALL_ARCH AND (LINUX OR WIN32))
py_test_modules( py_test_modules(
test_fleet_meta_optimizer_base MODULES test_fleet_meta_optimizer_base ENVS test_fleet_meta_optimizer_base MODULES test_fleet_meta_optimizer_base ENVS
...@@ -667,18 +640,6 @@ if((WITH_GPU ...@@ -667,18 +640,6 @@ if((WITH_GPU
test_fleet_recompute_meta_optimizer ENVS test_fleet_recompute_meta_optimizer ENVS
"http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python")
endif() endif()
if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
bash_test_modules(
test_dygraph_group_sharded_api
START_BASH
../../dist_test.sh
LABELS
"RUN_TYPE=DIST"
ENVS
"PADDLE_DIST_UT_PORT=21266;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
)
set_tests_properties(test_dygraph_group_sharded_api PROPERTIES TIMEOUT "120")
endif()
if(LOCAL_ALL_ARCH AND (LINUX OR WIN32)) if(LOCAL_ALL_ARCH AND (LINUX OR WIN32))
py_test_modules( py_test_modules(
test_fleet_private_function MODULES test_fleet_private_function ENVS test_fleet_private_function MODULES test_fleet_private_function ENVS
...@@ -760,14 +721,6 @@ if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT) ...@@ -760,14 +721,6 @@ if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT)
set_tests_properties(test_parallel_dygraph_sync_batch_norm set_tests_properties(test_parallel_dygraph_sync_batch_norm
PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=DIST") PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=DIST")
endif() endif()
if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT)
py_test_modules(
test_imperative_auto_mixed_precision MODULES
test_imperative_auto_mixed_precision ENVS
"http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python")
set_tests_properties(test_imperative_auto_mixed_precision
PROPERTIES TIMEOUT "300" LABELS "RUN_TYPE=DIST")
endif()
if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT) if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT)
py_test_modules( py_test_modules(
test_imperative_auto_mixed_precision_for_eager MODULES test_imperative_auto_mixed_precision_for_eager MODULES
...@@ -781,11 +734,6 @@ if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT) ...@@ -781,11 +734,6 @@ if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT)
test_mixed_precision MODULES test_mixed_precision ENVS test_mixed_precision MODULES test_mixed_precision ENVS
"http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python")
endif() endif()
if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT)
py_test_modules(
test_dygraph_recompute MODULES test_dygraph_recompute ENVS
"http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python")
endif()
if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT) if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT)
py_test_modules( py_test_modules(
test_dygraph_recompute_for_eager MODULES test_dygraph_recompute_for_eager test_dygraph_recompute_for_eager MODULES test_dygraph_recompute_for_eager
......
python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_dist_save_load.py
浏览文件 @ 4b803a4a
...@@ -33,6 +33,5 @@ class TestDistSaveLoad(unittest.TestCase): ...@@ -33,6 +33,5 @@ class TestDistSaveLoad(unittest.TestCase):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_group_sharded_api.py 已删除 100644 → 0
浏览文件 @ 2afa7b93
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
os.environ['FLAGS_enable_eager_mode'] = '0'
import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus
class TestDygraphGroupSharded(TestMultipleGpus):
# check group sharded logic as well as the accuracy with single mode
def test_dygraph_group_sharded(self):
self.run_mnist_2gpu('dygraph_group_sharded_api.py', eager_mode=False)
if __name__ == "__main__":
unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_group_sharded_api_for_eager.py
浏览文件 @ 4b803a4a
...@@ -12,10 +12,6 @@ ...@@ -12,10 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
os.environ['FLAGS_enable_eager_mode'] = '1'
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
......
python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py
浏览文件 @ 4b803a4a
...@@ -12,10 +12,6 @@ ...@@ -12,10 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
os.environ['FLAGS_enable_eager_mode'] = '1'
import random import random
import unittest import unittest
......
python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_save_for_auto_infer.py
浏览文件 @ 4b803a4a
...@@ -44,6 +44,5 @@ class TestSingleCard(unittest.TestCase): ...@@ -44,6 +44,5 @@ class TestSingleCard(unittest.TestCase):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py 已删除 100644 → 0
浏览文件 @ 2afa7b93
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus
class TestDygraphShardingOptimizerStage2(TestMultipleGpus):
# check sharding logic as well as the accuracy with single mode
def test_dygraph_sharding_optimizer_stage2(self):
self.run_mnist_2gpu(
'dygraph_sharding_optimizer_stage2.py', eager_mode=False
)
if __name__ == "__main__":
unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -23,18 +22,13 @@ class TestDygraphShardingStage2(TestMultipleGpus): ...@@ -23,18 +22,13 @@ class TestDygraphShardingStage2(TestMultipleGpus):
# check sharding logic as well as the accuracy with single mode # check sharding logic as well as the accuracy with single mode
def test_dygraph_sharding_stage2(self): def test_dygraph_sharding_stage2(self):
self.run_mnist_2gpu('dygraph_group_sharded_stage2.py') self.run_mnist_2gpu('dygraph_group_sharded_stage2.py')
self.run_mnist_2gpu('dygraph_sharding_stage2.py', eager_mode=False)
def test_dygraph_sharding_stage2_offload(self): def test_dygraph_sharding_stage2_offload(self):
self.run_mnist_2gpu('dygraph_group_sharded_stage2_offload.py') self.run_mnist_2gpu('dygraph_group_sharded_stage2_offload.py')
self.run_mnist_2gpu(
'dygraph_sharding_stage2_offload.py', eager_mode=False
)
def test_dygraph_sharding_stage2_with_comm_overlap(self): def test_dygraph_sharding_stage2_with_comm_overlap(self):
self.run_mnist_2gpu('dygraph_group_sharded_stage2_comm_overlap.py') self.run_mnist_2gpu('dygraph_group_sharded_stage2_comm_overlap.py')
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py 已删除 100644 → 0
浏览文件 @ 2afa7b93
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
os.environ['FLAGS_enable_eager_mode'] = '0'
import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus
class TestDygraphShardingStage3(TestMultipleGpus):
# check sharding logic as well as the accuracy with single mode
def test_dygraph_sharding_stage3(self):
self.run_mnist_2gpu('dygraph_sharding_stage3.py', eager_mode=False)
def test_dygraph_sharding_stage3_offload(self):
self.run_mnist_2gpu(
'dygraph_sharding_stage3_offload.py', eager_mode=False
)
if __name__ == "__main__":
unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3_for_eager.py
浏览文件 @ 4b803a4a
...@@ -12,11 +12,6 @@ ...@@ -12,11 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
os.environ['FLAGS_enable_eager_mode'] = '1'
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -33,5 +28,4 @@ class TestDygraphShardingStage3(TestMultipleGpus): ...@@ -33,5 +28,4 @@ class TestDygraphShardingStage3(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py 已删除 100644 → 0
浏览文件 @ 2afa7b93
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
os.environ['FLAGS_enable_eager_mode'] = '0'
import tempfile
import unittest
import numpy as np
from test_imperative_resnet import ResNet, optimizer_setting, train_parameters
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
import paddle.nn as nn
from paddle.autograd import PyLayer
from paddle.static import InputSpec
if fluid.core.is_compiled_with_cuda():
fluid.set_flags({"FLAGS_cudnn_deterministic": True})
class SimpleConv(fluid.dygraph.Layer):
def __init__(
self,
num_channels,
num_filters,
filter_size,
stride=1,
groups=1,
act=None,
):
super().__init__()
self._conv = paddle.nn.Conv2D(
in_channels=num_channels,
out_channels=num_filters,
kernel_size=filter_size,
stride=stride,
padding=(filter_size - 1) // 2,
groups=groups,
bias_attr=None,
)
def forward(self, inputs):
return self._conv(inputs)
class TestAutoCast(unittest.TestCase):
def amp_guard_white_op(self):
data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32')
with fluid.dygraph.guard():
conv2d = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
data = fluid.dygraph.to_variable(data)
with fluid.dygraph.amp_guard(True):
out_fp16 = conv2d(data)
with fluid.dygraph.amp_guard(False):
out_fp32 = conv2d(data)
self.assertTrue(data.dtype == fluid.core.VarDesc.VarType.FP32)
self.assertTrue(out_fp16.dtype == fluid.core.VarDesc.VarType.FP16)
self.assertTrue(out_fp32.dtype == fluid.core.VarDesc.VarType.FP32)
def test_amp_guard_white_op(self):
self.amp_guard_white_op()
def amp_guard_black_op(self):
data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32')
with fluid.dygraph.guard():
data = fluid.dygraph.to_variable(data)
with fluid.dygraph.amp_guard(True):
out_fp32 = paddle.mean(data)
self.assertTrue(data.dtype == fluid.core.VarDesc.VarType.FP32)
self.assertTrue(out_fp32.dtype == fluid.core.VarDesc.VarType.FP32)
def test_amp_guard_black_op(self):
self.amp_guard_black_op()
def custom_op_list(self):
with fluid.dygraph.guard():
tracer = fluid.framework._dygraph_tracer()
base_white_list = fluid.dygraph.amp.auto_cast.WHITE_LIST
base_black_list = fluid.dygraph.amp.auto_cast.BLACK_LIST
with fluid.dygraph.amp_guard(
custom_white_list=["log"], custom_black_list=["conv2d"]
):
white_list, black_list = tracer._get_amp_op_list()
self.assertTrue(
set(white_list)
== (set(base_white_list) | {"log"}) - {"conv2d"}
)
self.assertTrue(
set(black_list)
== (set(base_black_list) - {"log"}) | {"conv2d"}
)
base_white_list = fluid.dygraph.amp.auto_cast.PURE_FP16_WHITE_LIST
base_black_list = fluid.dygraph.amp.auto_cast.PURE_FP16_BLACK_LIST
with fluid.dygraph.amp_guard(
custom_white_list=["log"],
custom_black_list=["conv2d"],
level='O2',
):
white_list, black_list = tracer._get_amp_op_list()
self.assertTrue(
set(white_list)
== (set(base_white_list) | {"log"}) - {"conv2d"}
)
self.assertTrue(
set(black_list)
== (set(base_black_list) - {"log"}) | {"conv2d"}
)
def test_custom_op_list(self):
self.custom_op_list()
def custom_op_list_exception(self):
inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32)
def func():
with fluid.dygraph.guard():
model = SimpleConv(
num_channels=3,
num_filters=64,
filter_size=7,
stride=2,
act='relu',
)
with fluid.dygraph.amp_guard(
custom_white_list=["conv2d"], custom_black_list=["conv2d"]
):
inp = fluid.dygraph.to_variable(inp_np)
out = model(inp)
self.assertRaises(ValueError, func)
def test_custom_op_list_exception(self):
self.custom_op_list_exception()
def amp_guard_upsupported_fp16_op(self):
data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32')
with fluid.dygraph.guard():
conv2d = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
data = fluid.dygraph.to_variable(data)
with fluid.dygraph.amp_guard(True):
out_amp_fp16 = conv2d(data)
out_amp_fp32 = paddle.expand_as(
out_amp_fp16, out_amp_fp16
) # expand_as_v2 has no fp16 kernel
with fluid.dygraph.amp_guard(True, level='O2'):
out_purefp16_fp16 = conv2d(data)
out_purefp16_fp32 = paddle.expand_as(
out_purefp16_fp16, out_purefp16_fp16
) # expand_as_v2 has no fp16 kernel
self.assertTrue(data.dtype == fluid.core.VarDesc.VarType.FP32)
self.assertTrue(out_amp_fp16.dtype == fluid.core.VarDesc.VarType.FP16)
self.assertTrue(out_amp_fp32.dtype == fluid.core.VarDesc.VarType.FP32)
self.assertTrue(
out_purefp16_fp16.dtype == fluid.core.VarDesc.VarType.FP16
)
self.assertTrue(
out_purefp16_fp32.dtype == fluid.core.VarDesc.VarType.FP32
)
def test_amp_guard_upsupported_fp16_op(self):
self.amp_guard_upsupported_fp16_op()
def mode_exception(self):
def func():
data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32')
with fluid.dygraph.guard():
conv2d = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
data = fluid.dygraph.to_variable(data)
with fluid.dygraph.amp_guard(level='O'):
out = conv2d(data)
self.assertRaises(ValueError, func)
def test_mode_exception(self):
self.mode_exception()
class TestAmpScaler(unittest.TestCase):
def scale(self):
with fluid.dygraph.guard():
data = paddle.rand([10, 1024])
scaler = paddle.fluid.dygraph.AmpScaler(init_loss_scaling=1024)
scaled_data = scaler.scale(data)
self.assertEqual(
np.array_equal(scaled_data.numpy(), data.numpy() * 1024), True
)
def test_scale(self):
self.scale()
def minimize(self):
inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32)
def run_simple_conv(inp_np, use_scaler=True):
paddle.seed(10)
paddle.framework.random._manual_program_seed(10)
with fluid.dygraph.guard():
model = SimpleConv(
num_channels=3,
num_filters=64,
filter_size=7,
stride=2,
act='relu',
)
optimizer = fluid.optimizer.SGDOptimizer(
learning_rate=0.01, parameter_list=model.parameters()
)
scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024)
data = fluid.dygraph.to_variable(inp_np)
out = model(data)
loss = paddle.mean(out)
if use_scaler:
print('use scaler')
scaled_loss = scaler.scale(loss)
scaled_loss.backward()
optimize_ops, params_grads = scaler.minimize(
optimizer, scaled_loss
)
else:
print('use no scaler')
loss.backward()
optimize_ops, params_grads = optimizer.minimize(loss)
return optimize_ops, params_grads
outs_with_scaler = run_simple_conv(inp_np, use_scaler=True)
outs_no_scaler = run_simple_conv(inp_np, use_scaler=False)
self.assertEqual(
outs_with_scaler[0], []
) # optimize_ops is [] in dygraph mode
self.assertEqual(
outs_no_scaler[0], []
) # optimize_ops is [] in dygraph mode
for i in range(len(outs_with_scaler[1])):
# check each grad
np.testing.assert_allclose(
outs_with_scaler[1][i][1].numpy(),
outs_no_scaler[1][i][1].numpy(),
rtol=1e-05,
)
# check each parameter
np.testing.assert_allclose(
outs_with_scaler[1][i][0].numpy(),
outs_no_scaler[1][i][0].numpy(),
rtol=1e-05,
)
def test_minimize(self):
self.minimize()
def step(self):
inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32)
def run_simple_conv(inp_np, use_scaler=True):
paddle.seed(10)
paddle.framework.random._manual_program_seed(10)
with fluid.dygraph.guard():
model = SimpleConv(
num_channels=3,
num_filters=64,
filter_size=7,
stride=2,
act='relu',
)
optimizer = paddle.optimizer.SGD(
learning_rate=0.01, parameters=model.parameters()
)
scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
data = fluid.dygraph.to_variable(inp_np)
out = model(data)
loss = paddle.mean(out)
if use_scaler:
print('use scaler')
scaled_loss = scaler.scale(loss)
scaled_loss.backward()
scaler.step(optimizer)
scaler.update()
else:
print('use no scaler')
loss.backward()
optimizer.step()
return optimizer._parameter_list
outs_with_scaler = run_simple_conv(inp_np, use_scaler=True)
outs_no_scaler = run_simple_conv(inp_np, use_scaler=False)
for i in range(len(outs_with_scaler)):
# check each parameter
np.testing.assert_allclose(
outs_with_scaler[i].numpy(),
outs_no_scaler[i].numpy(),
rtol=1e-05,
)
def test_step(self):
self.step()
def nan_inf(self):
inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32)
inp_np[0][1][2][3] = np.nan
with fluid.dygraph.guard():
model = SimpleConv(
num_channels=3,
num_filters=64,
filter_size=7,
stride=2,
act='relu',
)
params_init = {}
for param in model.parameters():
params_init[param.name] = param.numpy()
optimizer = fluid.optimizer.SGDOptimizer(
learning_rate=0.01, parameter_list=model.parameters()
)
scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024)
data = fluid.dygraph.to_variable(inp_np)
out = model(data)
loss = paddle.mean(out)
scaled_loss = scaler.scale(loss)
scaled_loss.backward()
optimize_ops, params_grads = scaler.minimize(optimizer, scaled_loss)
self.assertEqual(scaler._found_inf.numpy() >= 1, True)
for param in model.parameters():
# param not update when tensor contains nan or inf
np.testing.assert_array_equal(
param.numpy(), params_init[param.name]
)
def test_nan_inf(self):
self.nan_inf()
def step_update_exception(self):
def func1():
model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True)
optimizer = paddle.optimizer.SGD(
learning_rate=0.01, parameters=model.parameters()
)
scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
data = paddle.rand([10, 3, 32, 32])
conv = model(data)
loss = paddle.mean(conv)
scaled = scaler.scale(loss)
scaled.backward()
scaler.unscale_(optimizer)
scaler.unscale_(optimizer)
self.assertRaises(RuntimeError, func1)
def func2():
model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True)
optimizer = paddle.optimizer.SGD(
learning_rate=0.01, parameters=model.parameters()
)
scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
data = paddle.rand([10, 3, 32, 32])
conv = model(data)
loss = paddle.mean(conv)
scaled = scaler.scale(loss)
scaled.backward()
scaler.step(optimizer)
scaler.unscale_(optimizer)
self.assertRaises(RuntimeError, func2)
def func3():
model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True)
optimizer = paddle.optimizer.SGD(
learning_rate=0.01, parameters=model.parameters()
)
scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
data = paddle.rand([10, 3, 32, 32])
conv = model(data)
loss = paddle.mean(conv)
scaled = scaler.scale(loss)
scaled.backward()
scaler.step(optimizer)
scaler.step(optimizer)
self.assertRaises(RuntimeError, func3)
def test_step_update_exception(self):
self.step_update_exception()
def test_get_and_set(self):
with fluid.dygraph.guard():
scaler = paddle.amp.GradScaler(
enable=True,
init_loss_scaling=1024,
incr_ratio=2.0,
decr_ratio=0.5,
incr_every_n_steps=1000,
decr_every_n_nan_or_inf=2,
use_dynamic_loss_scaling=True,
)
self.assertEqual(scaler.is_enable(), True)
self.assertEqual(scaler.get_init_loss_scaling() == 1024, True)
self.assertEqual(scaler.get_incr_ratio() == 2.0, True)
self.assertEqual(scaler.get_decr_ratio() == 0.5, True)
self.assertEqual(scaler.get_incr_every_n_steps() == 1000, True)
self.assertEqual(scaler.get_decr_every_n_nan_or_inf() == 2, True)
self.assertEqual(scaler.is_use_dynamic_loss_scaling(), True)
scaler.set_decr_every_n_nan_or_inf(4)
self.assertEqual(scaler.get_decr_every_n_nan_or_inf() == 4, True)
scaler.set_decr_ratio(0.1)
self.assertEqual(scaler.get_decr_ratio() == 0.1, True)
scaler.set_incr_every_n_steps(200)
self.assertEqual(scaler.get_incr_every_n_steps() == 200, True)
scaler.set_incr_ratio(3.0)
self.assertEqual(scaler.get_incr_ratio() == 3.0, True)
scaler.set_init_loss_scaling(100)
self.assertEqual(scaler.get_init_loss_scaling() == 100, True)
def test_state_dict_and_load_state_dict(self):
with fluid.dygraph.guard():
scaler1 = paddle.amp.GradScaler(
enable=True,
init_loss_scaling=14,
incr_ratio=233.0,
decr_ratio=0.523,
incr_every_n_steps=1090,
decr_every_n_nan_or_inf=20,
use_dynamic_loss_scaling=True,
)
scaler_state = scaler1.state_dict()
scaler2 = paddle.amp.GradScaler(enable=True)
scaler2.load_state_dict(scaler_state)
self.assertEqual(scaler2.get_init_loss_scaling() == 14, True)
self.assertEqual(scaler2.get_incr_ratio() == 233.0, True)
self.assertEqual(scaler2.get_decr_ratio() == 0.523, True)
self.assertEqual(scaler2.get_incr_every_n_steps() == 1090, True)
self.assertEqual(scaler2.get_decr_every_n_nan_or_inf() == 20, True)
scaler3 = paddle.amp.GradScaler(enable=False)
scaler3.load_state_dict(scaler_state)
self.assertFalse(scaler3.is_enable())
def test_state_dict_and_load_state_dict_error(self):
def test_error():
state_empty = {}
scaler = paddle.amp.GradScaler(enable=True)
scaler.load_state_dict(state_empty)
self.assertRaises(RuntimeError, test_error)
def reader_decorator(reader):
def __reader__():
for item in reader():
img = np.array(item[0]).astype('float32').reshape(3, 224, 224)
label = np.array(item[1]).astype('int64').reshape(1)
yield img, label
return __reader__
class TestGradScalerStateDict(unittest.TestCase):
def train_resnet(
self, enable_amp=True, use_data_loader=True, use_save_load=True
):
seed = 90
batch_size = train_parameters["batch_size"]
batch_num = 4
paddle.seed(seed)
paddle.framework.random._manual_program_seed(seed)
resnet = ResNet(use_cudnn=True)
optimizer = optimizer_setting(
train_parameters, parameter_list=resnet.parameters()
)
np.random.seed(seed)
train_reader = paddle.batch(
paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size
)
dy_param_init_value = {}
for param in resnet.parameters():
dy_param_init_value[param.name] = param.numpy()
program = None
scaler = paddle.amp.GradScaler(
enable=enable_amp, init_loss_scaling=2.0**10
)
if use_data_loader:
train_reader = paddle.batch(
reader_decorator(paddle.dataset.flowers.train(use_xmap=False)),
batch_size=batch_size,
drop_last=True,
)
train_loader = fluid.io.DataLoader.from_generator(
capacity=4,
use_double_buffer=True,
iterable=True,
return_list=True,
)
train_loader.set_sample_list_generator(train_reader)
train_reader = train_loader
for batch_id, data in enumerate(train_reader()):
if batch_id >= batch_num:
break
if use_data_loader:
img, label = data
else:
dy_x_data = np.array(
[x[0].reshape(3, 224, 224) for x in data]
).astype('float32')
if (
len(np.array([x[1] for x in data]).astype('int64'))
!= batch_size
):
continue
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(dy_x_data)
label = paddle.to_tensor(y_data)
label.stop_gradient = True
with paddle.amp.auto_cast(enable=enable_amp):
out = resnet(img)
loss = paddle.nn.functional.cross_entropy(input=out, label=label)
avg_loss = paddle.mean(x=loss)
dy_out = avg_loss.numpy()
scaled_loss = scaler.scale(avg_loss)
scaled_loss.backward()
scaler.minimize(optimizer, scaled_loss)
dy_grad_value = {}
for param in resnet.parameters():
if param.trainable:
np_array = np.array(param._grad_ivar().value().get_tensor())
dy_grad_value[
param.name + fluid.core.grad_var_suffix()
] = np_array
resnet.clear_gradients()
dy_param_value = {}
for param in resnet.parameters():
dy_param_value[param.name] = param.numpy()
if use_save_load and batch_id == 2:
paddle.save(scaler.state_dict(), 'ResNet_model.pdparams')
dict_load = paddle.load('ResNet_model.pdparams')
scaler.load_state_dict(dict_load)
if use_data_loader:
train_reader._reset()
return dy_out, dy_param_value, dy_grad_value
def test_with_state_dict(self):
def func_isinstance():
with fluid.dygraph.guard():
out_use_state_dict = self.train_resnet(
enable_amp=True, use_data_loader=True, use_save_load=True
)
out_no_state_dict = self.train_resnet(
enable_amp=True, use_data_loader=True, use_save_load=False
)
print('save_load:', out_use_state_dict[0], out_no_state_dict[0])
np.testing.assert_allclose(
out_use_state_dict[0], out_no_state_dict[0], rtol=1e-05
)
func_isinstance()
class TestAmpDecorator(unittest.TestCase):
def test_mode_exception(self):
def func():
with fluid.dygraph.guard():
model = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
opt = paddle.optimizer.SGD(parameters=model.parameters())
model, opt = paddle.amp.decorate(
models=model, optimizers=opt, level='O'
)
self.assertRaises(ValueError, func)
def test_input_type_exception(self):
def test_error_model():
class MyModel:
def __init__(self):
print("A fake Model")
model = MyModel()
with fluid.dygraph.guard():
paddle.amp.decorate(models=model, optimizers=None, level='O2')
self.assertRaises(TypeError, test_error_model)
def test_error_distributed_model():
model = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
model = paddle.DataParallel(model)
with fluid.dygraph.guard():
model = paddle.amp.decorate(models=model, level='O2')
self.assertRaises(RuntimeError, test_error_distributed_model)
def test_error_optimizer():
class MyOptimizer:
def __init__(self):
print("A fake Optimizer")
model = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
opt = MyOptimizer()
with fluid.dygraph.guard():
paddle.amp.decorate(models=model, optimizers=opt, level='O2')
self.assertRaises(TypeError, test_error_optimizer)
def test_set_master_weight(self):
model1 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
opt1 = paddle.optimizer.Adam(
learning_rate=0.0001,
parameters=model1.parameters(),
multi_precision=True,
)
model2 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
opt2 = paddle.optimizer.Adam(
learning_rate=0.0001,
parameters=model2.parameters(),
multi_precision=False,
)
model1, opt1 = paddle.amp.decorate(
models=model1, optimizers=opt1, level='O2', master_weight=None
)
self.assertEqual(opt1._multi_precision, True)
models, opt2 = paddle.amp.decorate(
models=[model1, model2],
optimizers=opt2,
level='O2',
master_weight=None,
)
self.assertEqual(opt2._multi_precision, True)
model3 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
opt3 = paddle.optimizer.Adam(
learning_rate=0.0001, parameters=model3.parameters()
)
model4 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
opt4 = paddle.optimizer.Adam(
learning_rate=0.0001, parameters=model4.parameters()
)
model3, opts = paddle.amp.decorate(
models=model3,
optimizers=[opt3, opt4],
level='O2',
master_weight=True,
)
self.assertEqual(opts[0]._multi_precision, True)
self.assertEqual(opts[1]._multi_precision, True)
models = [model3, model4]
optimizers = [opt3, opt4]
models, optimizers = paddle.amp.decorate(
models=models,
optimizers=optimizers,
level='O2',
master_weight=False,
)
self.assertEqual(optimizers[0]._multi_precision, False)
self.assertEqual(optimizers[1]._multi_precision, False)
def test_skip_BatchNorm_Layer_norm(self):
model = paddle.nn.LayerNorm(1)
model = paddle.amp.decorate(models=model, level='O2')
for param in model.parameters():
self.assertEqual((param.dtype == paddle.float32), True)
model = paddle.nn.BatchNorm(1)
model = paddle.amp.decorate(models=model, level='O2')
for param in model.parameters():
self.assertEqual((param.dtype == paddle.float32), True)
model = paddle.nn.BatchNorm1D(1)
model = paddle.amp.decorate(models=model, level='O2')
for param in model.parameters():
self.assertEqual((param.dtype == paddle.float32), True)
model = paddle.nn.BatchNorm2D(1)
model = paddle.amp.decorate(models=model, level='O2')
for param in model.parameters():
self.assertEqual((param.dtype == paddle.float32), True)
model = paddle.nn.BatchNorm3D(1)
model = paddle.amp.decorate(models=model, level='O2')
for param in model.parameters():
self.assertEqual((param.dtype == paddle.float32), True)
def test_floating_only(self):
model = paddle.nn.Linear(2, 4)
buffer = paddle.to_tensor(np.array([5]).astype("int32"))
model.register_buffer("buffer_name", buffer, persistable=True)
model = paddle.amp.decorate(models=model, level='O2')
self.assertEqual(
(model._buffers["buffer_name"].dtype == paddle.int32), True
)
class TestStateDictHookForAMP(unittest.TestCase):
def test_state_dict_hook(self):
def func_isinstance():
paddle.seed(100)
model = paddle.nn.Linear(2, 4)
model = paddle.amp.decorate(
models=model, level='O2', save_dtype='float32'
)
param_value_ori = {}
for param in model.parameters():
param_value_ori[param.name] = param.numpy()
state_dict = model.state_dict()
for key, value in state_dict.items():
state_dict[key] = value.cast("float16")
model.set_state_dict(state_dict)
param_value_now = {}
for param in model.parameters():
param_value_now[param.name] = param.numpy()
for key in param_value_ori.keys():
print(np.equal(param_value_ori[key], param_value_now[key]))
func_isinstance()
class TestPureFp16SaveLoad(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_dtype_exception(self):
def func():
paddle.disable_static()
model = paddle.nn.Conv2D(3, 2, 3, bias_attr=False)
opt = paddle.optimizer.SGD(parameters=model.parameters())
paddle.amp.decorate(
models=model, optimizers=opt, level='O2', save_dtype='int'
)
self.assertRaises(ValueError, func)
def train_resnet(
self, enable_amp=True, use_data_loader=True, use_save_load=True
):
seed = 90
batch_size = train_parameters["batch_size"]
batch_num = 4
paddle.seed(seed)
paddle.framework.random._manual_program_seed(seed)
resnet = ResNet(use_cudnn=True)
optimizer = optimizer_setting(
train_parameters, parameter_list=resnet.parameters()
)
np.random.seed(seed)
train_reader = paddle.batch(
paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size
)
dy_param_init_value = {}
for param in resnet.parameters():
dy_param_init_value[param.name] = param.numpy()
program = None
scaler = paddle.amp.GradScaler(
enable=enable_amp, init_loss_scaling=2.0**10
)
if use_data_loader:
train_reader = paddle.batch(
reader_decorator(paddle.dataset.flowers.train(use_xmap=False)),
batch_size=batch_size,
drop_last=True,
)
train_loader = fluid.io.DataLoader.from_generator(
capacity=4,
use_double_buffer=True,
iterable=True,
return_list=True,
)
train_loader.set_sample_list_generator(train_reader)
train_reader = train_loader
if enable_amp:
resnet, optimizer = paddle.amp.decorate(
models=resnet,
optimizers=optimizer,
level='O2',
save_dtype='float32',
)
for batch_id, data in enumerate(train_reader()):
if batch_id >= batch_num:
break
if use_data_loader:
img, label = data
else:
dy_x_data = np.array(
[x[0].reshape(3, 224, 224) for x in data]
).astype('float32')
if (
len(np.array([x[1] for x in data]).astype('int64'))
!= batch_size
):
continue
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(dy_x_data)
label = paddle.to_tensor(y_data)
label.stop_gradient = True
with paddle.amp.auto_cast(enable=enable_amp, level='O2'):
out = resnet(img)
loss = paddle.nn.functional.cross_entropy(input=out, label=label)
loss = paddle.cast(loss, 'float32')
avg_loss = paddle.mean(x=loss)
dy_out = avg_loss.numpy()
scaled_loss = scaler.scale(avg_loss)
scaled_loss.backward()
scaler.minimize(optimizer, scaled_loss)
dy_grad_value = {}
for param in resnet.parameters():
if param.trainable:
np_array = np.array(param._grad_ivar().value().get_tensor())
dy_grad_value[
param.name + fluid.core.grad_var_suffix()
] = np_array
resnet.clear_gradients()
dy_param_value = {}
for param in resnet.parameters():
dy_param_value[param.name] = param.numpy()
if use_save_load and batch_id == 2:
# paddle.save
obj = {
'model': resnet.state_dict(),
'opt': optimizer.state_dict(),
'scaler': scaler.state_dict(),
}
path = os.path.join(self.temp_dir.name, 'model.pdparams')
paddle.save(obj, path)
# paddle.load
obj_load = paddle.load(path)
resnet = ResNet(use_cudnn=True)
optimizer = optimizer_setting(
train_parameters, parameter_list=resnet.parameters()
)
resnet.set_state_dict(obj_load['model'])
optimizer.set_state_dict(obj_load['opt'])
scaler.load_state_dict(obj_load['scaler'])
resnet, optimizer = paddle.amp.decorate(
models=resnet,
optimizers=optimizer,
level='O2',
save_dtype='float32',
)
if use_data_loader:
train_reader._reset()
return dy_out, dy_param_value, dy_grad_value
def test_with_save_load(self):
def func_isinstance():
with fluid.dygraph.guard():
out_use_save_load = self.train_resnet(
enable_amp=True, use_data_loader=True, use_save_load=True
)
out_no_save_load = self.train_resnet(
enable_amp=True, use_data_loader=True, use_save_load=False
)
print('save_load:', out_use_save_load[0], out_no_save_load[0])
np.testing.assert_allclose(
out_use_save_load[0], out_no_save_load[0], rtol=1e-05
)
func_isinstance()
class TestPureFp16InferenceSaveLoad(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def inference_save_load(self):
BATCH_SIZE = 16
BATCH_NUM = 4
EPOCH_NUM = 4
IMAGE_SIZE = 784
CLASS_NUM = 10
# define a random dataset
class RandomDataset(paddle.io.Dataset):
def __init__(self, num_samples):
self.num_samples = num_samples
def __getitem__(self, idx):
image = np.random.random([IMAGE_SIZE]).astype('float32')
label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype(
'int64'
)
return image, label
def __len__(self):
return self.num_samples
class LinearNet(nn.Layer):
def __init__(self):
super().__init__()
self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM)
def forward(self, x):
return self._linear(x)
def train(layer, loader, loss_fn, opt):
for epoch_id in range(EPOCH_NUM):
for batch_id, (image, label) in enumerate(loader()):
with paddle.amp.auto_cast(
enable=True,
custom_white_list=None,
custom_black_list=None,
level='O2',
):
out = layer(image)
loss = loss_fn(out, label)
loss.backward()
opt.step()
opt.clear_grad()
# train
layer = LinearNet()
adam = paddle.optimizer.Adam(
learning_rate=0.001,
parameters=layer.parameters(),
multi_precision=True,
)
loss_fn = nn.CrossEntropyLoss()
layer, adam = paddle.amp.decorate(
models=layer, optimizers=adam, save_dtype='float32'
)
dataset = RandomDataset(BATCH_NUM * BATCH_SIZE)
loader = paddle.io.DataLoader(
dataset,
batch_size=BATCH_SIZE,
shuffle=True,
drop_last=True,
num_workers=2,
)
train(layer, loader, loss_fn, adam)
# save
path = os.path.join(self.temp_dir.name, 'example_model/linear')
paddle.jit.save(
layer, path, input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')]
)
# jit.load
loaded_layer = paddle.jit.load(path)
# inference
loaded_layer.eval()
x = np.random.randn(1, IMAGE_SIZE).astype('float32')
x_tensor = paddle.to_tensor(x)
pred = loaded_layer(x_tensor)
# load_inference_model
paddle.enable_static()
exe = paddle.static.Executor()
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.load_inference_model(path, exe)
tensor_img = x
results = exe.run(
inference_program,
feed={feed_target_names[0]: tensor_img},
fetch_list=fetch_targets,
)
print("pred.numpy()", pred.numpy())
print("result", results[0])
np.testing.assert_array_equal(pred.numpy(), results[0])
paddle.disable_static()
def test_inference_save_load(self):
self.inference_save_load()
class TestResnet2(unittest.TestCase):
"""
Use paddle-2.0 API
"""
def train_resnet(
self,
enable_amp=True,
level='O1',
use_data_loader=False,
use_param_group=False,
):
seed = 90
batch_size = train_parameters["batch_size"]
batch_num = 10
paddle.seed(seed)
paddle.framework.random._manual_program_seed(seed)
resnet = ResNet(use_cudnn=True)
if use_param_group:
conv_params = resnet.conv.parameters()
other_params = []
for p in resnet.parameters():
contains = False
for q in conv_params:
if p is q:
contains = True
if not contains:
other_params.append(p)
# NOTE(zhiqiu): The Membership test operations(in / not in) calls "is" and "equal",
# see details: https://docs.python.org/3/reference/expressions.html#membership-test-operations.
# So do not use other_params = [p for p in resnet.parameters() if p not in conv_params]
optimizer = paddle.optimizer.Momentum(
parameters=[
{'params': conv_params, 'learning_rate': 0.01},
{'params': other_params, 'learning_rate': 0.001},
],
multi_precision=True,
)
else:
optimizer = paddle.optimizer.SGD(parameters=resnet.parameters())
np.random.seed(seed)
train_reader = paddle.batch(
paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size
)
dy_param_init_value = {}
for param in resnet.parameters():
dy_param_init_value[param.name] = param.numpy()
program = None
scaler = paddle.amp.GradScaler(
enable=enable_amp, init_loss_scaling=2.0**10
)
if use_data_loader:
train_reader = paddle.batch(
reader_decorator(paddle.dataset.flowers.train(use_xmap=False)),
batch_size=batch_size,
drop_last=True,
)
train_loader = fluid.io.DataLoader.from_generator(
capacity=4,
use_double_buffer=True,
iterable=True,
return_list=True,
)
train_loader.set_sample_list_generator(train_reader)
train_reader = train_loader
if enable_amp and (level == 'O2'):
resnet = paddle.amp.decorate(models=resnet, level='O2')
for batch_id, data in enumerate(train_reader()):
if batch_id >= batch_num:
break
if use_data_loader:
img, label = data
else:
dy_x_data = np.array(
[x[0].reshape(3, 224, 224) for x in data]
).astype('float32')
if (
len(np.array([x[1] for x in data]).astype('int64'))
!= batch_size
):
continue
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(dy_x_data)
label = paddle.to_tensor(y_data)
label.stop_gradient = True
with paddle.amp.auto_cast(enable=enable_amp, level=level):
out = resnet(img)
loss = paddle.nn.functional.cross_entropy(input=out, label=label)
loss = paddle.cast(loss, 'float32')
avg_loss = paddle.mean(x=loss)
dy_out = avg_loss.numpy()
scaled_loss = scaler.scale(avg_loss)
scaled_loss.backward()
scaler.unscale_(optimizer)
scaler.step(optimizer)
scaler.update()
dy_grad_value = {}
for param in resnet.parameters():
if param.trainable:
np_array = np.array(param._grad_ivar().value().get_tensor())
dy_grad_value[
param.name + fluid.core.grad_var_suffix()
] = np_array
resnet.clear_gradients()
dy_param_value = {}
for param in resnet.parameters():
dy_param_value[param.name] = param.numpy()
if use_data_loader:
train_reader._reset()
return dy_out, dy_param_value, dy_grad_value
def test_resnet(self):
def func_isinstance():
with fluid.dygraph.guard():
out_fp32 = self.train_resnet(enable_amp=False)
out_amp = self.train_resnet(enable_amp=True)
out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2')
print(out_fp32[0], out_amp[0], out_pure_fp16[0])
np.testing.assert_allclose(
out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05
)
np.testing.assert_allclose(
out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01
)
func_isinstance()
def test_with_data_loader(self):
def func_isinstance():
with fluid.dygraph.guard():
out_fp32 = self.train_resnet(
enable_amp=False, use_data_loader=True
)
out_amp = self.train_resnet(
enable_amp=True, use_data_loader=True
)
out_pure_fp16 = self.train_resnet(
enable_amp=True, use_data_loader=True, level='O2'
)
print(out_fp32[0], out_amp[0], out_pure_fp16[0])
np.testing.assert_allclose(
out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05
)
np.testing.assert_allclose(
out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01
)
func_isinstance()
def test_param_group(self):
def func_isinstance():
with fluid.dygraph.guard():
out_fp32 = self.train_resnet(
enable_amp=False, use_data_loader=True, use_param_group=True
)
out_amp = self.train_resnet(
enable_amp=True, use_data_loader=True, use_param_group=True
)
out_pure_fp16 = self.train_resnet(
enable_amp=True,
use_data_loader=True,
use_param_group=True,
level='O2',
)
print(out_fp32[0], out_amp[0], out_pure_fp16[0])
np.testing.assert_allclose(
out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05
)
np.testing.assert_allclose(
out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01
)
func_isinstance()
class TestResnet(unittest.TestCase):
"""
Use paddle-1.x API
"""
def train_resnet(self, enable_amp=True, level='O1'):
seed = 90
batch_size = train_parameters["batch_size"]
batch_num = 1
with fluid.dygraph.guard():
paddle.seed(seed)
paddle.framework.random._manual_program_seed(seed)
resnet = ResNet(use_cudnn=True)
optimizer = optimizer_setting(
train_parameters, parameter_list=resnet.parameters()
)
optimizer = paddle.optimizer.Momentum(
parameters=resnet.parameters(), multi_precision=True
)
np.random.seed(seed)
train_reader = paddle.batch(
paddle.dataset.flowers.train(use_xmap=False),
batch_size=batch_size,
)
dy_param_init_value = {}
for param in resnet.parameters():
dy_param_init_value[param.name] = param.numpy()
program = None
scaler = paddle.fluid.dygraph.AmpScaler(
enable=enable_amp, init_loss_scaling=2.0**10
)
if enable_amp and (level == 'O2'):
resnet, optimizer = paddle.fluid.dygraph.amp_decorate(
models=resnet, optimizers=optimizer, level='O2'
)
for batch_id, data in enumerate(train_reader()):
if batch_id >= batch_num:
break
dy_x_data = np.array(
[x[0].reshape(3, 224, 224) for x in data]
).astype('float32')
if (
len(np.array([x[1] for x in data]).astype('int64'))
!= batch_size
):
continue
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = fluid.dygraph.to_variable(dy_x_data)
label = fluid.dygraph.to_variable(y_data)
label.stop_gradient = True
with paddle.fluid.dygraph.amp_guard(
enable=enable_amp, level=level
):
out = resnet(img)
loss = paddle.nn.functional.cross_entropy(
input=out, label=label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(x=loss)
dy_out = avg_loss.numpy()
scaled_loss = scaler.scale(avg_loss)
scaled_loss.backward()
scaler.minimize(optimizer, scaled_loss)
dy_grad_value = {}
for param in resnet.parameters():
if param.trainable:
np_array = np.array(
param._grad_ivar().value().get_tensor()
)
dy_grad_value[
param.name + fluid.core.grad_var_suffix()
] = np_array
resnet.clear_gradients()
dy_param_value = {}
for param in resnet.parameters():
dy_param_value[param.name] = param.numpy()
return dy_out, dy_param_value, dy_grad_value
def test_resnet(self):
def func_isinstance():
out_fp32 = self.train_resnet(enable_amp=False)
out_amp = self.train_resnet(enable_amp=True)
out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2')
print(out_fp32[0], out_amp[0], out_pure_fp16[0])
np.testing.assert_allclose(
out_fp32[0], out_amp[0], rtol=1e-05, atol=0.01
)
np.testing.assert_allclose(
out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.1
)
func_isinstance()
class TestLayerNormFp16(unittest.TestCase):
r'''layer_norm and batch_norm support mixed inputs, i.e., only input x is fp16
and other params are fp32.
'''
def test_layer_norm_fp16(self):
def func_isinstance():
if fluid.is_compiled_with_cuda():
with fluid.dygraph.guard(fluid.CUDAPlace(0)):
x = paddle.rand([2, 2, 2, 3])
layer_norm = paddle.nn.LayerNorm(x.shape[1:])
with paddle.amp.auto_cast(custom_white_list=['layer_norm']):
out = layer_norm(x)
self.assertTrue(
out.dtype == fluid.core.VarDesc.VarType.FP16
)
func_isinstance()
@unittest.skipIf(
paddle.is_compiled_with_cuda()
and not core.is_bfloat16_supported(core.CUDAPlace(0)),
"skip bf16 test if cuda is in use but bf16 is not supported by gpu arch.",
)
class TestBf16(unittest.TestCase):
'''
test amp for BF16
'''
def train(self, enable_amp=True, amp_level='O1'):
paddle.seed(100)
input = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0)
conv = paddle.nn.Conv2D(4, 6, (3, 3))
if amp_level == 'O2':
conv = paddle.amp.decorate(
models=conv, level=amp_level, dtype='bfloat16'
)
with paddle.amp.auto_cast(
enable=enable_amp, level=amp_level, dtype='bfloat16'
):
output = conv(input)
output = output.cast('float32')
return output.numpy()
def test_bf16(self):
def func_isinstance():
out_fp32 = self.train(enable_amp=False)
out_bf16_O1 = self.train(enable_amp=True, amp_level='O1')
out_bf16_O2 = self.train(enable_amp=True, amp_level='O2')
np.testing.assert_allclose(
out_fp32, out_bf16_O1, rtol=0.001, atol=0.1
)
np.testing.assert_allclose(
out_fp32, out_bf16_O2, rtol=0.001, atol=0.1
)
func_isinstance()
class TestAmpWithPyLyer(unittest.TestCase):
def test_pylayer(self):
class MyMM(PyLayer):
@staticmethod
def forward(ctx, a, b):
ctx.save_for_backward(a, b)
return a.mm(b)
@staticmethod
def backward(ctx, grad):
a, b = ctx.saved_tensor()
# NOTE(zhiqiu): a and b is float32 now, while grad is fp16 when forward runs with auto_cast()
# thus, the mm operation raise errors because of the dtype of inputs are inconsistent before.
return grad.mm(b.t()), a.t().mm(grad)
x = paddle.rand([10, 10])
y = paddle.rand([10, 10])
x.stop_gradient = False
y.stop_gradient = False
# with paddle.amp.auto_cast():
res = MyMM.apply(x, y)
loss = paddle.mean(res)
loss.backward()
class TestAmpWithHook(unittest.TestCase):
def test_hook_change_dtype(self):
def func_isinstance():
with paddle.fluid.dygraph.guard():
v = paddle.rand([3, 3])
v.stop_gradient = False
def foo(grad):
print('grad', grad, grad.dtype) # grad's dtype is float32
res = paddle.mm(grad, grad) # mm runs in fp16
print('res', res, res.dtype) # res's dtype is float16
return res
v.register_hook(foo)
with paddle.amp.auto_cast():
a = paddle.mm(v, v)
loss = a.sum()
self.assertRaises(RuntimeError, loss.backward)
func_isinstance()
def test_hook_change_place(self):
def func_isinstance():
with paddle.fluid.dygraph.guard():
v = paddle.rand([3, 3])
v.stop_gradient = False
def foo(grad):
res = grad.cpu() # change place
return res
v.register_hook(foo)
with paddle.amp.auto_cast():
a = paddle.mm(v, v)
loss = a.sum()
self.assertRaises(RuntimeError, loss.backward)
func_isinstance()
if __name__ == '__main__':
unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py
浏览文件 @ 4b803a4a
...@@ -13,9 +13,6 @@ ...@@ -13,9 +13,6 @@
# limitations under the License. # limitations under the License.
import os import os
os.environ['FLAGS_enable_eager_mode'] = '1'
import tempfile import tempfile
import unittest import unittest
......
python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -25,5 +24,4 @@ class TestParallelClassCenterSample(TestMultipleGpus): ...@@ -25,5 +24,4 @@ class TestParallelClassCenterSample(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -25,5 +24,4 @@ class TestModelParallelLayer(TestMultipleGpus): ...@@ -25,5 +24,4 @@ class TestModelParallelLayer(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -27,5 +26,4 @@ class TestDataParallelLayer(TestMultipleGpus): ...@@ -27,5 +26,4 @@ class TestDataParallelLayer(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py
浏览文件 @ 4b803a4a
...@@ -73,5 +73,4 @@ class TestHybridPipeParallel(TestMultipleGpus): ...@@ -73,5 +73,4 @@ class TestHybridPipeParallel(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -34,5 +33,4 @@ class TestHybridPipeParallelWithVirtualStage(TestMultipleGpus): ...@@ -34,5 +33,4 @@ class TestHybridPipeParallelWithVirtualStage(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py
浏览文件 @ 4b803a4a
...@@ -78,9 +78,6 @@ def start_local_trainers( ...@@ -78,9 +78,6 @@ def start_local_trainers(
"PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()),
} }
if not eager_mode:
proc_env["FLAGS_enable_eager_mode"] = "%d" % 0
current_env.update(proc_env) current_env.update(proc_env)
print("trainer proc env:{}".format(current_env)) print("trainer proc env:{}".format(current_env))
......
python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -29,5 +28,4 @@ class TestHybridParallel(TestMultipleGpus): ...@@ -29,5 +28,4 @@ class TestHybridParallel(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -41,5 +40,4 @@ class TestHybridParallel(TestMultipleGpus): ...@@ -41,5 +40,4 @@ class TestHybridParallel(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -27,5 +26,4 @@ class TestParallelMarginSoftmaxWithCrossEntropy(TestMultipleGpus): ...@@ -27,5 +26,4 @@ class TestParallelMarginSoftmaxWithCrossEntropy(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
浏览文件 @ 4b803a4a
...@@ -4,7 +4,6 @@ test_fleet_static_mp_layers,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_p ...@@ -4,7 +4,6 @@ test_fleet_static_mp_layers,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_p
test_dgc_op,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC test_dgc_op,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC
test_dgc_optimizer,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC test_dgc_optimizer,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC
test_parallel_margin_cross_entropy,,GPU,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL test_parallel_margin_cross_entropy,,GPU,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
test_dygraph_sharding_stage3,,GPU,350,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
test_parallel_dygraph_transformer,,GPU,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL;${NCCL_VERSION} VERSION_GREATER_EQUAL 2212 test_parallel_dygraph_transformer,,GPU,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL;${NCCL_VERSION} VERSION_GREATER_EQUAL 2212
test_parallel_dygraph_transformer,,ROCM,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_parallel_dygraph_transformer,,ROCM,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_fleet_fp16_allreduce_meta_optimizer,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_fp16_allreduce_meta_optimizer,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
...@@ -34,7 +33,6 @@ test_communicator_sync,,,,,test_runner.py,2,,FLAGS_communicator_send_queue_size= ...@@ -34,7 +33,6 @@ test_communicator_sync,,,,,test_runner.py,2,,FLAGS_communicator_send_queue_size=
test_fleet_pipeline_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_pipeline_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_fleet_gradient_merge_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_gradient_merge_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_fleet_amp_init,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_amp_init,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_dygraph_sharding_optimizer_stage2,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_fleet_meta_optimizer_base,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_meta_optimizer_base,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_fleet_raw_program_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_raw_program_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_parallel_dygraph_sharding_parallel,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_parallel_dygraph_sharding_parallel,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
...@@ -57,7 +55,6 @@ test_parallel_dygraph_sparse_embedding_over_height,,ROCM,350,DIST,../../dist_tes ...@@ -57,7 +55,6 @@ test_parallel_dygraph_sparse_embedding_over_height,,ROCM,350,DIST,../../dist_tes
test_distributed_strategy,LINUX;APPLE,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_distributed_strategy,LINUX;APPLE,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_auto_parallel_parallelizer,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_auto_parallel_parallelizer,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_fleet_recompute_meta_optimizer,LINUX;WIN32,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_recompute_meta_optimizer,LINUX;WIN32,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_dygraph_group_sharded_api,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_fleet_private_function,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_private_function,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_new_group,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_new_group.sh,2,,http_proxy=;https_proxy=, test_new_group,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_new_group.sh,2,,http_proxy=;https_proxy=,
test_c_comm_init_op,LINUX,GPU;XPU;ASCEND;ASCEND_CL,120,DIST,test_c_comm_init_op.sh,2,,http_proxy=;https_proxy=, test_c_comm_init_op,LINUX,GPU;XPU;ASCEND;ASCEND_CL,120,DIST,test_c_comm_init_op.sh,2,,http_proxy=;https_proxy=,
...@@ -65,10 +62,8 @@ test_ir_pass_pipeline,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=; ...@@ -65,10 +62,8 @@ test_ir_pass_pipeline,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;
test_parallel_dygraph_mnist,,GPU;ROCM,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_parallel_dygraph_mnist,,GPU;ROCM,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_parallel_dygraph_se_resnext,,GPU;ROCM,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_parallel_dygraph_se_resnext,,GPU;ROCM,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_parallel_dygraph_sync_batch_norm,,GPU;ROCM,120,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_parallel_dygraph_sync_batch_norm,,GPU;ROCM,120,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_imperative_auto_mixed_precision,,GPU;ROCM,300,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_imperative_auto_mixed_precision_for_eager,,GPU;ROCM,300,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_imperative_auto_mixed_precision_for_eager,,GPU;ROCM,300,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_mixed_precision,,GPU;ROCM,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_mixed_precision,,GPU;ROCM,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_dygraph_recompute,,GPU;ROCM,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_dygraph_recompute_for_eager,,GPU;ROCM,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_dygraph_recompute_for_eager,,GPU;ROCM,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_dist_mnist_dgc_nccl,,,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL OR WITH_RCCL;WITH_DGC test_dist_mnist_dgc_nccl,,,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL OR WITH_RCCL;WITH_DGC
test_dist_se_resnext_dgc,,,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL OR WITH_RCCL;WITH_DGC test_dist_se_resnext_dgc,,,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL OR WITH_RCCL;WITH_DGC
......
python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py
浏览文件 @ 4b803a4a
...@@ -77,10 +77,6 @@ class TestDistBase(unittest.TestCase): ...@@ -77,10 +77,6 @@ class TestDistBase(unittest.TestCase):
required_envs["GLOG_logtostderr"] = "1" required_envs["GLOG_logtostderr"] = "1"
required_envs["GLOO_LOG_LEVEL"] = "TRACE" required_envs["GLOO_LOG_LEVEL"] = "TRACE"
if eager_mode:
required_envs["FLAGS_enable_eager_mode"] = "%d" % 1
else:
required_envs["FLAGS_enable_eager_mode"] = "%d" % 0
self._run_cluster(model_file, required_envs) self._run_cluster(model_file, required_envs)
def _run_cluster(self, model_file, envs): def _run_cluster(self, model_file, envs):
......
python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -24,5 +23,4 @@ class TestCollectiveAllToAllSingle(TestMultipleGpus): ...@@ -24,5 +23,4 @@ class TestCollectiveAllToAllSingle(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -24,5 +23,4 @@ class TestCollectiveBatchIsendIrecv(TestMultipleGpus): ...@@ -24,5 +23,4 @@ class TestCollectiveBatchIsendIrecv(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py
浏览文件 @ 4b803a4a
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import unittest import unittest
from test_parallel_dygraph_dataparallel import TestMultipleGpus from test_parallel_dygraph_dataparallel import TestMultipleGpus
...@@ -24,5 +23,4 @@ class TestCollectiveReduceScatter(TestMultipleGpus): ...@@ -24,5 +23,4 @@ class TestCollectiveReduceScatter(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/dygraph_to_static/CMakeLists.txt
浏览文件 @ 4b803a4a
...@@ -4,7 +4,7 @@ file( ...@@ -4,7 +4,7 @@ file(
"test_*.py") "test_*.py")
string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
set(GC_ENVS FLAGS_eager_delete_tensor_gb=0.0) set(GC_ENVS FLAGS_eager_delete_tensor_gb=0.0)
set(DY2ST_EAGER_TEST_ENVS ${GC_ENVS} FLAGS_enable_eager_mode=1) set(DY2ST_EAGER_TEST_ENVS ${GC_ENVS})
set(TEST_EAGER_OPS set(TEST_EAGER_OPS
test_bmn test_bmn
...@@ -35,7 +35,7 @@ list(REMOVE_ITEM TEST_OPS test_lac) ...@@ -35,7 +35,7 @@ list(REMOVE_ITEM TEST_OPS test_lac)
# NOTE(Aurelius84): In case of Windows CI, if open ON_INFER, RWLOCK of Scope # NOTE(Aurelius84): In case of Windows CI, if open ON_INFER, RWLOCK of Scope
# will be removed and will cause some random failed in multi-thread. # will be removed and will cause some random failed in multi-thread.
if(WITH_PYTHON) if(WITH_PYTHON)
py_test_modules(test_lac MODULES test_lac ENVS FLAGS_enable_eager_mode=1) py_test_modules(test_lac MODULES test_lac)
set_tests_properties(test_lac PROPERTIES TIMEOUT 120) set_tests_properties(test_lac PROPERTIES TIMEOUT 120)
endif() endif()
......
python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py
浏览文件 @ 4b803a4a
...@@ -12,10 +12,8 @@ ...@@ -12,10 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
os.environ["FLAGS_enable_eager_mode"] = "0"
import math import math
import os
import tempfile import tempfile
import time import time
import unittest import unittest
......
python/paddle/fluid/tests/unittests/test_collective_api_base.py
浏览文件 @ 4b803a4a
...@@ -327,11 +327,6 @@ class TestDistBase(unittest.TestCase): ...@@ -327,11 +327,6 @@ class TestDistBase(unittest.TestCase):
'NVIDIA_TF32_OVERRIDE', '' 'NVIDIA_TF32_OVERRIDE', ''
) )
if eager_mode:
required_envs["FLAGS_enable_eager_mode"] = "%d" % 1
else:
required_envs["FLAGS_enable_eager_mode"] = "%d" % 0
tr0_out, tr1_out, pid0, pid1 = self._run_cluster( tr0_out, tr1_out, pid0, pid1 = self._run_cluster(
model_file, required_envs model_file, required_envs
) )
......
python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py
浏览文件 @ 4b803a4a
...@@ -124,9 +124,6 @@ def start_local_trainers( ...@@ -124,9 +124,6 @@ def start_local_trainers(
"PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()),
} }
if not eager_mode:
proc_env["FLAGS_enable_eager_mode"] = "%d" % 0
proc_env["FLAGS_allocator_strategy"] = allocator_strategy proc_env["FLAGS_allocator_strategy"] = allocator_strategy
if allocator_strategy == "auto_growth": if allocator_strategy == "auto_growth":
proc_env["FLAGS_fraction_of_gpu_memory_to_use"] = "0.1" proc_env["FLAGS_fraction_of_gpu_memory_to_use"] = "0.1"
...@@ -241,5 +238,4 @@ class TestGradientCheckInEagerMode(TestMultipleGpus): ...@@ -241,5 +238,4 @@ class TestGradientCheckInEagerMode(TestMultipleGpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
python/paddle/fluid/tests/unittests/xpu/test_parallel_dygraph_dataparallel.py
浏览文件 @ 4b803a4a
...@@ -80,9 +80,6 @@ def start_local_trainers( ...@@ -80,9 +80,6 @@ def start_local_trainers(
"PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()),
} }
if not eager_mode:
proc_env["FLAGS_enable_eager_mode"] = "%d" % 0
current_env.update(proc_env) current_env.update(proc_env)
print("trainer proc env:{}".format(current_env)) print("trainer proc env:{}".format(current_env))
...@@ -152,7 +149,6 @@ class TestGradientCheckInEagerMode(TestMultipleXpus): ...@@ -152,7 +149,6 @@ class TestGradientCheckInEagerMode(TestMultipleXpus):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
os.environ["BKCL_PCIE_RING"] = "1" os.environ["BKCL_PCIE_RING"] = "1"
os.environ["BKCL_CCIX_RING"] = "0" os.environ["BKCL_CCIX_RING"] = "0"
unittest.main() unittest.main()
python/paddle/jit/api.py
浏览文件 @ 4b803a4a
...@@ -1558,8 +1558,6 @@ class TracedLayer: ...@@ -1558,8 +1558,6 @@ class TracedLayer:
Examples: Examples:
.. code-block:: python: .. code-block:: python:
import os
os.environ['FLAGS_enable_eager_mode'] = '0'
import paddle import paddle
class ExampleLayer(paddle.nn.Layer): class ExampleLayer(paddle.nn.Layer):
...@@ -1610,8 +1608,6 @@ class TracedLayer: ...@@ -1610,8 +1608,6 @@ class TracedLayer:
Examples: Examples:
.. code-block:: python: .. code-block:: python:
import os
os.environ['FLAGS_enable_eager_mode'] = '0'
import paddle import paddle
class ExampleLayer(paddle.nn.Layer): class ExampleLayer(paddle.nn.Layer):
...@@ -1716,8 +1712,6 @@ class TracedLayer: ...@@ -1716,8 +1712,6 @@ class TracedLayer:
Examples: Examples:
.. code-block:: python: .. code-block:: python:
import os
os.environ['FLAGS_enable_eager_mode'] = '0'
import numpy as np import numpy as np
import paddle import paddle
......
python/paddle/tests/test_dist_hapi_model.py
浏览文件 @ 4b803a4a
...@@ -78,9 +78,6 @@ def start_local_trainers( ...@@ -78,9 +78,6 @@ def start_local_trainers(
"PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()),
} }
if not eager_mode:
proc_env["FLAGS_enable_eager_mode"] = "%d" % 0
current_env.update(proc_env) current_env.update(proc_env)
print("trainer proc env:{}".format(current_env)) print("trainer proc env:{}".format(current_env))
...@@ -148,5 +145,4 @@ class TestMultipleGpus(unittest.TestCase): ...@@ -148,5 +145,4 @@ class TestMultipleGpus(unittest.TestCase):
if __name__ == "__main__": if __name__ == "__main__":
os.environ["FLAGS_enable_eager_mode"] = "1"
unittest.main() unittest.main()
tools/final_ut_parallel_rule.py
浏览文件 @ 4b803a4a
...@@ -47,7 +47,6 @@ def classify_cases_by_mem(rootPath): ...@@ -47,7 +47,6 @@ def classify_cases_by_mem(rootPath):
case_exec_200 = [ case_exec_200 = [
'test_post_training_quantization_mnist', 'test_post_training_quantization_mnist',
'test_imperative_auto_mixed_precision',
'test_trt_dynamic_shape_ernie_fp16_ser_deser', 'test_trt_dynamic_shape_ernie_fp16_ser_deser',
'test_trt_dynamic_shape_ernie', 'test_trt_dynamic_shape_ernie',
'test_layer_norm_op', 'test_layer_norm_op',
......
tools/parallel_UT_rule.py
浏览文件 @ 4b803a4a
...@@ -1033,7 +1033,6 @@ FOURTH_HIGH_PARALLEL_JOB_NEW = [ ...@@ -1033,7 +1033,6 @@ FOURTH_HIGH_PARALLEL_JOB_NEW = [
'test_elementwise_div_grad_grad', 'test_elementwise_div_grad_grad',
'test_minus_op', 'test_minus_op',
'test_shard_index_op', 'test_shard_index_op',
'test_dygraph_recompute',
'test_momentum_op', 'test_momentum_op',
'test_modelaverage', 'test_modelaverage',
'test_compare_reduce_op', 'test_compare_reduce_op',
...@@ -2406,7 +2405,6 @@ TETRAD_PARALLEL_JOB = [ ...@@ -2406,7 +2405,6 @@ TETRAD_PARALLEL_JOB = [
'test_matrix_rank_op', 'test_matrix_rank_op',
'test_margin_cross_entropy_op', 'test_margin_cross_entropy_op',
'test_elementwise_pow_op', 'test_elementwise_pow_op',
'test_dygraph_recompute',
'test_qr_op', 'test_qr_op',
'test_dygraph_spectral_norm', 'test_dygraph_spectral_norm',
'test_cumsum_op', 'test_cumsum_op',
......
tools/windows/run_unittests.sh
浏览文件 @ 4b803a4a
...@@ -239,7 +239,6 @@ long_time_test="^test_gru_op$|\ ...@@ -239,7 +239,6 @@ long_time_test="^test_gru_op$|\
^test_sequence_conv$|\ ^test_sequence_conv$|\
^test_sgd_op$|\ ^test_sgd_op$|\
^test_transformer$|\ ^test_transformer$|\
^test_imperative_auto_mixed_precision$|\
^test_trt_matmul_quant_dequant$|\ ^test_trt_matmul_quant_dequant$|\
^test_strided_slice_op$" ^test_strided_slice_op$"
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或