From 4b803a4a8b2989a64ba5786f12ae7dcd739e5dc3 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Tue, 20 Dec 2022 12:54:20 +0800 Subject: [PATCH] delete FLAGS_enable_eager_mode (#49036) * delete FLAGS_enable_eager_mode --- python/paddle/autograd/py_layer.py | 4 - python/paddle/fluid/framework.py | 2 +- .../test_collective_process_group_xccl.py | 3 - .../fluid/tests/unittests/CMakeLists.txt | 6 +- .../unittests/collective/fleet/CMakeLists.txt | 52 - .../fleet/test_dygraph_dist_save_load.py | 1 - .../fleet/test_dygraph_group_sharded_api.py | 32 - ...est_dygraph_group_sharded_api_for_eager.py | 4 - .../fleet/test_dygraph_recompute_for_eager.py | 4 - .../fleet/test_dygraph_save_for_auto_infer.py | 1 - .../test_dygraph_sharding_optimizer_stage2.py | 30 - .../fleet/test_dygraph_sharding_stage2.py | 6 - .../fleet/test_dygraph_sharding_stage3.py | 37 - .../test_dygraph_sharding_stage3_for_eager.py | 6 - .../test_imperative_auto_mixed_precision.py | 1485 ----------------- ...perative_auto_mixed_precision_for_eager.py | 3 - .../test_parallel_class_center_sample.py | 2 - .../fleet/test_parallel_dygraph_mp_layers.py | 2 - ...parallel_dygraph_no_sync_gradient_check.py | 2 - ...test_parallel_dygraph_pipeline_parallel.py | 1 - ...ph_pipeline_parallel_with_virtual_stage.py | 2 - .../fleet/test_parallel_dygraph_qat.py | 3 - ...test_parallel_dygraph_sharding_parallel.py | 2 - .../test_parallel_dygraph_tensor_parallel.py | 2 - .../test_parallel_margin_cross_entropy.py | 2 - .../unittests/collective/fleet/testslist.csv | 5 - .../multinode/test_collective_multi_nodes.py | 4 - .../test_collective_alltoall_single.py | 2 - .../test_collective_batch_isend_irecv.py | 2 - .../test_collective_reduce_scatter.py | 2 - .../dygraph_to_static/CMakeLists.txt | 4 +- .../dygraph_to_static/test_resnet_v2.py | 4 +- .../unittests/test_collective_api_base.py | 5 - .../test_parallel_dygraph_dataparallel.py | 4 - .../xpu/test_parallel_dygraph_dataparallel.py | 4 - python/paddle/jit/api.py | 6 - python/paddle/tests/test_dist_hapi_model.py | 4 - tools/final_ut_parallel_rule.py | 1 - tools/parallel_UT_rule.py | 2 - tools/windows/run_unittests.sh | 1 - 40 files changed, 5 insertions(+), 1739 deletions(-) delete mode 100644 python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_group_sharded_api.py delete mode 100644 python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py delete mode 100644 python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py delete mode 100644 python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py diff --git a/python/paddle/autograd/py_layer.py b/python/paddle/autograd/py_layer.py index d90e437438c..39892ca5c66 100644 --- a/python/paddle/autograd/py_layer.py +++ b/python/paddle/autograd/py_layer.py @@ -462,8 +462,6 @@ class EagerPyLayerContext: Examples: .. code-block:: python - import os - os.environ['FLAGS_enable_eager_mode'] = '1' import paddle from paddle.autograd import PyLayer import numpy as np @@ -503,8 +501,6 @@ class EagerPyLayerContext: Examples: .. code-block:: python - import os - os.environ['FLAGS_enable_eager_mode'] = '1' import paddle from paddle.autograd import PyLayer import numpy as np diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 383cf6eb61a..9327882a9cb 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -71,7 +71,7 @@ ZERO_VAR_SUFFIX = core.kZeroVarSuffix() CONTROL_DEP_VAR_PREFIX = core.kControlDepVarName() _dygraph_tracer_ = None -_in_eager_mode_ = os.environ.get('FLAGS_enable_eager_mode', '1') == '1' +_in_eager_mode_ = True _global_expected_place_ = None _current_device = None global_prog_seed = 0 diff --git a/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py b/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py index 391336a3695..8a1f90359f0 100644 --- a/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py +++ b/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py @@ -57,9 +57,6 @@ def start_local_trainers( "PADDLE_DISTRI_CUSTOM_DEVICE_TYPE": "custom_cpu", } - if not eager_mode: - proc_env["FLAGS_enable_eager_mode"] = "%d" % 0 - current_env.update(proc_env) print("trainer proc env:{}".format(current_env)) diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index 9967d432064..c31d73aec00 100755 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -517,7 +517,7 @@ endforeach() set(TEST_EAGER_OPS test_jit_save_load test_translated_layer) foreach(TEST_OP ${TEST_EAGER_OPS}) list(REMOVE_ITEM TEST_OPS ${TEST_OP}) - py_test_modules(${TEST_OP} MODULES ${TEST_OP} ENVS FLAGS_enable_eager_mode=1) + py_test_modules(${TEST_OP} MODULES ${TEST_OP}) endforeach() if((NOT WITH_GPU) @@ -1256,10 +1256,6 @@ py_test_modules( set_tests_properties(test_add_reader_dependency_for_interpretercore PROPERTIES TIMEOUT 120) -py_test_modules(test_renorm_op_without_eager MODULES test_renorm_op ENVS - FLAGS_enable_eager_mode=0) - -set_tests_properties(test_renorm_op_without_eager PROPERTIES TIMEOUT 120) py_test_modules( test_eager_deletion_padding_rnn_for_interpretercore MODULES test_eager_deletion_padding_rnn ENVS FLAGS_CONVERT_GRAPH_TO_PROGRAM=true) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt b/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt index a0252356b65..d8fa0fb0285 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt @@ -51,20 +51,6 @@ if(WITH_NCCL) "120") endif() endif() -if(WITH_NCCL) - if((WITH_GPU) AND LOCAL_ALL_PLAT) - bash_test_modules( - test_dygraph_sharding_stage3 - START_BASH - ../../dist_test.sh - LABELS - "RUN_TYPE=DIST" - ENVS - "PADDLE_DIST_UT_PORT=21202;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python" - ) - set_tests_properties(test_dygraph_sharding_stage3 PROPERTIES TIMEOUT "350") - endif() -endif() if(WITH_NCCL) if(${NCCL_VERSION} VERSION_GREATER_EQUAL 2212) if((WITH_GPU) AND LOCAL_ALL_PLAT) @@ -401,19 +387,6 @@ if(LOCAL_ALL_ARCH AND (LINUX OR WIN32)) test_fleet_amp_init MODULES test_fleet_amp_init ENVS "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") endif() -if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT) - bash_test_modules( - test_dygraph_sharding_optimizer_stage2 - START_BASH - ../../dist_test.sh - LABELS - "RUN_TYPE=DIST" - ENVS - "PADDLE_DIST_UT_PORT=21238;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python" - ) - set_tests_properties(test_dygraph_sharding_optimizer_stage2 PROPERTIES TIMEOUT - "120") -endif() if(LOCAL_ALL_ARCH AND (LINUX OR WIN32)) py_test_modules( test_fleet_meta_optimizer_base MODULES test_fleet_meta_optimizer_base ENVS @@ -667,18 +640,6 @@ if((WITH_GPU test_fleet_recompute_meta_optimizer ENVS "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") endif() -if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT) - bash_test_modules( - test_dygraph_group_sharded_api - START_BASH - ../../dist_test.sh - LABELS - "RUN_TYPE=DIST" - ENVS - "PADDLE_DIST_UT_PORT=21266;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python" - ) - set_tests_properties(test_dygraph_group_sharded_api PROPERTIES TIMEOUT "120") -endif() if(LOCAL_ALL_ARCH AND (LINUX OR WIN32)) py_test_modules( test_fleet_private_function MODULES test_fleet_private_function ENVS @@ -760,14 +721,6 @@ if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT) set_tests_properties(test_parallel_dygraph_sync_batch_norm PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=DIST") endif() -if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT) - py_test_modules( - test_imperative_auto_mixed_precision MODULES - test_imperative_auto_mixed_precision ENVS - "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") - set_tests_properties(test_imperative_auto_mixed_precision - PROPERTIES TIMEOUT "300" LABELS "RUN_TYPE=DIST") -endif() if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT) py_test_modules( test_imperative_auto_mixed_precision_for_eager MODULES @@ -781,11 +734,6 @@ if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT) test_mixed_precision MODULES test_mixed_precision ENVS "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") endif() -if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT) - py_test_modules( - test_dygraph_recompute MODULES test_dygraph_recompute ENVS - "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") -endif() if((WITH_GPU OR WITH_ROCM) AND LOCAL_ALL_PLAT) py_test_modules( test_dygraph_recompute_for_eager MODULES test_dygraph_recompute_for_eager diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_dist_save_load.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_dist_save_load.py index d5bec10c7ae..e600a44200b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_dist_save_load.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_dist_save_load.py @@ -33,6 +33,5 @@ class TestDistSaveLoad(unittest.TestCase): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_group_sharded_api.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_group_sharded_api.py deleted file mode 100644 index 1302c322ce7..00000000000 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_group_sharded_api.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -os.environ['FLAGS_enable_eager_mode'] = '0' - -import unittest - -from test_parallel_dygraph_dataparallel import TestMultipleGpus - - -class TestDygraphGroupSharded(TestMultipleGpus): - - # check group sharded logic as well as the accuracy with single mode - def test_dygraph_group_sharded(self): - self.run_mnist_2gpu('dygraph_group_sharded_api.py', eager_mode=False) - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_group_sharded_api_for_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_group_sharded_api_for_eager.py index 331974edfbc..4bc1fbecf37 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_group_sharded_api_for_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_group_sharded_api_for_eager.py @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - -os.environ['FLAGS_enable_eager_mode'] = '1' - import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py index b8ef3ea18c5..087609b62da 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - -os.environ['FLAGS_enable_eager_mode'] = '1' - import random import unittest diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_save_for_auto_infer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_save_for_auto_infer.py index fdcfd76bd66..f78c459c270 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_save_for_auto_infer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_save_for_auto_infer.py @@ -44,6 +44,5 @@ class TestSingleCard(unittest.TestCase): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py deleted file mode 100644 index 8087fc98b6a..00000000000 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from test_parallel_dygraph_dataparallel import TestMultipleGpus - - -class TestDygraphShardingOptimizerStage2(TestMultipleGpus): - - # check sharding logic as well as the accuracy with single mode - def test_dygraph_sharding_optimizer_stage2(self): - self.run_mnist_2gpu( - 'dygraph_sharding_optimizer_stage2.py', eager_mode=False - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py index 0dd1d4961b3..1c3e77102b0 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -23,18 +22,13 @@ class TestDygraphShardingStage2(TestMultipleGpus): # check sharding logic as well as the accuracy with single mode def test_dygraph_sharding_stage2(self): self.run_mnist_2gpu('dygraph_group_sharded_stage2.py') - self.run_mnist_2gpu('dygraph_sharding_stage2.py', eager_mode=False) def test_dygraph_sharding_stage2_offload(self): self.run_mnist_2gpu('dygraph_group_sharded_stage2_offload.py') - self.run_mnist_2gpu( - 'dygraph_sharding_stage2_offload.py', eager_mode=False - ) def test_dygraph_sharding_stage2_with_comm_overlap(self): self.run_mnist_2gpu('dygraph_group_sharded_stage2_comm_overlap.py') if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py deleted file mode 100644 index ea4edbd3493..00000000000 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -os.environ['FLAGS_enable_eager_mode'] = '0' - -import unittest - -from test_parallel_dygraph_dataparallel import TestMultipleGpus - - -class TestDygraphShardingStage3(TestMultipleGpus): - - # check sharding logic as well as the accuracy with single mode - def test_dygraph_sharding_stage3(self): - self.run_mnist_2gpu('dygraph_sharding_stage3.py', eager_mode=False) - - def test_dygraph_sharding_stage3_offload(self): - self.run_mnist_2gpu( - 'dygraph_sharding_stage3_offload.py', eager_mode=False - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3_for_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3_for_eager.py index a6f9436a4f0..a0c1dfee4b4 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3_for_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3_for_eager.py @@ -12,11 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - -os.environ['FLAGS_enable_eager_mode'] = '1' - -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -33,5 +28,4 @@ class TestDygraphShardingStage3(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py deleted file mode 100644 index 2af1f4adec9..00000000000 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py +++ /dev/null @@ -1,1485 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -os.environ['FLAGS_enable_eager_mode'] = '0' - -import tempfile -import unittest - -import numpy as np -from test_imperative_resnet import ResNet, optimizer_setting, train_parameters - -import paddle -import paddle.fluid as fluid -import paddle.fluid.core as core -import paddle.nn as nn -from paddle.autograd import PyLayer -from paddle.static import InputSpec - -if fluid.core.is_compiled_with_cuda(): - fluid.set_flags({"FLAGS_cudnn_deterministic": True}) - - -class SimpleConv(fluid.dygraph.Layer): - def __init__( - self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None, - ): - super().__init__() - self._conv = paddle.nn.Conv2D( - in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=None, - ) - - def forward(self, inputs): - return self._conv(inputs) - - -class TestAutoCast(unittest.TestCase): - def amp_guard_white_op(self): - data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') - with fluid.dygraph.guard(): - conv2d = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - data = fluid.dygraph.to_variable(data) - with fluid.dygraph.amp_guard(True): - out_fp16 = conv2d(data) - - with fluid.dygraph.amp_guard(False): - out_fp32 = conv2d(data) - - self.assertTrue(data.dtype == fluid.core.VarDesc.VarType.FP32) - self.assertTrue(out_fp16.dtype == fluid.core.VarDesc.VarType.FP16) - self.assertTrue(out_fp32.dtype == fluid.core.VarDesc.VarType.FP32) - - def test_amp_guard_white_op(self): - self.amp_guard_white_op() - - def amp_guard_black_op(self): - data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') - with fluid.dygraph.guard(): - data = fluid.dygraph.to_variable(data) - with fluid.dygraph.amp_guard(True): - out_fp32 = paddle.mean(data) - - self.assertTrue(data.dtype == fluid.core.VarDesc.VarType.FP32) - self.assertTrue(out_fp32.dtype == fluid.core.VarDesc.VarType.FP32) - - def test_amp_guard_black_op(self): - self.amp_guard_black_op() - - def custom_op_list(self): - with fluid.dygraph.guard(): - tracer = fluid.framework._dygraph_tracer() - base_white_list = fluid.dygraph.amp.auto_cast.WHITE_LIST - base_black_list = fluid.dygraph.amp.auto_cast.BLACK_LIST - with fluid.dygraph.amp_guard( - custom_white_list=["log"], custom_black_list=["conv2d"] - ): - white_list, black_list = tracer._get_amp_op_list() - self.assertTrue( - set(white_list) - == (set(base_white_list) | {"log"}) - {"conv2d"} - ) - - self.assertTrue( - set(black_list) - == (set(base_black_list) - {"log"}) | {"conv2d"} - ) - - base_white_list = fluid.dygraph.amp.auto_cast.PURE_FP16_WHITE_LIST - base_black_list = fluid.dygraph.amp.auto_cast.PURE_FP16_BLACK_LIST - with fluid.dygraph.amp_guard( - custom_white_list=["log"], - custom_black_list=["conv2d"], - level='O2', - ): - white_list, black_list = tracer._get_amp_op_list() - self.assertTrue( - set(white_list) - == (set(base_white_list) | {"log"}) - {"conv2d"} - ) - - self.assertTrue( - set(black_list) - == (set(base_black_list) - {"log"}) | {"conv2d"} - ) - - def test_custom_op_list(self): - self.custom_op_list() - - def custom_op_list_exception(self): - inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32) - - def func(): - with fluid.dygraph.guard(): - model = SimpleConv( - num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu', - ) - with fluid.dygraph.amp_guard( - custom_white_list=["conv2d"], custom_black_list=["conv2d"] - ): - inp = fluid.dygraph.to_variable(inp_np) - out = model(inp) - - self.assertRaises(ValueError, func) - - def test_custom_op_list_exception(self): - self.custom_op_list_exception() - - def amp_guard_upsupported_fp16_op(self): - data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') - with fluid.dygraph.guard(): - conv2d = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - data = fluid.dygraph.to_variable(data) - with fluid.dygraph.amp_guard(True): - out_amp_fp16 = conv2d(data) - out_amp_fp32 = paddle.expand_as( - out_amp_fp16, out_amp_fp16 - ) # expand_as_v2 has no fp16 kernel - - with fluid.dygraph.amp_guard(True, level='O2'): - out_purefp16_fp16 = conv2d(data) - out_purefp16_fp32 = paddle.expand_as( - out_purefp16_fp16, out_purefp16_fp16 - ) # expand_as_v2 has no fp16 kernel - self.assertTrue(data.dtype == fluid.core.VarDesc.VarType.FP32) - self.assertTrue(out_amp_fp16.dtype == fluid.core.VarDesc.VarType.FP16) - self.assertTrue(out_amp_fp32.dtype == fluid.core.VarDesc.VarType.FP32) - self.assertTrue( - out_purefp16_fp16.dtype == fluid.core.VarDesc.VarType.FP16 - ) - self.assertTrue( - out_purefp16_fp32.dtype == fluid.core.VarDesc.VarType.FP32 - ) - - def test_amp_guard_upsupported_fp16_op(self): - self.amp_guard_upsupported_fp16_op() - - def mode_exception(self): - def func(): - data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') - with fluid.dygraph.guard(): - conv2d = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - data = fluid.dygraph.to_variable(data) - with fluid.dygraph.amp_guard(level='O'): - out = conv2d(data) - - self.assertRaises(ValueError, func) - - def test_mode_exception(self): - self.mode_exception() - - -class TestAmpScaler(unittest.TestCase): - def scale(self): - with fluid.dygraph.guard(): - data = paddle.rand([10, 1024]) - scaler = paddle.fluid.dygraph.AmpScaler(init_loss_scaling=1024) - scaled_data = scaler.scale(data) - self.assertEqual( - np.array_equal(scaled_data.numpy(), data.numpy() * 1024), True - ) - - def test_scale(self): - self.scale() - - def minimize(self): - inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32) - - def run_simple_conv(inp_np, use_scaler=True): - paddle.seed(10) - paddle.framework.random._manual_program_seed(10) - with fluid.dygraph.guard(): - model = SimpleConv( - num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu', - ) - optimizer = fluid.optimizer.SGDOptimizer( - learning_rate=0.01, parameter_list=model.parameters() - ) - scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024) - data = fluid.dygraph.to_variable(inp_np) - - out = model(data) - loss = paddle.mean(out) - if use_scaler: - print('use scaler') - scaled_loss = scaler.scale(loss) - scaled_loss.backward() - optimize_ops, params_grads = scaler.minimize( - optimizer, scaled_loss - ) - else: - print('use no scaler') - loss.backward() - optimize_ops, params_grads = optimizer.minimize(loss) - return optimize_ops, params_grads - - outs_with_scaler = run_simple_conv(inp_np, use_scaler=True) - outs_no_scaler = run_simple_conv(inp_np, use_scaler=False) - - self.assertEqual( - outs_with_scaler[0], [] - ) # optimize_ops is [] in dygraph mode - self.assertEqual( - outs_no_scaler[0], [] - ) # optimize_ops is [] in dygraph mode - for i in range(len(outs_with_scaler[1])): - # check each grad - np.testing.assert_allclose( - outs_with_scaler[1][i][1].numpy(), - outs_no_scaler[1][i][1].numpy(), - rtol=1e-05, - ) - # check each parameter - np.testing.assert_allclose( - outs_with_scaler[1][i][0].numpy(), - outs_no_scaler[1][i][0].numpy(), - rtol=1e-05, - ) - - def test_minimize(self): - self.minimize() - - def step(self): - inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32) - - def run_simple_conv(inp_np, use_scaler=True): - paddle.seed(10) - paddle.framework.random._manual_program_seed(10) - with fluid.dygraph.guard(): - model = SimpleConv( - num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu', - ) - optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=model.parameters() - ) - scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - data = fluid.dygraph.to_variable(inp_np) - - out = model(data) - loss = paddle.mean(out) - if use_scaler: - print('use scaler') - scaled_loss = scaler.scale(loss) - scaled_loss.backward() - scaler.step(optimizer) - scaler.update() - else: - print('use no scaler') - loss.backward() - optimizer.step() - return optimizer._parameter_list - - outs_with_scaler = run_simple_conv(inp_np, use_scaler=True) - outs_no_scaler = run_simple_conv(inp_np, use_scaler=False) - - for i in range(len(outs_with_scaler)): - # check each parameter - np.testing.assert_allclose( - outs_with_scaler[i].numpy(), - outs_no_scaler[i].numpy(), - rtol=1e-05, - ) - - def test_step(self): - self.step() - - def nan_inf(self): - inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32) - inp_np[0][1][2][3] = np.nan - with fluid.dygraph.guard(): - model = SimpleConv( - num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu', - ) - params_init = {} - for param in model.parameters(): - params_init[param.name] = param.numpy() - optimizer = fluid.optimizer.SGDOptimizer( - learning_rate=0.01, parameter_list=model.parameters() - ) - scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024) - data = fluid.dygraph.to_variable(inp_np) - - out = model(data) - loss = paddle.mean(out) - scaled_loss = scaler.scale(loss) - scaled_loss.backward() - optimize_ops, params_grads = scaler.minimize(optimizer, scaled_loss) - self.assertEqual(scaler._found_inf.numpy() >= 1, True) - - for param in model.parameters(): - # param not update when tensor contains nan or inf - np.testing.assert_array_equal( - param.numpy(), params_init[param.name] - ) - - def test_nan_inf(self): - self.nan_inf() - - def step_update_exception(self): - def func1(): - model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=model.parameters() - ) - scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - data = paddle.rand([10, 3, 32, 32]) - conv = model(data) - loss = paddle.mean(conv) - scaled = scaler.scale(loss) - scaled.backward() - scaler.unscale_(optimizer) - scaler.unscale_(optimizer) - - self.assertRaises(RuntimeError, func1) - - def func2(): - model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=model.parameters() - ) - scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - data = paddle.rand([10, 3, 32, 32]) - conv = model(data) - loss = paddle.mean(conv) - scaled = scaler.scale(loss) - scaled.backward() - scaler.step(optimizer) - scaler.unscale_(optimizer) - - self.assertRaises(RuntimeError, func2) - - def func3(): - model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=model.parameters() - ) - scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - data = paddle.rand([10, 3, 32, 32]) - conv = model(data) - loss = paddle.mean(conv) - scaled = scaler.scale(loss) - scaled.backward() - scaler.step(optimizer) - scaler.step(optimizer) - - self.assertRaises(RuntimeError, func3) - - def test_step_update_exception(self): - self.step_update_exception() - - def test_get_and_set(self): - with fluid.dygraph.guard(): - scaler = paddle.amp.GradScaler( - enable=True, - init_loss_scaling=1024, - incr_ratio=2.0, - decr_ratio=0.5, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=2, - use_dynamic_loss_scaling=True, - ) - self.assertEqual(scaler.is_enable(), True) - self.assertEqual(scaler.get_init_loss_scaling() == 1024, True) - self.assertEqual(scaler.get_incr_ratio() == 2.0, True) - self.assertEqual(scaler.get_decr_ratio() == 0.5, True) - self.assertEqual(scaler.get_incr_every_n_steps() == 1000, True) - self.assertEqual(scaler.get_decr_every_n_nan_or_inf() == 2, True) - self.assertEqual(scaler.is_use_dynamic_loss_scaling(), True) - scaler.set_decr_every_n_nan_or_inf(4) - self.assertEqual(scaler.get_decr_every_n_nan_or_inf() == 4, True) - scaler.set_decr_ratio(0.1) - self.assertEqual(scaler.get_decr_ratio() == 0.1, True) - scaler.set_incr_every_n_steps(200) - self.assertEqual(scaler.get_incr_every_n_steps() == 200, True) - scaler.set_incr_ratio(3.0) - self.assertEqual(scaler.get_incr_ratio() == 3.0, True) - scaler.set_init_loss_scaling(100) - self.assertEqual(scaler.get_init_loss_scaling() == 100, True) - - def test_state_dict_and_load_state_dict(self): - with fluid.dygraph.guard(): - scaler1 = paddle.amp.GradScaler( - enable=True, - init_loss_scaling=14, - incr_ratio=233.0, - decr_ratio=0.523, - incr_every_n_steps=1090, - decr_every_n_nan_or_inf=20, - use_dynamic_loss_scaling=True, - ) - scaler_state = scaler1.state_dict() - scaler2 = paddle.amp.GradScaler(enable=True) - scaler2.load_state_dict(scaler_state) - self.assertEqual(scaler2.get_init_loss_scaling() == 14, True) - self.assertEqual(scaler2.get_incr_ratio() == 233.0, True) - self.assertEqual(scaler2.get_decr_ratio() == 0.523, True) - self.assertEqual(scaler2.get_incr_every_n_steps() == 1090, True) - self.assertEqual(scaler2.get_decr_every_n_nan_or_inf() == 20, True) - - scaler3 = paddle.amp.GradScaler(enable=False) - scaler3.load_state_dict(scaler_state) - self.assertFalse(scaler3.is_enable()) - - def test_state_dict_and_load_state_dict_error(self): - def test_error(): - state_empty = {} - scaler = paddle.amp.GradScaler(enable=True) - scaler.load_state_dict(state_empty) - - self.assertRaises(RuntimeError, test_error) - - -def reader_decorator(reader): - def __reader__(): - for item in reader(): - img = np.array(item[0]).astype('float32').reshape(3, 224, 224) - label = np.array(item[1]).astype('int64').reshape(1) - yield img, label - - return __reader__ - - -class TestGradScalerStateDict(unittest.TestCase): - def train_resnet( - self, enable_amp=True, use_data_loader=True, use_save_load=True - ): - seed = 90 - - batch_size = train_parameters["batch_size"] - batch_num = 4 - - paddle.seed(seed) - paddle.framework.random._manual_program_seed(seed) - - resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting( - train_parameters, parameter_list=resnet.parameters() - ) - np.random.seed(seed) - train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size - ) - - dy_param_init_value = {} - for param in resnet.parameters(): - dy_param_init_value[param.name] = param.numpy() - - program = None - scaler = paddle.amp.GradScaler( - enable=enable_amp, init_loss_scaling=2.0**10 - ) - - if use_data_loader: - train_reader = paddle.batch( - reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True, - ) - train_loader = fluid.io.DataLoader.from_generator( - capacity=4, - use_double_buffer=True, - iterable=True, - return_list=True, - ) - train_loader.set_sample_list_generator(train_reader) - train_reader = train_loader - - for batch_id, data in enumerate(train_reader()): - if batch_id >= batch_num: - break - if use_data_loader: - img, label = data - else: - dy_x_data = np.array( - [x[0].reshape(3, 224, 224) for x in data] - ).astype('float32') - if ( - len(np.array([x[1] for x in data]).astype('int64')) - != batch_size - ): - continue - y_data = ( - np.array([x[1] for x in data]) - .astype('int64') - .reshape(-1, 1) - ) - - img = paddle.to_tensor(dy_x_data) - label = paddle.to_tensor(y_data) - label.stop_gradient = True - - with paddle.amp.auto_cast(enable=enable_amp): - out = resnet(img) - - loss = paddle.nn.functional.cross_entropy(input=out, label=label) - avg_loss = paddle.mean(x=loss) - - dy_out = avg_loss.numpy() - - scaled_loss = scaler.scale(avg_loss) - scaled_loss.backward() - - scaler.minimize(optimizer, scaled_loss) - - dy_grad_value = {} - for param in resnet.parameters(): - if param.trainable: - np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[ - param.name + fluid.core.grad_var_suffix() - ] = np_array - - resnet.clear_gradients() - - dy_param_value = {} - for param in resnet.parameters(): - dy_param_value[param.name] = param.numpy() - - if use_save_load and batch_id == 2: - paddle.save(scaler.state_dict(), 'ResNet_model.pdparams') - dict_load = paddle.load('ResNet_model.pdparams') - scaler.load_state_dict(dict_load) - if use_data_loader: - train_reader._reset() - return dy_out, dy_param_value, dy_grad_value - - def test_with_state_dict(self): - def func_isinstance(): - with fluid.dygraph.guard(): - out_use_state_dict = self.train_resnet( - enable_amp=True, use_data_loader=True, use_save_load=True - ) - out_no_state_dict = self.train_resnet( - enable_amp=True, use_data_loader=True, use_save_load=False - ) - print('save_load:', out_use_state_dict[0], out_no_state_dict[0]) - np.testing.assert_allclose( - out_use_state_dict[0], out_no_state_dict[0], rtol=1e-05 - ) - - func_isinstance() - - -class TestAmpDecorator(unittest.TestCase): - def test_mode_exception(self): - def func(): - with fluid.dygraph.guard(): - model = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - opt = paddle.optimizer.SGD(parameters=model.parameters()) - model, opt = paddle.amp.decorate( - models=model, optimizers=opt, level='O' - ) - - self.assertRaises(ValueError, func) - - def test_input_type_exception(self): - def test_error_model(): - class MyModel: - def __init__(self): - print("A fake Model") - - model = MyModel() - with fluid.dygraph.guard(): - paddle.amp.decorate(models=model, optimizers=None, level='O2') - - self.assertRaises(TypeError, test_error_model) - - def test_error_distributed_model(): - model = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - model = paddle.DataParallel(model) - with fluid.dygraph.guard(): - model = paddle.amp.decorate(models=model, level='O2') - - self.assertRaises(RuntimeError, test_error_distributed_model) - - def test_error_optimizer(): - class MyOptimizer: - def __init__(self): - print("A fake Optimizer") - - model = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - opt = MyOptimizer() - with fluid.dygraph.guard(): - paddle.amp.decorate(models=model, optimizers=opt, level='O2') - - self.assertRaises(TypeError, test_error_optimizer) - - def test_set_master_weight(self): - model1 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - opt1 = paddle.optimizer.Adam( - learning_rate=0.0001, - parameters=model1.parameters(), - multi_precision=True, - ) - - model2 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - opt2 = paddle.optimizer.Adam( - learning_rate=0.0001, - parameters=model2.parameters(), - multi_precision=False, - ) - - model1, opt1 = paddle.amp.decorate( - models=model1, optimizers=opt1, level='O2', master_weight=None - ) - self.assertEqual(opt1._multi_precision, True) - - models, opt2 = paddle.amp.decorate( - models=[model1, model2], - optimizers=opt2, - level='O2', - master_weight=None, - ) - self.assertEqual(opt2._multi_precision, True) - - model3 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - opt3 = paddle.optimizer.Adam( - learning_rate=0.0001, parameters=model3.parameters() - ) - - model4 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - opt4 = paddle.optimizer.Adam( - learning_rate=0.0001, parameters=model4.parameters() - ) - - model3, opts = paddle.amp.decorate( - models=model3, - optimizers=[opt3, opt4], - level='O2', - master_weight=True, - ) - self.assertEqual(opts[0]._multi_precision, True) - self.assertEqual(opts[1]._multi_precision, True) - - models = [model3, model4] - optimizers = [opt3, opt4] - models, optimizers = paddle.amp.decorate( - models=models, - optimizers=optimizers, - level='O2', - master_weight=False, - ) - self.assertEqual(optimizers[0]._multi_precision, False) - self.assertEqual(optimizers[1]._multi_precision, False) - - def test_skip_BatchNorm_Layer_norm(self): - model = paddle.nn.LayerNorm(1) - model = paddle.amp.decorate(models=model, level='O2') - for param in model.parameters(): - self.assertEqual((param.dtype == paddle.float32), True) - - model = paddle.nn.BatchNorm(1) - model = paddle.amp.decorate(models=model, level='O2') - for param in model.parameters(): - self.assertEqual((param.dtype == paddle.float32), True) - - model = paddle.nn.BatchNorm1D(1) - model = paddle.amp.decorate(models=model, level='O2') - for param in model.parameters(): - self.assertEqual((param.dtype == paddle.float32), True) - - model = paddle.nn.BatchNorm2D(1) - model = paddle.amp.decorate(models=model, level='O2') - for param in model.parameters(): - self.assertEqual((param.dtype == paddle.float32), True) - - model = paddle.nn.BatchNorm3D(1) - model = paddle.amp.decorate(models=model, level='O2') - for param in model.parameters(): - self.assertEqual((param.dtype == paddle.float32), True) - - def test_floating_only(self): - model = paddle.nn.Linear(2, 4) - buffer = paddle.to_tensor(np.array([5]).astype("int32")) - model.register_buffer("buffer_name", buffer, persistable=True) - model = paddle.amp.decorate(models=model, level='O2') - self.assertEqual( - (model._buffers["buffer_name"].dtype == paddle.int32), True - ) - - -class TestStateDictHookForAMP(unittest.TestCase): - def test_state_dict_hook(self): - def func_isinstance(): - paddle.seed(100) - model = paddle.nn.Linear(2, 4) - model = paddle.amp.decorate( - models=model, level='O2', save_dtype='float32' - ) - param_value_ori = {} - for param in model.parameters(): - param_value_ori[param.name] = param.numpy() - - state_dict = model.state_dict() - for key, value in state_dict.items(): - state_dict[key] = value.cast("float16") - model.set_state_dict(state_dict) - - param_value_now = {} - for param in model.parameters(): - param_value_now[param.name] = param.numpy() - - for key in param_value_ori.keys(): - print(np.equal(param_value_ori[key], param_value_now[key])) - - func_isinstance() - - -class TestPureFp16SaveLoad(unittest.TestCase): - def setUp(self): - self.temp_dir = tempfile.TemporaryDirectory() - - def tearDown(self): - self.temp_dir.cleanup() - - def test_save_dtype_exception(self): - def func(): - paddle.disable_static() - model = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) - opt = paddle.optimizer.SGD(parameters=model.parameters()) - paddle.amp.decorate( - models=model, optimizers=opt, level='O2', save_dtype='int' - ) - - self.assertRaises(ValueError, func) - - def train_resnet( - self, enable_amp=True, use_data_loader=True, use_save_load=True - ): - seed = 90 - - batch_size = train_parameters["batch_size"] - batch_num = 4 - - paddle.seed(seed) - paddle.framework.random._manual_program_seed(seed) - - resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting( - train_parameters, parameter_list=resnet.parameters() - ) - np.random.seed(seed) - train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size - ) - - dy_param_init_value = {} - for param in resnet.parameters(): - dy_param_init_value[param.name] = param.numpy() - - program = None - scaler = paddle.amp.GradScaler( - enable=enable_amp, init_loss_scaling=2.0**10 - ) - - if use_data_loader: - train_reader = paddle.batch( - reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True, - ) - train_loader = fluid.io.DataLoader.from_generator( - capacity=4, - use_double_buffer=True, - iterable=True, - return_list=True, - ) - train_loader.set_sample_list_generator(train_reader) - train_reader = train_loader - - if enable_amp: - resnet, optimizer = paddle.amp.decorate( - models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32', - ) - - for batch_id, data in enumerate(train_reader()): - if batch_id >= batch_num: - break - if use_data_loader: - img, label = data - else: - dy_x_data = np.array( - [x[0].reshape(3, 224, 224) for x in data] - ).astype('float32') - if ( - len(np.array([x[1] for x in data]).astype('int64')) - != batch_size - ): - continue - y_data = ( - np.array([x[1] for x in data]) - .astype('int64') - .reshape(-1, 1) - ) - - img = paddle.to_tensor(dy_x_data) - label = paddle.to_tensor(y_data) - label.stop_gradient = True - - with paddle.amp.auto_cast(enable=enable_amp, level='O2'): - out = resnet(img) - - loss = paddle.nn.functional.cross_entropy(input=out, label=label) - loss = paddle.cast(loss, 'float32') - avg_loss = paddle.mean(x=loss) - - dy_out = avg_loss.numpy() - - scaled_loss = scaler.scale(avg_loss) - scaled_loss.backward() - - scaler.minimize(optimizer, scaled_loss) - - dy_grad_value = {} - for param in resnet.parameters(): - if param.trainable: - np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[ - param.name + fluid.core.grad_var_suffix() - ] = np_array - - resnet.clear_gradients() - - dy_param_value = {} - for param in resnet.parameters(): - dy_param_value[param.name] = param.numpy() - - if use_save_load and batch_id == 2: - # paddle.save - obj = { - 'model': resnet.state_dict(), - 'opt': optimizer.state_dict(), - 'scaler': scaler.state_dict(), - } - path = os.path.join(self.temp_dir.name, 'model.pdparams') - paddle.save(obj, path) - # paddle.load - obj_load = paddle.load(path) - resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting( - train_parameters, parameter_list=resnet.parameters() - ) - resnet.set_state_dict(obj_load['model']) - optimizer.set_state_dict(obj_load['opt']) - scaler.load_state_dict(obj_load['scaler']) - resnet, optimizer = paddle.amp.decorate( - models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32', - ) - - if use_data_loader: - train_reader._reset() - return dy_out, dy_param_value, dy_grad_value - - def test_with_save_load(self): - def func_isinstance(): - with fluid.dygraph.guard(): - out_use_save_load = self.train_resnet( - enable_amp=True, use_data_loader=True, use_save_load=True - ) - out_no_save_load = self.train_resnet( - enable_amp=True, use_data_loader=True, use_save_load=False - ) - print('save_load:', out_use_save_load[0], out_no_save_load[0]) - np.testing.assert_allclose( - out_use_save_load[0], out_no_save_load[0], rtol=1e-05 - ) - - func_isinstance() - - -class TestPureFp16InferenceSaveLoad(unittest.TestCase): - def setUp(self): - self.temp_dir = tempfile.TemporaryDirectory() - - def tearDown(self): - self.temp_dir.cleanup() - - def inference_save_load(self): - BATCH_SIZE = 16 - BATCH_NUM = 4 - EPOCH_NUM = 4 - IMAGE_SIZE = 784 - CLASS_NUM = 10 - - # define a random dataset - class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): - self.num_samples = num_samples - - def __getitem__(self, idx): - image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype( - 'int64' - ) - return image, label - - def __len__(self): - return self.num_samples - - class LinearNet(nn.Layer): - def __init__(self): - super().__init__() - self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) - - def forward(self, x): - return self._linear(x) - - def train(layer, loader, loss_fn, opt): - for epoch_id in range(EPOCH_NUM): - for batch_id, (image, label) in enumerate(loader()): - with paddle.amp.auto_cast( - enable=True, - custom_white_list=None, - custom_black_list=None, - level='O2', - ): - out = layer(image) - loss = loss_fn(out, label) - loss.backward() - opt.step() - opt.clear_grad() - - # train - layer = LinearNet() - adam = paddle.optimizer.Adam( - learning_rate=0.001, - parameters=layer.parameters(), - multi_precision=True, - ) - loss_fn = nn.CrossEntropyLoss() - layer, adam = paddle.amp.decorate( - models=layer, optimizers=adam, save_dtype='float32' - ) - dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - loader = paddle.io.DataLoader( - dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2, - ) - - train(layer, loader, loss_fn, adam) - - # save - path = os.path.join(self.temp_dir.name, 'example_model/linear') - paddle.jit.save( - layer, path, input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')] - ) - - # jit.load - loaded_layer = paddle.jit.load(path) - - # inference - loaded_layer.eval() - x = np.random.randn(1, IMAGE_SIZE).astype('float32') - x_tensor = paddle.to_tensor(x) - pred = loaded_layer(x_tensor) - - # load_inference_model - paddle.enable_static() - exe = paddle.static.Executor() - [ - inference_program, - feed_target_names, - fetch_targets, - ] = paddle.static.load_inference_model(path, exe) - tensor_img = x - results = exe.run( - inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets, - ) - print("pred.numpy()", pred.numpy()) - print("result", results[0]) - np.testing.assert_array_equal(pred.numpy(), results[0]) - paddle.disable_static() - - def test_inference_save_load(self): - self.inference_save_load() - - -class TestResnet2(unittest.TestCase): - """ - Use paddle-2.0 API - """ - - def train_resnet( - self, - enable_amp=True, - level='O1', - use_data_loader=False, - use_param_group=False, - ): - seed = 90 - - batch_size = train_parameters["batch_size"] - batch_num = 10 - - paddle.seed(seed) - paddle.framework.random._manual_program_seed(seed) - - resnet = ResNet(use_cudnn=True) - - if use_param_group: - conv_params = resnet.conv.parameters() - other_params = [] - for p in resnet.parameters(): - contains = False - for q in conv_params: - if p is q: - contains = True - if not contains: - other_params.append(p) - # NOTE(zhiqiu): The Membership test operations(in / not in) calls "is" and "equal", - # see details: https://docs.python.org/3/reference/expressions.html#membership-test-operations. - # So do not use other_params = [p for p in resnet.parameters() if p not in conv_params] - optimizer = paddle.optimizer.Momentum( - parameters=[ - {'params': conv_params, 'learning_rate': 0.01}, - {'params': other_params, 'learning_rate': 0.001}, - ], - multi_precision=True, - ) - else: - optimizer = paddle.optimizer.SGD(parameters=resnet.parameters()) - - np.random.seed(seed) - train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size - ) - - dy_param_init_value = {} - for param in resnet.parameters(): - dy_param_init_value[param.name] = param.numpy() - - program = None - scaler = paddle.amp.GradScaler( - enable=enable_amp, init_loss_scaling=2.0**10 - ) - - if use_data_loader: - train_reader = paddle.batch( - reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True, - ) - train_loader = fluid.io.DataLoader.from_generator( - capacity=4, - use_double_buffer=True, - iterable=True, - return_list=True, - ) - train_loader.set_sample_list_generator(train_reader) - train_reader = train_loader - - if enable_amp and (level == 'O2'): - resnet = paddle.amp.decorate(models=resnet, level='O2') - - for batch_id, data in enumerate(train_reader()): - if batch_id >= batch_num: - break - if use_data_loader: - img, label = data - else: - dy_x_data = np.array( - [x[0].reshape(3, 224, 224) for x in data] - ).astype('float32') - if ( - len(np.array([x[1] for x in data]).astype('int64')) - != batch_size - ): - continue - y_data = ( - np.array([x[1] for x in data]) - .astype('int64') - .reshape(-1, 1) - ) - - img = paddle.to_tensor(dy_x_data) - label = paddle.to_tensor(y_data) - label.stop_gradient = True - - with paddle.amp.auto_cast(enable=enable_amp, level=level): - out = resnet(img) - - loss = paddle.nn.functional.cross_entropy(input=out, label=label) - loss = paddle.cast(loss, 'float32') - avg_loss = paddle.mean(x=loss) - - dy_out = avg_loss.numpy() - - scaled_loss = scaler.scale(avg_loss) - scaled_loss.backward() - scaler.unscale_(optimizer) - scaler.step(optimizer) - scaler.update() - - dy_grad_value = {} - for param in resnet.parameters(): - if param.trainable: - np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[ - param.name + fluid.core.grad_var_suffix() - ] = np_array - - resnet.clear_gradients() - - dy_param_value = {} - for param in resnet.parameters(): - dy_param_value[param.name] = param.numpy() - if use_data_loader: - train_reader._reset() - return dy_out, dy_param_value, dy_grad_value - - def test_resnet(self): - def func_isinstance(): - with fluid.dygraph.guard(): - out_fp32 = self.train_resnet(enable_amp=False) - out_amp = self.train_resnet(enable_amp=True) - out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2') - print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose( - out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 - ) - np.testing.assert_allclose( - out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 - ) - - func_isinstance() - - def test_with_data_loader(self): - def func_isinstance(): - with fluid.dygraph.guard(): - out_fp32 = self.train_resnet( - enable_amp=False, use_data_loader=True - ) - out_amp = self.train_resnet( - enable_amp=True, use_data_loader=True - ) - out_pure_fp16 = self.train_resnet( - enable_amp=True, use_data_loader=True, level='O2' - ) - print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose( - out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 - ) - np.testing.assert_allclose( - out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 - ) - - func_isinstance() - - def test_param_group(self): - def func_isinstance(): - with fluid.dygraph.guard(): - out_fp32 = self.train_resnet( - enable_amp=False, use_data_loader=True, use_param_group=True - ) - out_amp = self.train_resnet( - enable_amp=True, use_data_loader=True, use_param_group=True - ) - out_pure_fp16 = self.train_resnet( - enable_amp=True, - use_data_loader=True, - use_param_group=True, - level='O2', - ) - print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose( - out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 - ) - np.testing.assert_allclose( - out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 - ) - - func_isinstance() - - -class TestResnet(unittest.TestCase): - """ - Use paddle-1.x API - """ - - def train_resnet(self, enable_amp=True, level='O1'): - seed = 90 - - batch_size = train_parameters["batch_size"] - batch_num = 1 - - with fluid.dygraph.guard(): - paddle.seed(seed) - paddle.framework.random._manual_program_seed(seed) - - resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting( - train_parameters, parameter_list=resnet.parameters() - ) - optimizer = paddle.optimizer.Momentum( - parameters=resnet.parameters(), multi_precision=True - ) - np.random.seed(seed) - train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size, - ) - - dy_param_init_value = {} - for param in resnet.parameters(): - dy_param_init_value[param.name] = param.numpy() - - program = None - scaler = paddle.fluid.dygraph.AmpScaler( - enable=enable_amp, init_loss_scaling=2.0**10 - ) - - if enable_amp and (level == 'O2'): - resnet, optimizer = paddle.fluid.dygraph.amp_decorate( - models=resnet, optimizers=optimizer, level='O2' - ) - - for batch_id, data in enumerate(train_reader()): - if batch_id >= batch_num: - break - dy_x_data = np.array( - [x[0].reshape(3, 224, 224) for x in data] - ).astype('float32') - if ( - len(np.array([x[1] for x in data]).astype('int64')) - != batch_size - ): - continue - y_data = ( - np.array([x[1] for x in data]) - .astype('int64') - .reshape(-1, 1) - ) - img = fluid.dygraph.to_variable(dy_x_data) - label = fluid.dygraph.to_variable(y_data) - label.stop_gradient = True - with paddle.fluid.dygraph.amp_guard( - enable=enable_amp, level=level - ): - out = resnet(img) - - loss = paddle.nn.functional.cross_entropy( - input=out, label=label, reduction='none', use_softmax=False - ) - avg_loss = paddle.mean(x=loss) - - dy_out = avg_loss.numpy() - - scaled_loss = scaler.scale(avg_loss) - scaled_loss.backward() - - scaler.minimize(optimizer, scaled_loss) - - dy_grad_value = {} - for param in resnet.parameters(): - if param.trainable: - np_array = np.array( - param._grad_ivar().value().get_tensor() - ) - dy_grad_value[ - param.name + fluid.core.grad_var_suffix() - ] = np_array - - resnet.clear_gradients() - - dy_param_value = {} - for param in resnet.parameters(): - dy_param_value[param.name] = param.numpy() - - return dy_out, dy_param_value, dy_grad_value - - def test_resnet(self): - def func_isinstance(): - out_fp32 = self.train_resnet(enable_amp=False) - out_amp = self.train_resnet(enable_amp=True) - out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2') - print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose( - out_fp32[0], out_amp[0], rtol=1e-05, atol=0.01 - ) - np.testing.assert_allclose( - out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.1 - ) - - func_isinstance() - - -class TestLayerNormFp16(unittest.TestCase): - r'''layer_norm and batch_norm support mixed inputs, i.e., only input x is fp16 - and other params are fp32. - ''' - - def test_layer_norm_fp16(self): - def func_isinstance(): - if fluid.is_compiled_with_cuda(): - with fluid.dygraph.guard(fluid.CUDAPlace(0)): - x = paddle.rand([2, 2, 2, 3]) - layer_norm = paddle.nn.LayerNorm(x.shape[1:]) - with paddle.amp.auto_cast(custom_white_list=['layer_norm']): - out = layer_norm(x) - - self.assertTrue( - out.dtype == fluid.core.VarDesc.VarType.FP16 - ) - - func_isinstance() - - -@unittest.skipIf( - paddle.is_compiled_with_cuda() - and not core.is_bfloat16_supported(core.CUDAPlace(0)), - "skip bf16 test if cuda is in use but bf16 is not supported by gpu arch.", -) -class TestBf16(unittest.TestCase): - ''' - test amp for BF16 - ''' - - def train(self, enable_amp=True, amp_level='O1'): - paddle.seed(100) - input = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) - conv = paddle.nn.Conv2D(4, 6, (3, 3)) - if amp_level == 'O2': - conv = paddle.amp.decorate( - models=conv, level=amp_level, dtype='bfloat16' - ) - with paddle.amp.auto_cast( - enable=enable_amp, level=amp_level, dtype='bfloat16' - ): - output = conv(input) - output = output.cast('float32') - return output.numpy() - - def test_bf16(self): - def func_isinstance(): - out_fp32 = self.train(enable_amp=False) - out_bf16_O1 = self.train(enable_amp=True, amp_level='O1') - out_bf16_O2 = self.train(enable_amp=True, amp_level='O2') - np.testing.assert_allclose( - out_fp32, out_bf16_O1, rtol=0.001, atol=0.1 - ) - np.testing.assert_allclose( - out_fp32, out_bf16_O2, rtol=0.001, atol=0.1 - ) - - func_isinstance() - - -class TestAmpWithPyLyer(unittest.TestCase): - def test_pylayer(self): - class MyMM(PyLayer): - @staticmethod - def forward(ctx, a, b): - ctx.save_for_backward(a, b) - return a.mm(b) - - @staticmethod - def backward(ctx, grad): - a, b = ctx.saved_tensor() - # NOTE(zhiqiu): a and b is float32 now, while grad is fp16 when forward runs with auto_cast() - # thus, the mm operation raise errors because of the dtype of inputs are inconsistent before. - return grad.mm(b.t()), a.t().mm(grad) - - x = paddle.rand([10, 10]) - y = paddle.rand([10, 10]) - x.stop_gradient = False - y.stop_gradient = False - - # with paddle.amp.auto_cast(): - res = MyMM.apply(x, y) - loss = paddle.mean(res) - loss.backward() - - -class TestAmpWithHook(unittest.TestCase): - def test_hook_change_dtype(self): - def func_isinstance(): - with paddle.fluid.dygraph.guard(): - v = paddle.rand([3, 3]) - v.stop_gradient = False - - def foo(grad): - print('grad', grad, grad.dtype) # grad's dtype is float32 - res = paddle.mm(grad, grad) # mm runs in fp16 - print('res', res, res.dtype) # res's dtype is float16 - return res - - v.register_hook(foo) - with paddle.amp.auto_cast(): - a = paddle.mm(v, v) - loss = a.sum() - self.assertRaises(RuntimeError, loss.backward) - - func_isinstance() - - def test_hook_change_place(self): - def func_isinstance(): - with paddle.fluid.dygraph.guard(): - v = paddle.rand([3, 3]) - v.stop_gradient = False - - def foo(grad): - res = grad.cpu() # change place - return res - - v.register_hook(foo) - with paddle.amp.auto_cast(): - a = paddle.mm(v, v) - loss = a.sum() - self.assertRaises(RuntimeError, loss.backward) - - func_isinstance() - - -if __name__ == '__main__': - unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py index b5d36dfebaa..9be07f9279d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py @@ -13,9 +13,6 @@ # limitations under the License. import os - -os.environ['FLAGS_enable_eager_mode'] = '1' - import tempfile import unittest diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py index a583afda701..6ff1d0c7ed3 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -25,5 +24,4 @@ class TestParallelClassCenterSample(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py index aeee6a46676..f4491d26fdf 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -25,5 +24,4 @@ class TestModelParallelLayer(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py index e6467e07815..e06d85fc841 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -27,5 +26,4 @@ class TestDataParallelLayer(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py index 924ccad4279..8341cfef432 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py @@ -73,5 +73,4 @@ class TestHybridPipeParallel(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py index edad4f18a2a..821e3ce987b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -34,5 +33,4 @@ class TestHybridPipeParallelWithVirtualStage(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py index 6cc7cf5ab92..8fb9955c584 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py @@ -78,9 +78,6 @@ def start_local_trainers( "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } - if not eager_mode: - proc_env["FLAGS_enable_eager_mode"] = "%d" % 0 - current_env.update(proc_env) print("trainer proc env:{}".format(current_env)) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py index d0b17e86257..fcfaf72c449 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -29,5 +28,4 @@ class TestHybridParallel(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py index c01d6ed6f08..575ff563503 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -41,5 +40,4 @@ class TestHybridParallel(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py index b4f7cfb39d8..c53b73d5ae3 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -27,5 +26,4 @@ class TestParallelMarginSoftmaxWithCrossEntropy(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv b/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv index 4dfe1c35e8e..954e6b7b42a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv +++ b/python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv @@ -4,7 +4,6 @@ test_fleet_static_mp_layers,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_p test_dgc_op,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC test_dgc_optimizer,,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC test_parallel_margin_cross_entropy,,GPU,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL -test_dygraph_sharding_stage3,,GPU,350,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL test_parallel_dygraph_transformer,,GPU,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL;${NCCL_VERSION} VERSION_GREATER_EQUAL 2212 test_parallel_dygraph_transformer,,ROCM,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_fp16_allreduce_meta_optimizer,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., @@ -34,7 +33,6 @@ test_communicator_sync,,,,,test_runner.py,2,,FLAGS_communicator_send_queue_size= test_fleet_pipeline_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_gradient_merge_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_amp_init,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., -test_dygraph_sharding_optimizer_stage2,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_meta_optimizer_base,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_raw_program_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_parallel_dygraph_sharding_parallel,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., @@ -57,7 +55,6 @@ test_parallel_dygraph_sparse_embedding_over_height,,ROCM,350,DIST,../../dist_tes test_distributed_strategy,LINUX;APPLE,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_auto_parallel_parallelizer,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_recompute_meta_optimizer,LINUX;WIN32,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., -test_dygraph_group_sharded_api,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_fleet_private_function,LINUX;WIN32,,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_new_group,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_new_group.sh,2,,http_proxy=;https_proxy=, test_c_comm_init_op,LINUX,GPU;XPU;ASCEND;ASCEND_CL,120,DIST,test_c_comm_init_op.sh,2,,http_proxy=;https_proxy=, @@ -65,10 +62,8 @@ test_ir_pass_pipeline,,,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=; test_parallel_dygraph_mnist,,GPU;ROCM,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_parallel_dygraph_se_resnext,,GPU;ROCM,200,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_parallel_dygraph_sync_batch_norm,,GPU;ROCM,120,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., -test_imperative_auto_mixed_precision,,GPU;ROCM,300,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_imperative_auto_mixed_precision_for_eager,,GPU;ROCM,300,DIST,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_mixed_precision,,GPU;ROCM,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., -test_dygraph_recompute,,GPU;ROCM,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_dygraph_recompute_for_eager,,GPU;ROCM,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_dist_mnist_dgc_nccl,,,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL OR WITH_RCCL;WITH_DGC test_dist_se_resnext_dgc,,,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL OR WITH_RCCL;WITH_DGC diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py b/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py index 7d438cddf3b..64fdd36a037 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py @@ -77,10 +77,6 @@ class TestDistBase(unittest.TestCase): required_envs["GLOG_logtostderr"] = "1" required_envs["GLOO_LOG_LEVEL"] = "TRACE" - if eager_mode: - required_envs["FLAGS_enable_eager_mode"] = "%d" % 1 - else: - required_envs["FLAGS_enable_eager_mode"] = "%d" % 0 self._run_cluster(model_file, required_envs) def _run_cluster(self, model_file, envs): diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py index 6915f07e8fa..741fd49bd59 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -24,5 +23,4 @@ class TestCollectiveAllToAllSingle(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py b/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py index 892e813660e..77bd7cdbea8 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -24,5 +23,4 @@ class TestCollectiveBatchIsendIrecv(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py index 634a7ecee0c..0e6ef3f8469 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from test_parallel_dygraph_dataparallel import TestMultipleGpus @@ -24,5 +23,4 @@ class TestCollectiveReduceScatter(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/CMakeLists.txt b/python/paddle/fluid/tests/unittests/dygraph_to_static/CMakeLists.txt index dba92870d3a..e1e14bd75c9 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/CMakeLists.txt @@ -4,7 +4,7 @@ file( "test_*.py") string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") set(GC_ENVS FLAGS_eager_delete_tensor_gb=0.0) -set(DY2ST_EAGER_TEST_ENVS ${GC_ENVS} FLAGS_enable_eager_mode=1) +set(DY2ST_EAGER_TEST_ENVS ${GC_ENVS}) set(TEST_EAGER_OPS test_bmn @@ -35,7 +35,7 @@ list(REMOVE_ITEM TEST_OPS test_lac) # NOTE(Aurelius84): In case of Windows CI, if open ON_INFER, RWLOCK of Scope # will be removed and will cause some random failed in multi-thread. if(WITH_PYTHON) - py_test_modules(test_lac MODULES test_lac ENVS FLAGS_enable_eager_mode=1) + py_test_modules(test_lac MODULES test_lac) set_tests_properties(test_lac PROPERTIES TIMEOUT 120) endif() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py index 65fe63e523f..134274a9ed4 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - -os.environ["FLAGS_enable_eager_mode"] = "0" import math +import os import tempfile import time import unittest diff --git a/python/paddle/fluid/tests/unittests/test_collective_api_base.py b/python/paddle/fluid/tests/unittests/test_collective_api_base.py index 9a53ab77622..89fed82b87c 100644 --- a/python/paddle/fluid/tests/unittests/test_collective_api_base.py +++ b/python/paddle/fluid/tests/unittests/test_collective_api_base.py @@ -327,11 +327,6 @@ class TestDistBase(unittest.TestCase): 'NVIDIA_TF32_OVERRIDE', '' ) - if eager_mode: - required_envs["FLAGS_enable_eager_mode"] = "%d" % 1 - else: - required_envs["FLAGS_enable_eager_mode"] = "%d" % 0 - tr0_out, tr1_out, pid0, pid1 = self._run_cluster( model_file, required_envs ) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py index 5fd7f3beb11..8a460152f95 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py @@ -124,9 +124,6 @@ def start_local_trainers( "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } - if not eager_mode: - proc_env["FLAGS_enable_eager_mode"] = "%d" % 0 - proc_env["FLAGS_allocator_strategy"] = allocator_strategy if allocator_strategy == "auto_growth": proc_env["FLAGS_fraction_of_gpu_memory_to_use"] = "0.1" @@ -241,5 +238,4 @@ class TestGradientCheckInEagerMode(TestMultipleGpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_parallel_dygraph_dataparallel.py b/python/paddle/fluid/tests/unittests/xpu/test_parallel_dygraph_dataparallel.py index 8bca3300c8a..bafc523983b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_parallel_dygraph_dataparallel.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_parallel_dygraph_dataparallel.py @@ -80,9 +80,6 @@ def start_local_trainers( "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } - if not eager_mode: - proc_env["FLAGS_enable_eager_mode"] = "%d" % 0 - current_env.update(proc_env) print("trainer proc env:{}".format(current_env)) @@ -152,7 +149,6 @@ class TestGradientCheckInEagerMode(TestMultipleXpus): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" os.environ["BKCL_PCIE_RING"] = "1" os.environ["BKCL_CCIX_RING"] = "0" unittest.main() diff --git a/python/paddle/jit/api.py b/python/paddle/jit/api.py index b6ec7012634..30dc5fd1d6f 100644 --- a/python/paddle/jit/api.py +++ b/python/paddle/jit/api.py @@ -1558,8 +1558,6 @@ class TracedLayer: Examples: .. code-block:: python: - import os - os.environ['FLAGS_enable_eager_mode'] = '0' import paddle class ExampleLayer(paddle.nn.Layer): @@ -1610,8 +1608,6 @@ class TracedLayer: Examples: .. code-block:: python: - import os - os.environ['FLAGS_enable_eager_mode'] = '0' import paddle class ExampleLayer(paddle.nn.Layer): @@ -1716,8 +1712,6 @@ class TracedLayer: Examples: .. code-block:: python: - import os - os.environ['FLAGS_enable_eager_mode'] = '0' import numpy as np import paddle diff --git a/python/paddle/tests/test_dist_hapi_model.py b/python/paddle/tests/test_dist_hapi_model.py index e2696dc986a..4d759f97207 100644 --- a/python/paddle/tests/test_dist_hapi_model.py +++ b/python/paddle/tests/test_dist_hapi_model.py @@ -78,9 +78,6 @@ def start_local_trainers( "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } - if not eager_mode: - proc_env["FLAGS_enable_eager_mode"] = "%d" % 0 - current_env.update(proc_env) print("trainer proc env:{}".format(current_env)) @@ -148,5 +145,4 @@ class TestMultipleGpus(unittest.TestCase): if __name__ == "__main__": - os.environ["FLAGS_enable_eager_mode"] = "1" unittest.main() diff --git a/tools/final_ut_parallel_rule.py b/tools/final_ut_parallel_rule.py index ae91a89af86..b069f64b92c 100644 --- a/tools/final_ut_parallel_rule.py +++ b/tools/final_ut_parallel_rule.py @@ -47,7 +47,6 @@ def classify_cases_by_mem(rootPath): case_exec_200 = [ 'test_post_training_quantization_mnist', - 'test_imperative_auto_mixed_precision', 'test_trt_dynamic_shape_ernie_fp16_ser_deser', 'test_trt_dynamic_shape_ernie', 'test_layer_norm_op', diff --git a/tools/parallel_UT_rule.py b/tools/parallel_UT_rule.py index c99e6a2ae54..e65c8234423 100755 --- a/tools/parallel_UT_rule.py +++ b/tools/parallel_UT_rule.py @@ -1033,7 +1033,6 @@ FOURTH_HIGH_PARALLEL_JOB_NEW = [ 'test_elementwise_div_grad_grad', 'test_minus_op', 'test_shard_index_op', - 'test_dygraph_recompute', 'test_momentum_op', 'test_modelaverage', 'test_compare_reduce_op', @@ -2406,7 +2405,6 @@ TETRAD_PARALLEL_JOB = [ 'test_matrix_rank_op', 'test_margin_cross_entropy_op', 'test_elementwise_pow_op', - 'test_dygraph_recompute', 'test_qr_op', 'test_dygraph_spectral_norm', 'test_cumsum_op', diff --git a/tools/windows/run_unittests.sh b/tools/windows/run_unittests.sh index 6f2e84f427a..8c220fde462 100644 --- a/tools/windows/run_unittests.sh +++ b/tools/windows/run_unittests.sh @@ -239,7 +239,6 @@ long_time_test="^test_gru_op$|\ ^test_sequence_conv$|\ ^test_sgd_op$|\ ^test_transformer$|\ -^test_imperative_auto_mixed_precision$|\ ^test_trt_matmul_quant_dequant$|\ ^test_strided_slice_op$" -- GitLab