From d7e74e634de7ec122eeafbaedf87ac48375896d7 Mon Sep 17 00:00:00 2001 From: Thunderbrook <52529258+Thunderbrook@users.noreply.github.com> Date: Wed, 14 Sep 2022 13:37:45 +0800 Subject: [PATCH] remove unitest (#45503) --- .../fluid/tests/unittests/CMakeLists.txt | 6 - .../unittests/test_dist_fleet_grad_clip.py | 144 ------------------ .../unittests/test_dist_fleet_heter_ctr.py | 61 -------- .../unittests/test_dist_fleet_ps_gpu_ctr.py | 60 -------- 4 files changed, 271 deletions(-) delete mode 100644 python/paddle/fluid/tests/unittests/test_dist_fleet_grad_clip.py delete mode 100644 python/paddle/fluid/tests/unittests/test_dist_fleet_heter_ctr.py delete mode 100644 python/paddle/fluid/tests/unittests/test_dist_fleet_ps_gpu_ctr.py diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index a76b9d1789b..58debc39dd9 100755 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -18,9 +18,6 @@ string(REPLACE ".py" "" DIST_TEST_OPS "${DIST_TEST_OPS}") if((NOT WITH_GPU) AND (NOT WITH_XPU) AND NOT (WITH_ASCEND OR WITH_ASCEND_CL)) - list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_grad_clip") - list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_heter_ctr") - list(REMOVE_ITEM DIST_TEST_OPS "test_dist_fleet_ps_gpu_ctr") list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_batch_merge") endif() list(APPEND DIST_TEST_OPS test_parallel_dygraph_dataparallel) @@ -527,9 +524,6 @@ if((NOT WITH_GPU) AND (NOT WITH_XPU) AND NOT (WITH_ASCEND OR WITH_ASCEND_CL)) list(REMOVE_ITEM TEST_OPS "test_fleet_graph_execution_meta_optimizer") - list(REMOVE_ITEM TEST_OPS "test_dist_fleet_grad_clip") - list(REMOVE_ITEM TEST_OPS "test_dist_fleet_heter_ctr") - list(REMOVE_ITEM TEST_OPS "test_dist_fleet_ps_gpu_ctr") list(REMOVE_ITEM TEST_OPS "test_dist_mnist_batch_merge") endif() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_grad_clip.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_grad_clip.py deleted file mode 100644 index f929bc2ae1e..00000000000 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_grad_clip.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import os -import unittest -from test_dist_fleet_base import TestFleetBase - - -class TestDistGeoClipByGlobalNorm(TestFleetBase): - - def _setup_config(self): - self._mode = "geo" - self._reader = "dataset" - self._geo_sgd_need_push_nums = 5 - self._grad_clip_mode = 3 - - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): - required_envs = { - "PATH": os.getenv("PATH", ""), - "PYTHONPATH": os.getenv("PYTHONPATH", ""), - "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), - "FLAGS_rpc_deadline": "5000", # 5sec to fail fast - "http_proxy": "" - } - required_envs.update(need_envs) - - tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) - - def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=True) - - def _setup_config1(self): - self._sync_mode = False - self._grad_clip_mode = 2 - - -class TestDistASyncClipByValue(TestFleetBase): - - def _setup_config(self): - self._mode = "async" - self._reader = "dataset" - self._grad_clip_mode = 1 - - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): - required_envs = { - "PATH": os.getenv("PATH", ""), - "PYTHONPATH": os.getenv("PYTHONPATH", ""), - "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), - "FLAGS_rpc_deadline": "5000", # 5sec to fail fast - "http_proxy": "" - } - required_envs.update(need_envs) - - tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) - - def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=True) - - -class TestDistASyncClipByNorm(TestFleetBase): - - def _setup_config(self): - self._mode = "async" - self._reader = "dataset" - self._grad_clip_mode = 2 - - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): - required_envs = { - "PATH": os.getenv("PATH", ""), - "PYTHONPATH": os.getenv("PYTHONPATH", ""), - "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), - "FLAGS_rpc_deadline": "5000", # 5sec to fail fast - "http_proxy": "" - } - required_envs.update(need_envs) - - tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) - - def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=True) - - -class TestDistASyncClipByGlobalNorm(TestFleetBase): - - def _setup_config(self): - self._mode = "async" - self._reader = "dataset" - self._grad_clip_mode = 3 - - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): - required_envs = { - "PATH": os.getenv("PATH", ""), - "PYTHONPATH": os.getenv("PYTHONPATH", ""), - "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), - "FLAGS_rpc_deadline": "5000", # 5sec to fail fast - "http_proxy": "" - } - required_envs.update(need_envs) - - tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) - - def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=True) - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_ctr.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_ctr.py deleted file mode 100644 index eaae0eff55a..00000000000 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_ctr.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import os -import unittest -import tempfile -from test_dist_fleet_heter_base import TestFleetHeterBase -import paddle - -paddle.enable_static() - - -class TestDistHeterDatasetAsync2x2(TestFleetHeterBase): - - def _setup_config(self): - self._mode = "async" - self._reader = "dataset" - - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): - required_envs = { - "PATH": os.getenv("PATH", ""), - "PYTHONPATH": os.getenv("PYTHONPATH", ""), - "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), - "FLAGS_rpc_deadline": "5000", # 5sec to fail fast - "http_proxy": "", - "CPU_NUM": "2" - } - - required_envs.update(need_envs) - - if check_error_log: - required_envs["GLOG_v"] = "4" - required_envs["GLOG_logtostderr"] = "1" - - tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) - - def test_dist_train(self): - self.check_with_place("dist_fleet_heter_pipeline_ctr.py", - delta=1e-5, - check_error_log=True) - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps_gpu_ctr.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps_gpu_ctr.py deleted file mode 100644 index d29ea0daad6..00000000000 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps_gpu_ctr.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import os -import unittest -import tempfile -from test_dist_fleet_base import TestFleetBase - - -class TestPsGPUAsyncDataset2x2(TestFleetBase): - - def _setup_config(self): - self._mode = "async" - self._reader = "dataset" - - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): - required_envs = { - "PATH": os.getenv("PATH", ""), - "PYTHONPATH": os.getenv("PYTHONPATH", ""), - "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), - "FLAGS_rpc_deadline": "5000", # 5sec to fail fast - "http_proxy": "", - "SAVE_MODEL": "1", - "Debug": "1", - "DEVICE": "gpu" - } - - required_envs.update(need_envs) - - if check_error_log: - required_envs["GLOG_v"] = "3" - required_envs["GLOG_logtostderr"] = "1" - - tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) - - def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=True) - - -if __name__ == '__main__': - unittest.main() -- GitLab