From 3b90e67eb3a73098cc294cc8d9f7f4d6b8552029 Mon Sep 17 00:00:00 2001 From: malin10 Date: Sat, 26 Sep 2020 18:05:17 +0800 Subject: [PATCH] test=develop, rm netifaces --- Dockerfile | 1 - .../distributed/fleet/base/role_maker.py | 35 +++++++++++-------- .../fluid/incubate/fleet/base/role_maker.py | 35 +++++++++++-------- .../fluid/tests/unittests/test_fleet.py | 6 +--- .../tests/unittests/test_fleet_nocvm_1.py | 6 +--- .../tests/unittests/test_fleet_rolemaker.py | 6 +--- .../tests/unittests/test_fleet_rolemaker_2.py | 6 +--- .../tests/unittests/test_fleet_rolemaker_3.py | 6 +--- .../tests/unittests/test_fleet_rolemaker_4.py | 6 ---- .../unittests/test_fleet_rolemaker_new.py | 28 --------------- .../unittests/test_fleet_unitaccessor.py | 6 +--- .../fluid/tests/unittests/test_fleet_util.py | 5 --- python/requirements.txt | 2 -- tools/dockerfile/Dockerfile.ubuntu | 1 - .../Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 | 1 - .../Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 | 1 - 16 files changed, 48 insertions(+), 103 deletions(-) diff --git a/Dockerfile b/Dockerfile index b92ac228a8d..a49373eaf0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -207,7 +207,6 @@ RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \ rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \ apt-get install libprotobuf-dev -y -RUN pip --no-cache-dir install -U netifaces==0.10.9 # Older versions of patchelf limited the size of the files being processed and were fixed in this pr. # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa diff --git a/python/paddle/distributed/fleet/base/role_maker.py b/python/paddle/distributed/fleet/base/role_maker.py index f66f013e4db..20356c2b49b 100644 --- a/python/paddle/distributed/fleet/base/role_maker.py +++ b/python/paddle/distributed/fleet/base/role_maker.py @@ -254,26 +254,33 @@ class Gloo(object): """ get default physical interface """ - import netifaces - gateways = netifaces.gateways() - if gateways.get(netifaces.AF_INET) != None: - gateway = gateways[netifaces.AF_INET] - if len(gateway) > 0 and len(gateway[0]) > 1: - return gateway[0][1] + res = os.popen("route -A inet").read().strip().split("\n") + + gateway_idx = None + iface_idx = None + for item in res: + item = item.split() + if "Gateway" in item and "Iface" in item: + gateway_idx = item.index("Gateway") + iface_idx = item.index("Iface") + elif gateway_idx != None and iface_idx != None: + gateway = None + if len(item) > gateway_idx: + gateway = item[gateway_idx] + if gateway and gateway != '*' and gateway != "0.0.0.0" and len( + item) > iface_idx: + return item[iface_idx] return "lo" def __get_default_iface_from_interfaces(self): """ get default physical interface """ - import netifaces - for intf_name in netifaces.interfaces(): - addresses = netifaces.ifaddresses(intf_name) - if netifaces.AF_INET in addresses: - ipv4_addresses = addresses[netifaces.AF_INET] - for ipv4_address in ipv4_addresses: - if 'broadcast' in ipv4_address: - return intf_name + res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split( + "\n") + for item in res: + if "BROADCAST" in item: + return item.split(":")[1].strip() return "lo" def barrier(self, comm_world): diff --git a/python/paddle/fluid/incubate/fleet/base/role_maker.py b/python/paddle/fluid/incubate/fleet/base/role_maker.py index be27a7c5214..8fdfc773ce7 100644 --- a/python/paddle/fluid/incubate/fleet/base/role_maker.py +++ b/python/paddle/fluid/incubate/fleet/base/role_maker.py @@ -955,26 +955,33 @@ class GeneralRoleMaker(RoleMakerBase): """ get default physical interface """ - import netifaces - gateways = netifaces.gateways() - if gateways.get(netifaces.AF_INET) != None: - gateway = gateways[netifaces.AF_INET] - if len(gateway) > 0 and len(gateway[0]) > 1: - return gateway[0][1] + res = os.popen("route -A inet").read().strip().split("\n") + + gateway_idx = None + iface_idx = None + for item in res: + item = item.split() + if "Gateway" in item and "Iface" in item: + gateway_idx = item.index("Gateway") + iface_idx = item.index("Iface") + elif gateway_idx != None and iface_idx != None: + gateway = None + if len(item) > gateway_idx: + gateway = item[gateway_idx] + if gateway and gateway != '*' and gateway != "0.0.0.0" and len( + item) > iface_idx: + return item[iface_idx] return "lo" def __get_default_iface_from_interfaces(self): """ get default physical interface """ - import netifaces - for intf_name in netifaces.interfaces(): - addresses = netifaces.ifaddresses(intf_name) - if netifaces.AF_INET in addresses: - ipv4_addresses = addresses[netifaces.AF_INET] - for ipv4_address in ipv4_addresses: - if 'broadcast' in ipv4_address: - return intf_name + res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split( + "\n") + for item in res: + if "BROADCAST" in item: + return item.split(":")[1].strip() return "lo" def __start_kv_server(self, http_server_d, size_d): diff --git a/python/paddle/fluid/tests/unittests/test_fleet.py b/python/paddle/fluid/tests/unittests/test_fleet.py index a705d5ee661..35969179a39 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet.py +++ b/python/paddle/fluid/tests/unittests/test_fleet.py @@ -37,11 +37,7 @@ class TestFleet1(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py b/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py index b2b6136797b..e6138296a6c 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py @@ -36,11 +36,7 @@ class TestFleet1(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py index 7f1ad5d52d8..4655b628dab 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py @@ -64,11 +64,7 @@ class TestCloudRoleMaker(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py index dae79071616..e87d52752c8 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py @@ -36,11 +36,7 @@ class TestCloudRoleMaker2(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import RoleMakerBase - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_2") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py index 0fa852eeeeb..396d705508b 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py @@ -36,11 +36,7 @@ class TestCloudRoleMaker(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py index 6cb40eef27e..2db4fdeef4d 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py @@ -47,12 +47,6 @@ class TestCloudRoleMaker(unittest.TestCase): print("warning: no fleet, skip test_pslib_4") return - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_4") - return - class FakeStream(): """ it is a fake stream only for test. diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py index 992fbbbe266..26a56fb08b4 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py @@ -70,12 +70,6 @@ class TestCloudRoleMaker(unittest.TestCase): os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["PADDLE_TRAINER_ID"] = "0" - try: - import netifaces - except: - print("warning: no netifaces, skip test_tr_rolemaker") - return - ro = role_maker.PaddleCloudRoleMaker(is_collective=False) self.assertTrue(ro._is_worker()) ro = role_maker.PaddleCloudRoleMaker(is_collective=False) @@ -110,12 +104,6 @@ class TestCloudRoleMaker(unittest.TestCase): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" - try: - import netifaces - except: - print("warning: no netifaces, skip test_ps_rolemaker") - return - ro = role_maker.PaddleCloudRoleMaker( is_collective=False, init_gloo=False) self.assertEqual(ro._server_index(), 0) @@ -131,11 +119,6 @@ class TestCloudRoleMaker(unittest.TestCase): def test_traing_role(self): """Test training role.""" os.environ["TRAINING_ROLE"] = "TEST" - try: - import netifaces - except: - print("warning: no netifaces, skip test_training_role") - return ro = role_maker.PaddleCloudRoleMaker(is_collective=False) self.assertRaises(ValueError, ro._generate_role) @@ -150,11 +133,6 @@ class TestUserDefinedRoleMaker(unittest.TestCase): pass def test_ps_rolemaker(self): - try: - import netifaces - except: - print("warning: no netifaces, skip test_ps_rolemaker") - return ro = role_maker.UserDefinedRoleMaker( is_collective=False, @@ -169,12 +147,6 @@ class TestUserDefinedRoleMaker(unittest.TestCase): self.assertEqual(ro._role_id(), 0) def test_tr_rolemaker(self): - try: - import netifaces - except: - print("warning: no netifaces, skip test_tr_rolemaker") - return - ro = role_maker.UserDefinedRoleMaker( is_collective=False, init_gloo=False, diff --git a/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py b/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py index 7a255e5da14..286fcc03c67 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py @@ -36,11 +36,7 @@ class TestFleet1(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_util.py b/python/paddle/fluid/tests/unittests/test_fleet_util.py index b5c22b192a1..a3a526aaa61 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_util.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_util.py @@ -100,11 +100,6 @@ class TestFleetUtil(unittest.TestCase): def test_get_file_shard(self): import paddle.distributed.fleet as fleet self.assertRaises(Exception, fleet.util.get_file_shard, "files") - try: - import netifaces - except: - print("warning: no netifaces, skip test_get_file_shard") - return role = role_maker.UserDefinedRoleMaker( is_collective=False, diff --git a/python/requirements.txt b/python/requirements.txt index 6a88d61a94c..1f62139a9a6 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -17,5 +17,3 @@ decorator prettytable astor pathlib -netifaces ; platform_system != "Windows" -netifaces ; python_version>="3.5" and platform_system == "Windows" diff --git a/tools/dockerfile/Dockerfile.ubuntu b/tools/dockerfile/Dockerfile.ubuntu index 9fe58885fa5..927b0d0b8b1 100644 --- a/tools/dockerfile/Dockerfile.ubuntu +++ b/tools/dockerfile/Dockerfile.ubuntu @@ -207,7 +207,6 @@ RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/ cd binutils-2.27 && \ ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz -RUN pip --no-cache-dir install -U netifaces==0.10.9 # Older versions of patchelf limited the size of the files being processed and were fixed in this pr. # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa diff --git a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 index e3a3374b943..b140d42ed9c 100644 --- a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 +++ b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 @@ -212,7 +212,6 @@ RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \ rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \ apt-get install libprotobuf-dev -y -RUN pip --no-cache-dir install -U netifaces==0.10.9 # ccache 3.7.9 RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ diff --git a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 index c27fdcea240..ac71b202730 100644 --- a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 +++ b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 @@ -221,7 +221,6 @@ RUN wget -q https://paddle-ci.gz.bcebos.com/binutils_2.27.orig.tar.gz && \ ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz RUN apt-get install libprotobuf-dev -y -RUN pip --no-cache-dir install -U netifaces==0.10.9 # ccache 3.7.9 RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ -- GitLab