diff --git a/Dockerfile b/Dockerfile index b92ac228a8d50da93f8c0bfe2f6af31fc784f2c7..a49373eaf0c694cef7c289ae634f262334cc46af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -207,7 +207,6 @@ RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \ rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \ apt-get install libprotobuf-dev -y -RUN pip --no-cache-dir install -U netifaces==0.10.9 # Older versions of patchelf limited the size of the files being processed and were fixed in this pr. # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa diff --git a/python/paddle/distributed/fleet/base/role_maker.py b/python/paddle/distributed/fleet/base/role_maker.py index f66f013e4dbaadd534d6859b7ba6530779c82a3b..20356c2b49bac8798e31ed55c347a7a215908fa2 100644 --- a/python/paddle/distributed/fleet/base/role_maker.py +++ b/python/paddle/distributed/fleet/base/role_maker.py @@ -254,26 +254,33 @@ class Gloo(object): """ get default physical interface """ - import netifaces - gateways = netifaces.gateways() - if gateways.get(netifaces.AF_INET) != None: - gateway = gateways[netifaces.AF_INET] - if len(gateway) > 0 and len(gateway[0]) > 1: - return gateway[0][1] + res = os.popen("route -A inet").read().strip().split("\n") + + gateway_idx = None + iface_idx = None + for item in res: + item = item.split() + if "Gateway" in item and "Iface" in item: + gateway_idx = item.index("Gateway") + iface_idx = item.index("Iface") + elif gateway_idx != None and iface_idx != None: + gateway = None + if len(item) > gateway_idx: + gateway = item[gateway_idx] + if gateway and gateway != '*' and gateway != "0.0.0.0" and len( + item) > iface_idx: + return item[iface_idx] return "lo" def __get_default_iface_from_interfaces(self): """ get default physical interface """ - import netifaces - for intf_name in netifaces.interfaces(): - addresses = netifaces.ifaddresses(intf_name) - if netifaces.AF_INET in addresses: - ipv4_addresses = addresses[netifaces.AF_INET] - for ipv4_address in ipv4_addresses: - if 'broadcast' in ipv4_address: - return intf_name + res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split( + "\n") + for item in res: + if "BROADCAST" in item: + return item.split(":")[1].strip() return "lo" def barrier(self, comm_world): diff --git a/python/paddle/fluid/incubate/fleet/base/role_maker.py b/python/paddle/fluid/incubate/fleet/base/role_maker.py index be27a7c5214e6b4b730d14cb4a64118f24506860..8fdfc773ce746b8cb52042ca8cfb6680836e1df6 100644 --- a/python/paddle/fluid/incubate/fleet/base/role_maker.py +++ b/python/paddle/fluid/incubate/fleet/base/role_maker.py @@ -955,26 +955,33 @@ class GeneralRoleMaker(RoleMakerBase): """ get default physical interface """ - import netifaces - gateways = netifaces.gateways() - if gateways.get(netifaces.AF_INET) != None: - gateway = gateways[netifaces.AF_INET] - if len(gateway) > 0 and len(gateway[0]) > 1: - return gateway[0][1] + res = os.popen("route -A inet").read().strip().split("\n") + + gateway_idx = None + iface_idx = None + for item in res: + item = item.split() + if "Gateway" in item and "Iface" in item: + gateway_idx = item.index("Gateway") + iface_idx = item.index("Iface") + elif gateway_idx != None and iface_idx != None: + gateway = None + if len(item) > gateway_idx: + gateway = item[gateway_idx] + if gateway and gateway != '*' and gateway != "0.0.0.0" and len( + item) > iface_idx: + return item[iface_idx] return "lo" def __get_default_iface_from_interfaces(self): """ get default physical interface """ - import netifaces - for intf_name in netifaces.interfaces(): - addresses = netifaces.ifaddresses(intf_name) - if netifaces.AF_INET in addresses: - ipv4_addresses = addresses[netifaces.AF_INET] - for ipv4_address in ipv4_addresses: - if 'broadcast' in ipv4_address: - return intf_name + res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split( + "\n") + for item in res: + if "BROADCAST" in item: + return item.split(":")[1].strip() return "lo" def __start_kv_server(self, http_server_d, size_d): diff --git a/python/paddle/fluid/tests/unittests/test_fleet.py b/python/paddle/fluid/tests/unittests/test_fleet.py index a705d5ee661fd5d0d28b791d6db4624b78281743..35969179a397897af8ba3788f7cf67fa54aadd1b 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet.py +++ b/python/paddle/fluid/tests/unittests/test_fleet.py @@ -37,11 +37,7 @@ class TestFleet1(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py b/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py index b2b6136797ba460f9f829d5df4c7041664b424cb..e6138296a6c337d0f59be7070cddaaaaea84fb03 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py @@ -36,11 +36,7 @@ class TestFleet1(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py index 7f1ad5d52d8f0b5a6b5bd83ea3a158f123e870ea..4655b628dab4d5315073f2006329dc3976f42b76 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py @@ -64,11 +64,7 @@ class TestCloudRoleMaker(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py index dae7907161697107a50eaf1b1501881f74509b76..e87d52752c852d0baef787add54090a24818e68e 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py @@ -36,11 +36,7 @@ class TestCloudRoleMaker2(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import RoleMakerBase - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_2") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py index 0fa852eeeebe9c8fbb056fca388a0af2c8f92842..396d705508b3488dadf059ed4527a06f0d916a44 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py @@ -36,11 +36,7 @@ class TestCloudRoleMaker(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py index 6cb40eef27e4d93606167232dba3fc181af3c17a..2db4fdeef4d6a3a46de4514263c6ce65bffe9fe7 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py @@ -47,12 +47,6 @@ class TestCloudRoleMaker(unittest.TestCase): print("warning: no fleet, skip test_pslib_4") return - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_4") - return - class FakeStream(): """ it is a fake stream only for test. diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py index 992fbbbe2661d7059fc2acccb2b7fda5a357157a..26a56fb08b4fe3ff9fbe9b2b50fe64916bd538d6 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py @@ -70,12 +70,6 @@ class TestCloudRoleMaker(unittest.TestCase): os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["PADDLE_TRAINER_ID"] = "0" - try: - import netifaces - except: - print("warning: no netifaces, skip test_tr_rolemaker") - return - ro = role_maker.PaddleCloudRoleMaker(is_collective=False) self.assertTrue(ro._is_worker()) ro = role_maker.PaddleCloudRoleMaker(is_collective=False) @@ -110,12 +104,6 @@ class TestCloudRoleMaker(unittest.TestCase): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" - try: - import netifaces - except: - print("warning: no netifaces, skip test_ps_rolemaker") - return - ro = role_maker.PaddleCloudRoleMaker( is_collective=False, init_gloo=False) self.assertEqual(ro._server_index(), 0) @@ -131,11 +119,6 @@ class TestCloudRoleMaker(unittest.TestCase): def test_traing_role(self): """Test training role.""" os.environ["TRAINING_ROLE"] = "TEST" - try: - import netifaces - except: - print("warning: no netifaces, skip test_training_role") - return ro = role_maker.PaddleCloudRoleMaker(is_collective=False) self.assertRaises(ValueError, ro._generate_role) @@ -150,11 +133,6 @@ class TestUserDefinedRoleMaker(unittest.TestCase): pass def test_ps_rolemaker(self): - try: - import netifaces - except: - print("warning: no netifaces, skip test_ps_rolemaker") - return ro = role_maker.UserDefinedRoleMaker( is_collective=False, @@ -169,12 +147,6 @@ class TestUserDefinedRoleMaker(unittest.TestCase): self.assertEqual(ro._role_id(), 0) def test_tr_rolemaker(self): - try: - import netifaces - except: - print("warning: no netifaces, skip test_tr_rolemaker") - return - ro = role_maker.UserDefinedRoleMaker( is_collective=False, init_gloo=False, diff --git a/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py b/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py index 7a255e5da14dacc1a5552642640e3ffe1e4eaad4..286fcc03c6717e397a61560702ccabd457d527a4 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py @@ -36,11 +36,7 @@ class TestFleet1(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_util.py b/python/paddle/fluid/tests/unittests/test_fleet_util.py index b5c22b192a16dec43c5e5243c34b970f489f5bb7..a3a526aaa6158ca5174cb8a10400275bab4f7270 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_util.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_util.py @@ -100,11 +100,6 @@ class TestFleetUtil(unittest.TestCase): def test_get_file_shard(self): import paddle.distributed.fleet as fleet self.assertRaises(Exception, fleet.util.get_file_shard, "files") - try: - import netifaces - except: - print("warning: no netifaces, skip test_get_file_shard") - return role = role_maker.UserDefinedRoleMaker( is_collective=False, diff --git a/python/requirements.txt b/python/requirements.txt index 6a88d61a94c182db3320e4de30f45be4c852f70f..1f62139a9a6dcbb7c4f393160aafa0fe89ac02ab 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -17,5 +17,3 @@ decorator prettytable astor pathlib -netifaces ; platform_system != "Windows" -netifaces ; python_version>="3.5" and platform_system == "Windows" diff --git a/tools/dockerfile/Dockerfile.ubuntu b/tools/dockerfile/Dockerfile.ubuntu index 9fe58885fa553671cf5c08bd51295f271f4df668..927b0d0b8b1a2f7cadd99463cc3dddb5c6521925 100644 --- a/tools/dockerfile/Dockerfile.ubuntu +++ b/tools/dockerfile/Dockerfile.ubuntu @@ -207,7 +207,6 @@ RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/ cd binutils-2.27 && \ ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz -RUN pip --no-cache-dir install -U netifaces==0.10.9 # Older versions of patchelf limited the size of the files being processed and were fixed in this pr. # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa diff --git a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 index e3a3374b943bc955d54afbef9755ed5147fad7d2..b140d42ed9c530e97eced1f586b664d45b73fb74 100644 --- a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 +++ b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16 @@ -212,7 +212,6 @@ RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \ rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \ apt-get install libprotobuf-dev -y -RUN pip --no-cache-dir install -U netifaces==0.10.9 # ccache 3.7.9 RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ diff --git a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 index c27fdcea2401c26b1ef1dd377c42930b6e74fcf0..ac71b202730927f157117bf04ef80056ac4d9e03 100644 --- a/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 +++ b/tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16 @@ -221,7 +221,6 @@ RUN wget -q https://paddle-ci.gz.bcebos.com/binutils_2.27.orig.tar.gz && \ ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz RUN apt-get install libprotobuf-dev -y -RUN pip --no-cache-dir install -U netifaces==0.10.9 # ccache 3.7.9 RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \