From 6822307745ebb65cf7e28110e49870bfdcbd7bfb Mon Sep 17 00:00:00 2001 From: 123malin Date: Mon, 28 Sep 2020 13:26:48 +0800 Subject: [PATCH] test=develop, rm netifaces (#27581) * test=develop, rm netifaces --- .../distributed/fleet/base/role_maker.py | 35 +++++++++++-------- .../fluid/incubate/fleet/base/role_maker.py | 35 +++++++++++-------- .../fluid/tests/unittests/test_fleet.py | 6 +--- .../tests/unittests/test_fleet_nocvm_1.py | 6 +--- .../tests/unittests/test_fleet_rolemaker.py | 6 +--- .../tests/unittests/test_fleet_rolemaker_2.py | 6 +--- .../tests/unittests/test_fleet_rolemaker_3.py | 6 +--- .../tests/unittests/test_fleet_rolemaker_4.py | 6 ---- .../unittests/test_fleet_rolemaker_new.py | 28 --------------- .../unittests/test_fleet_unitaccessor.py | 6 +--- .../fluid/tests/unittests/test_fleet_util.py | 5 --- python/requirements.txt | 2 -- 12 files changed, 48 insertions(+), 99 deletions(-) diff --git a/python/paddle/distributed/fleet/base/role_maker.py b/python/paddle/distributed/fleet/base/role_maker.py index f78b0b4e06..94ef61ef91 100644 --- a/python/paddle/distributed/fleet/base/role_maker.py +++ b/python/paddle/distributed/fleet/base/role_maker.py @@ -253,26 +253,33 @@ class Gloo(object): """ get default physical interface """ - import netifaces - gateways = netifaces.gateways() - if gateways.get(netifaces.AF_INET) != None: - gateway = gateways[netifaces.AF_INET] - if len(gateway) > 0 and len(gateway[0]) > 1: - return gateway[0][1] + res = os.popen("route -A inet").read().strip().split("\n") + + gateway_idx = None + iface_idx = None + for item in res: + item = item.split() + if "Gateway" in item and "Iface" in item: + gateway_idx = item.index("Gateway") + iface_idx = item.index("Iface") + elif gateway_idx != None and iface_idx != None: + gateway = None + if len(item) > gateway_idx: + gateway = item[gateway_idx] + if gateway and gateway != '*' and gateway != "0.0.0.0" and len( + item) > iface_idx: + return item[iface_idx] return "lo" def __get_default_iface_from_interfaces(self): """ get default physical interface """ - import netifaces - for intf_name in netifaces.interfaces(): - addresses = netifaces.ifaddresses(intf_name) - if netifaces.AF_INET in addresses: - ipv4_addresses = addresses[netifaces.AF_INET] - for ipv4_address in ipv4_addresses: - if 'broadcast' in ipv4_address: - return intf_name + res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split( + "\n") + for item in res: + if "BROADCAST" in item: + return item.split(":")[1].strip() return "lo" def barrier(self, comm_world): diff --git a/python/paddle/fluid/incubate/fleet/base/role_maker.py b/python/paddle/fluid/incubate/fleet/base/role_maker.py index 250ee165bb..6db2e65bcf 100644 --- a/python/paddle/fluid/incubate/fleet/base/role_maker.py +++ b/python/paddle/fluid/incubate/fleet/base/role_maker.py @@ -955,26 +955,33 @@ class GeneralRoleMaker(RoleMakerBase): """ get default physical interface """ - import netifaces - gateways = netifaces.gateways() - if gateways.get(netifaces.AF_INET) != None: - gateway = gateways[netifaces.AF_INET] - if len(gateway) > 0 and len(gateway[0]) > 1: - return gateway[0][1] + res = os.popen("route -A inet").read().strip().split("\n") + + gateway_idx = None + iface_idx = None + for item in res: + item = item.split() + if "Gateway" in item and "Iface" in item: + gateway_idx = item.index("Gateway") + iface_idx = item.index("Iface") + elif gateway_idx != None and iface_idx != None: + gateway = None + if len(item) > gateway_idx: + gateway = item[gateway_idx] + if gateway and gateway != '*' and gateway != "0.0.0.0" and len( + item) > iface_idx: + return item[iface_idx] return "lo" def __get_default_iface_from_interfaces(self): """ get default physical interface """ - import netifaces - for intf_name in netifaces.interfaces(): - addresses = netifaces.ifaddresses(intf_name) - if netifaces.AF_INET in addresses: - ipv4_addresses = addresses[netifaces.AF_INET] - for ipv4_address in ipv4_addresses: - if 'broadcast' in ipv4_address: - return intf_name + res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split( + "\n") + for item in res: + if "BROADCAST" in item: + return item.split(":")[1].strip() return "lo" def __start_kv_server(self, http_server_d, size_d): diff --git a/python/paddle/fluid/tests/unittests/test_fleet.py b/python/paddle/fluid/tests/unittests/test_fleet.py index a705d5ee66..35969179a3 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet.py +++ b/python/paddle/fluid/tests/unittests/test_fleet.py @@ -37,11 +37,7 @@ class TestFleet1(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py b/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py index b2b6136797..e6138296a6 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py @@ -36,11 +36,7 @@ class TestFleet1(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py index 7f1ad5d52d..4655b628da 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py @@ -64,11 +64,7 @@ class TestCloudRoleMaker(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py index dae7907161..e87d52752c 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py @@ -36,11 +36,7 @@ class TestCloudRoleMaker2(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import RoleMakerBase - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_2") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py index 0fa852eeee..396d705508 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py @@ -36,11 +36,7 @@ class TestCloudRoleMaker(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py index 7d8a71a236..6ce1eabe24 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py @@ -47,12 +47,6 @@ class TestCloudRoleMaker(unittest.TestCase): print("warning: no fleet, skip test_pslib_4") return - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_4") - return - class FakeStream(): """ it is a fake stream only for test. diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py index 992fbbbe26..26a56fb08b 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py @@ -70,12 +70,6 @@ class TestCloudRoleMaker(unittest.TestCase): os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["PADDLE_TRAINER_ID"] = "0" - try: - import netifaces - except: - print("warning: no netifaces, skip test_tr_rolemaker") - return - ro = role_maker.PaddleCloudRoleMaker(is_collective=False) self.assertTrue(ro._is_worker()) ro = role_maker.PaddleCloudRoleMaker(is_collective=False) @@ -110,12 +104,6 @@ class TestCloudRoleMaker(unittest.TestCase): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" - try: - import netifaces - except: - print("warning: no netifaces, skip test_ps_rolemaker") - return - ro = role_maker.PaddleCloudRoleMaker( is_collective=False, init_gloo=False) self.assertEqual(ro._server_index(), 0) @@ -131,11 +119,6 @@ class TestCloudRoleMaker(unittest.TestCase): def test_traing_role(self): """Test training role.""" os.environ["TRAINING_ROLE"] = "TEST" - try: - import netifaces - except: - print("warning: no netifaces, skip test_training_role") - return ro = role_maker.PaddleCloudRoleMaker(is_collective=False) self.assertRaises(ValueError, ro._generate_role) @@ -150,11 +133,6 @@ class TestUserDefinedRoleMaker(unittest.TestCase): pass def test_ps_rolemaker(self): - try: - import netifaces - except: - print("warning: no netifaces, skip test_ps_rolemaker") - return ro = role_maker.UserDefinedRoleMaker( is_collective=False, @@ -169,12 +147,6 @@ class TestUserDefinedRoleMaker(unittest.TestCase): self.assertEqual(ro._role_id(), 0) def test_tr_rolemaker(self): - try: - import netifaces - except: - print("warning: no netifaces, skip test_tr_rolemaker") - return - ro = role_maker.UserDefinedRoleMaker( is_collective=False, init_gloo=False, diff --git a/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py b/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py index 7a255e5da1..286fcc03c6 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py @@ -36,11 +36,7 @@ class TestFleet1(unittest.TestCase): from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker - try: - import netifaces - except: - print("warning: no netifaces, skip test_pslib_1") - return + os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_util.py b/python/paddle/fluid/tests/unittests/test_fleet_util.py index b5c22b192a..a3a526aaa6 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_util.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_util.py @@ -100,11 +100,6 @@ class TestFleetUtil(unittest.TestCase): def test_get_file_shard(self): import paddle.distributed.fleet as fleet self.assertRaises(Exception, fleet.util.get_file_shard, "files") - try: - import netifaces - except: - print("warning: no netifaces, skip test_get_file_shard") - return role = role_maker.UserDefinedRoleMaker( is_collective=False, diff --git a/python/requirements.txt b/python/requirements.txt index 6a88d61a94..1f62139a9a 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -17,5 +17,3 @@ decorator prettytable astor pathlib -netifaces ; platform_system != "Windows" -netifaces ; python_version>="3.5" and platform_system == "Windows" -- GitLab