test=develop, rm netifaces

Dockerfile

@@ -207,7 +207,6 @@ RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz
     export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \
     rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \
     apt-get install libprotobuf-dev -y
-RUN pip --no-cache-dir install -U netifaces==0.10.9
 
 # Older versions of patchelf limited the size of the files being processed and were fixed in this pr.
 # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa

python/paddle/distributed/fleet/base/role_maker.py

@@ -254,26 +254,33 @@ class Gloo(object):
         """
         get default physical interface
         """
-        import netifaces
-        gateways = netifaces.gateways()
-        if gateways.get(netifaces.AF_INET) != None:
-            gateway = gateways[netifaces.AF_INET]
-            if len(gateway) > 0 and len(gateway[0]) > 1:
-                return gateway[0][1]
+        res = os.popen("route -A inet").read().strip().split("\n")
+
+        gateway_idx = None
+        iface_idx = None
+        for item in res:
+            item = item.split()
+            if "Gateway" in item and "Iface" in item:
+                gateway_idx = item.index("Gateway")
+                iface_idx = item.index("Iface")
+            elif gateway_idx != None and iface_idx != None:
+                gateway = None
+                if len(item) > gateway_idx:
+                    gateway = item[gateway_idx]
+                if gateway and gateway != '*' and gateway != "0.0.0.0" and len(
+                        item) > iface_idx:
+                    return item[iface_idx]
         return "lo"
 
     def __get_default_iface_from_interfaces(self):
         """
         get default physical interface
         """
-        import netifaces
-        for intf_name in netifaces.interfaces():
-            addresses = netifaces.ifaddresses(intf_name)
-            if netifaces.AF_INET in addresses:
-                ipv4_addresses = addresses[netifaces.AF_INET]
-                for ipv4_address in ipv4_addresses:
-                    if 'broadcast' in ipv4_address:
-                        return intf_name
+        res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split(
+            "\n")
+        for item in res:
+            if "BROADCAST" in item:
+                return item.split(":")[1].strip()
         return "lo"
 
     def barrier(self, comm_world):

python/paddle/fluid/incubate/fleet/base/role_maker.py

@@ -955,26 +955,33 @@ class GeneralRoleMaker(RoleMakerBase):
         """
         get default physical interface
         """
-        import netifaces
-        gateways = netifaces.gateways()
-        if gateways.get(netifaces.AF_INET) != None:
-            gateway = gateways[netifaces.AF_INET]
-            if len(gateway) > 0 and len(gateway[0]) > 1:
-                return gateway[0][1]
+        res = os.popen("route -A inet").read().strip().split("\n")
+
+        gateway_idx = None
+        iface_idx = None
+        for item in res:
+            item = item.split()
+            if "Gateway" in item and "Iface" in item:
+                gateway_idx = item.index("Gateway")
+                iface_idx = item.index("Iface")
+            elif gateway_idx != None and iface_idx != None:
+                gateway = None
+                if len(item) > gateway_idx:
+                    gateway = item[gateway_idx]
+                if gateway and gateway != '*' and gateway != "0.0.0.0" and len(
+                        item) > iface_idx:
+                    return item[iface_idx]
         return "lo"
 
     def __get_default_iface_from_interfaces(self):
         """
         get default physical interface
         """
-        import netifaces
-        for intf_name in netifaces.interfaces():
-            addresses = netifaces.ifaddresses(intf_name)
-            if netifaces.AF_INET in addresses:
-                ipv4_addresses = addresses[netifaces.AF_INET]
-                for ipv4_address in ipv4_addresses:
-                    if 'broadcast' in ipv4_address:
-                        return intf_name
+        res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split(
+            "\n")
+        for item in res:
+            if "BROADCAST" in item:
+                return item.split(":")[1].strip()
         return "lo"
 
     def __start_kv_server(self, http_server_d, size_d):

python/paddle/fluid/tests/unittests/test_fleet.py

@@ -37,11 +37,7 @@ class TestFleet1(unittest.TestCase):
         from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
         from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_pslib_1")
-            return
+
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"
         os.environ["TRAINING_ROLE"] = "TRAINER"

python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py

@@ -36,11 +36,7 @@ class TestFleet1(unittest.TestCase):
         from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
         from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_pslib_1")
-            return
+
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"
         os.environ["TRAINING_ROLE"] = "TRAINER"

python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py

@@ -64,11 +64,7 @@ class TestCloudRoleMaker(unittest.TestCase):
         from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
         from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_pslib_1")
-            return
+
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"
         os.environ["TRAINING_ROLE"] = "TRAINER"

python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py

@@ -36,11 +36,7 @@ class TestCloudRoleMaker2(unittest.TestCase):
         from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
         from paddle.fluid.incubate.fleet.base.role_maker import RoleMakerBase
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_pslib_2")
-            return
+
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"
         os.environ["TRAINING_ROLE"] = "TRAINER"

python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py

@@ -36,11 +36,7 @@ class TestCloudRoleMaker(unittest.TestCase):
         from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
         from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_pslib_1")
-            return
+
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"
         os.environ["TRAINING_ROLE"] = "TRAINER"

python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py

@@ -47,12 +47,6 @@ class TestCloudRoleMaker(unittest.TestCase):
             print("warning: no fleet, skip test_pslib_4")
             return
 
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_pslib_4")
-            return
-
         class FakeStream():
             """
             it is a fake stream only for test.

python/paddle/fluid/tests/unittests/test_fleet_rolemaker_new.py

@@ -70,12 +70,6 @@ class TestCloudRoleMaker(unittest.TestCase):
         os.environ["TRAINING_ROLE"] = "TRAINER"
         os.environ["PADDLE_TRAINER_ID"] = "0"
 
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_tr_rolemaker")
-            return
-
         ro = role_maker.PaddleCloudRoleMaker(is_collective=False)
         self.assertTrue(ro._is_worker())
         ro = role_maker.PaddleCloudRoleMaker(is_collective=False)
@@ -110,12 +104,6 @@ class TestCloudRoleMaker(unittest.TestCase):
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"
 
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_ps_rolemaker")
-            return
-
         ro = role_maker.PaddleCloudRoleMaker(
             is_collective=False, init_gloo=False)
         self.assertEqual(ro._server_index(), 0)
@@ -131,11 +119,6 @@ class TestCloudRoleMaker(unittest.TestCase):
     def test_traing_role(self):
         """Test training role."""
         os.environ["TRAINING_ROLE"] = "TEST"
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_training_role")
-            return
 
         ro = role_maker.PaddleCloudRoleMaker(is_collective=False)
         self.assertRaises(ValueError, ro._generate_role)
@@ -150,11 +133,6 @@ class TestUserDefinedRoleMaker(unittest.TestCase):
         pass
 
     def test_ps_rolemaker(self):
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_ps_rolemaker")
-            return
 
         ro = role_maker.UserDefinedRoleMaker(
             is_collective=False,
@@ -169,12 +147,6 @@ class TestUserDefinedRoleMaker(unittest.TestCase):
         self.assertEqual(ro._role_id(), 0)
 
     def test_tr_rolemaker(self):
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_tr_rolemaker")
-            return
-
         ro = role_maker.UserDefinedRoleMaker(
             is_collective=False,
             init_gloo=False,

python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py

@@ -36,11 +36,7 @@ class TestFleet1(unittest.TestCase):
         from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
         from paddle.fluid.incubate.fleet.parameter_server.pslib import PSLib
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_pslib_1")
-            return
+
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"
         os.environ["TRAINING_ROLE"] = "TRAINER"

python/paddle/fluid/tests/unittests/test_fleet_util.py

@@ -100,11 +100,6 @@ class TestFleetUtil(unittest.TestCase):
     def test_get_file_shard(self):
         import paddle.distributed.fleet as fleet
         self.assertRaises(Exception, fleet.util.get_file_shard, "files")
-        try:
-            import netifaces
-        except:
-            print("warning: no netifaces, skip test_get_file_shard")
-            return
 
         role = role_maker.UserDefinedRoleMaker(
             is_collective=False,

python/requirements.txt

@@ -17,5 +17,3 @@ decorator
 prettytable
 astor
 pathlib
-netifaces ; platform_system != "Windows"
-netifaces ; python_version>="3.5" and platform_system == "Windows"

tools/dockerfile/Dockerfile.ubuntu

@@ -207,7 +207,6 @@ RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/
     cd binutils-2.27 && \
     ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz
 
-RUN pip --no-cache-dir install -U netifaces==0.10.9
 
 # Older versions of patchelf limited the size of the files being processed and were fixed in this pr.
 # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa

tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc48_ubuntu16

@@ -212,7 +212,6 @@ RUN wget --no-check-certificate https://pslib.bj.bcebos.com/openmpi-1.4.5.tar.gz
     export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH && export PATH=/usr/local/bin:$PATH && cd .. && \
     rm -rf openmpi-1.4.5.tar.gz && pip --no-cache-dir install mpi4py && ln -fs /bin/bash /bin/sh && \
     apt-get install libprotobuf-dev -y
-RUN pip --no-cache-dir install -U netifaces==0.10.9
 
 # ccache 3.7.9
 RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \

tools/manylinux1/Dockerfile.cuda10_cudnn7_gcc8_ubuntu16

@@ -221,7 +221,6 @@ RUN wget -q https://paddle-ci.gz.bcebos.com/binutils_2.27.orig.tar.gz && \
     ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz
 
 RUN apt-get install libprotobuf-dev -y
-RUN pip --no-cache-dir install -U netifaces==0.10.9 
 
 # ccache 3.7.9
 RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \