add fleet util, add some interface in hdfs util (#18752)

* add fleet util (fleet/utils/fleet_util.py): functions for users' convenience
* add some interface in hdfs util : hdfs is_file、hdfs cat

paddle/fluid/pybind/fleet_wrapper_py.cc

@@ -43,6 +43,7 @@ void BindFleetWrapper(py::module* m) {
   py::class_<framework::FleetWrapper>(*m, "Fleet")
       .def(py::init())
       .def("push_dense", &framework::FleetWrapper::PushDenseVarsSync)
+      .def("pull_dense", &framework::FleetWrapper::PullDenseVarsSync)
       .def("init_server", &framework::FleetWrapper::InitServer)
       .def("run_server", &framework::FleetWrapper::RunServer)
       .def("init_worker", &framework::FleetWrapper::InitWorker)

python/paddle/fluid/incubate/fleet/utils/hdfs.py

@@ -92,6 +92,22 @@ class HDFSClient(object):
 
         return ret_code, ret_out, ret_err
 
+    def cat(self, hdfs_path=None):
+        if self.is_file(hdfs_path):
+            exist_cmd = ['-cat', hdfs_path]
+            returncode, output, errors = self.__run_hdfs_cmd(
+                exist_cmd, retry_times=1)
+            if returncode != 0:
+                _logger.error("HDFS cat HDFS path: {} failed".format(hdfs_path))
+                return ""
+            else:
+                _logger.error("HDFS cat HDFS path: {} succeed".format(
+                    hdfs_path))
+                return output.strip()
+
+        else:
+            return ""
+
     def is_exist(self, hdfs_path=None):
         """
         whether the remote HDFS path exists
@@ -141,6 +157,32 @@ class HDFSClient(object):
                 hdfs_path))
             return True
 
+    def is_file(self, hdfs_path=None):
+        """
+        whether the remote HDFS path is file
+
+        Args:
+            hdfs_path(str): the hdfs file path
+
+        Returns:
+            True or False
+        """
+
+        if not self.is_exist(hdfs_path):
+            return False
+
+        dir_cmd = ['-test', '-d', hdfs_path]
+        returncode, output, errors = self.__run_hdfs_cmd(dir_cmd, retry_times=1)
+
+        if returncode == 0:
+            _logger.error("HDFS path: {} failed is not a file".format(
+                hdfs_path))
+            return False
+        else:
+            _logger.info("HDFS path: {} successfully is a file".format(
+                hdfs_path))
+            return True
+
     def delete(self, hdfs_path):
         """
         Remove a file or directory from HDFS.

python/setup.py.in

@@ -133,7 +133,8 @@ packages=['paddle',
           'paddle.fluid.incubate.fleet.parameter_server',
           'paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler',
           'paddle.fluid.incubate.fleet.parameter_server.pslib',
-          'paddle.fluid.incubate.fleet.collective']
+          'paddle.fluid.incubate.fleet.collective',
+          'paddle.fluid.incubate.fleet.utils']
 
 with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
     setup_requires = f.read().splitlines()