helper.py

#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


class FileSystem(object):
    """
    A file system that support async_executor hadoop client desc. 

    Args:
        fs_type (string): fs_type, for example is "afs"
        user (string): hadoop param
        passwd (string): hadoop param
        hadoop bin (string): hadoop param
    Examples:
        fs = FileSystm()
    """

    def __init__(self,
                 fs_type="afs",
                 uri="afs://tianqi.afs.baidu.com:9902",
                 user=None,
                 passwd=None,
                 hadoop_bin=""):
        assert user != None
        assert passwd != None
        assert hadoop_bin != None
        import ps_pb2 as pslib
        self.fs_client = pslib.FsClientParameter()
        #if fs_type == "afs":
        #    fs_client.fs_type = pslib.FsApiType.AFS
        #else:
        #    fs_client.fs_type = pslib.FsApiType.HDFS
        self.fs_client.uri = uri
        self.fs_client.user = user
        self.fs_client.passwd = passwd
        #self.fs_client.buffer_size = 0
        self.fs_client.hadoop_bin = hadoop_bin
        #self.fs_client.afs_conf = afs_conf if not afs_conf else ""

    def get_desc(self):
        """
        get hadoop desc.
        """
        return self.fs_client


class MPIHelper(object):
    """
    MPIHelper is a wrapper of mpi4py, support get_rank get_size etc.
    Args:
        No params
    Examples:
        mh = MPIHelper()
        mh.get_ip()
    """

    def __init__(self):
        from mpi4py import MPI
        self.comm = MPI.COMM_WORLD
        self.MPI = MPI

    def get_rank(self):
        return self.comm.Get_rank()

    def get_size(self):
        return self.comm.Get_size()

    def get_ip(self):
        import socket
        local_ip = socket.gethostbyname(socket.gethostname())
        return local_ip

    def get_hostname(self):
        import socket
        return socket.gethostname()

    def finalize(self):
        self.MPI.Finalize()