helper.py

from mpi4py import MPI

class FileSystem(object):
    def __init__(self, fs_type="afs",
                 uri="afs://tianqi.afs.baidu.com:9902",
                 user=None,
                 passwd=None,
                 hadoop_bin="",
                 afs_conf=None):
        assert user not None
        assert passwd not None
        assert hadoop_bin not None
        fs_client = pslib.FsClientParameter()
        if fs_type == "afs":
            fs_client.fs_type = pslib.FsApiType.AFS
        else:
            fs_client.fs_type = pslib.FsApiType.HDFS
        fs_client.uri = uri
        fs_client.user = user
        fs_client.passwd = passwd
        fs_client.buffer_size = 0
        fs_client.afs_conf = afs_conf if not afs_conf else ""


class MPIHelper(object):
    def __init__(self):
        self.comm = MPI.COMM_WORLD

    def get_rank(self):
        return self.comm.Get_rank()

    def get_size(self):
        return self.comm.Get_size()

    def get_ip(self):
        import socket
        local_ip = socket.gethostbyname(socket.gethostname())
        return local_ip

    def get_hostname(self):
        import socket
        return socket.gethostname()