helper.py

#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from mpi4py import MPI
import ps_pb2 as pslib

class FileSystem(object):
    def __init__(self, fs_type="afs",
                 uri="afs://tianqi.afs.baidu.com:9902",
                 user=None,
                 passwd=None,
                 hadoop_bin="",
                 afs_conf=None):
        assert user != None
        assert passwd != None
        assert hadoop_bin != None
        self.fs_client = pslib.FsClientParameter()
        #if fs_type == "afs":
        #    fs_client.fs_type = pslib.FsApiType.AFS
        #else:
        #    fs_client.fs_type = pslib.FsApiType.HDFS
        self.fs_client.uri = uri
        self.fs_client.user = user
        self.fs_client.passwd = passwd
        #self.fs_client.buffer_size = 0
        self.fs_client.hadoop_bin = hadoop_bin
        #self.fs_client.afs_conf = afs_conf if not afs_conf else ""

    def get_desc(self):
        return self.fs_client

class MPIHelper(object):
    def __init__(self):
        self.comm = MPI.COMM_WORLD

    def get_rank(self):
        return self.comm.Get_rank()

    def get_size(self):
        return self.comm.Get_size()

    def get_ip(self):
        import socket
        local_ip = socket.gethostbyname(socket.gethostname())
        return local_ip

    def get_hostname(self):
        import socket
        return socket.gethostname()

    def finalize(self):
        MPI.Finalize()