integration.py

#!/usr/bin/python
########################################################
#
# Copyright 2019 Baidu Inc. All Rights Reserved.
#
########################################################
"""
integration
"""
import os
import sys
import tarfile
import socket
import urllib
# global setting
g_root_path = os.path.split(os.path.realpath(__file__))[0]
g_tool_file = os.path.split(os.path.realpath(__file__))[1]

def error_message(msg):
    """Print the error message in red."""
    print("\033[1;31m[ERROR] %s\033[0m" % msg)
def info_message(msg):
    """Print the info message in green."""
    print("\033[1;32m[INFO] %s\033[0m" % msg)
    # -1:failed 0:success 1:no such file

class Downloader(object):
    """Downloader"""

    def __init__(self):
        """init"""
        self.__root_path = ""
        self.__download_path = ""
        self.__download_platforms = []
        self.__ftp_client = FtpClient(g_ftp_host, g_ftp_port, g_ftp_username, g_ftp_password)

    def __del__(self):
        """delete"""
        del self.__ftp_client
    
    def download(self):
        """download depends"""
        ret = 0
        # parse arg
        if self.__parse_arg() != 0:
            ret = -1
        else:
            self.__print_env()
        # get need modules
        parent_module_name = [""]
        parent_version_id = [""]
        need_modules = {}
        if ret == 0:
            if self.__valid_version_depend(self.__root_path + "/" + g_json_file_name) != 0:
                ret = -1
            else:
                self.__get_need_modules(parent_module_name, parent_version_id, need_modules)
        # make downlaod path
        if ret == 0:
            if self.__make_dir(self.__download_path) != 0:
                ret = -1
        # download modules
        if ret == 0:
            for platform in self.__download_platforms:
                info_message("**** download " + platform + " ****")
                self.downloaded_map = {}
                self.downloading_statck = []
                self.downloading_statck.append(parent_module_name[0])
                has_error = 0
                for key in need_modules:
                    if self.__download_module(key, need_modules[key],
                        parent_module_name[0], parent_version_id, platform) != 0:
                        has_error = 1
                        break
                if has_error == 1:
                    ret = -1
                    break
        # last log
        if ret == 0:
            info_message("==== success ====")
        else:
            info_message("==== failed ====")
        return ret

    def __get_need_modules(self, parent_module_name, parent_version_id, need_modules):
        parent_module_name[0] = self.version_depend_obj["module"]
        parent_version_id[0] = "v_" + self.version_depend_obj["version"] + "_" + self.version_depend_obj["branch"]
        module_depends = self.version_depend_obj["depends"]
        for i in module_depends:
            need_modules[i["name"]] = i["link"]

    def __valid_module_name(self, module_name):
        ret = 0
        if module_name not in g_supported_modules:
            ret = -1
        if ret != 0:
            supported_modules = ""
            for i in g_supported_modules:
                supported_modules += " " + i
            error_message("module name error: " + module_name + " (supported modules:" + supported_modules + ")")
        return ret

    def __valid_version(self, version):
        ret = 0
        number_list = version.split(".")
        if len(number_list) == 4:
            for i in number_list:
                if not i.isdigit():
                    ret = -1
                    break
        else:
            ret = -1
        if ret != 0:
            error_message("version format error: " + version + " (eg: 1.0.0.0)")
        return ret

    def __valid_branch(self, branch):
        ret = 0
        if branch == "":
            ret = -1
        else:
            search_obj = re.search("[^a-zA-Z0-9_-]", branch)
            if search_obj:
                ret = -1
        if ret != 0:
            error_message("branch format error: " + branch + " (regular expression: [a-zA-z0-9_-])")
        return ret

    def __valid_version_id(self, version_id):
        ret = 0
        if version_id not in g_branch_list:
            part_list = version_id.split("_")
            if len(part_list) < 3:
                ret = -1
            else:
                if part_list[0] != "v" or self.__valid_version(part_list[1]) != 0:
                    ret = -1
                else:
                    index = len(part_list[0]) + len(part_list[1]) + 2
                    if self.__valid_branch(version_id[index:]) != 0:
                        ret = -1
        if ret != 0:
            error_message("version_id format error: " + version_id + " (eg: v_1.0.0.0_MyBranch, develop or stable)")
        return ret

    def __valid_version_depend(self, json_file):
        ret = 0
        info_message("read " + json_file + " ...")
        json_str = [""]
        if self.__read_json_file(json_file, json_str) != 0:
            ret = -1
        else:
            try:
                json_obj = json.loads(json_str[0])
                if self.__valid_module_name(json_obj["module"]) != 0:
                    ret = -1
                if self.__valid_version(json_obj["version"]) != 0:
                    ret = -1
                if self.__valid_branch(json_obj["branch"]) != 0:
                    ret = -1
                if self.__valid_branch(json_obj["link"]) != 0:
                    ret = -1
                module_depends = json_obj["depends"]
                for i in module_depends:
                    if self.__valid_module_name(i["name"]) != 0:
                        ret = -1
                    if self.__valid_version_id(i["version_id"]) != 0:
                        ret = -1
                if ret == 0:
                    self.version_depend_obj = json_obj
            except:
                ret = -1
                error_message("version_depend format error: " + json_file)
        return ret

    def __make_dir(self, path):
        ret = 0
        try:
            if not os.path.exists(path):
                os.makedirs(path)
        except:
            ret = -1
            error_message("make directory failed: " + path)
        return ret

    def __remove_path(self, path):
        ret = 0
        try:
            if os.path.exists(path):
                info_message("remove " + path + " ...")
                shutil.rmtree(path)
        except:
            ret = -1
            error_message("remove " + path + " failed")
        return ret

    def __remove_file(self, file_path):
        if os.path.isfile(file_path):
            info_message("remove " + file_path + " ...")
            os.remove(file_path)


    def __get_module_depends(self, json_file, module_depends):
        ret = 0
        info_message("read " + json_file + " ...")
        # to read json file
        json_str = [""]
        if self.__read_json_file(json_file, json_str) != 0:
            ret = -1
        # parse json data
        if ret == 0:
            try:
                json_obj = json.loads(json_str[0])
                module_depends += json_obj["depends"]
            except:
                ret = -1
                error_message("json format error: " + json_file)
        return ret

    def __download_module_depends(self, json_file, parent_module_name, parent_version_id, platform):
        ret = 0
        module_depends = []
        if self.__get_module_depends(json_file, module_depends) != 0:
            ret = -1
        if ret == 0:
            for module in module_depends:
                if self.__download_module(module["name"], \
                    module["version_id"], parent_module_name, parent_version_id, platform) != 0:
                    ret = -1
                    break
        return ret
    def __download_platform(self, module_name, link, platform):
            file_path = # TODO: 通过version_depend.json获取
            print "downloading " + g_depends[moudle_name]
            download_file(file_path, link)
            print "done. saved to " + file_path
            target_path = # TODO: 
            print "tar file ..."
            __tar_file(file_path, target_path)

    def __download_module(self, module_name, link, parent_module_name, parent_version_id, platform):
        ret = 0
        info_message("download " + module_name + ": " + link + " " + platform + " ...")
        # check is downloaded
        is_download = 0
        if ret == 0:
            if module_name in self.downloaded_map:
                is_download = 1
                info_message(module_name + "-" + actual_version_id[0] + " already downloaded")
            else:
                self.downloading_statck.append(module_name)
        # download
        if ret == 0 and is_download == 0:
            if self.__download_platform(module_name, link, platform) != 0:
                ret = -1
            else:
                json_file = self.__download_path + "/" + module_name + "/" + platform + "/" + g_json_file_name
                ret = self.__download_module_depends(json_file, module_name, actual_version_id[0], platform) # recursion
            # update downloading statck
            if ret == 0:
                self.downloaded_map[module_name] = module
                self.downloading_statck.pop()
        return ret

    def __get_actual_version_id(self, module_name, version_id, actual_version_id):
        ret = 0
        actual_version_id[0] = version_id
        if version_id in g_branch_list:
            local_file = self.__download_path + "/" + module_name + "-" + g_tag_file
            remote_file = g_ftp_root + "/" + module_name + "/" + module_name + "-" + g_tag_file
            if self.__ftp_client.download_file(local_file, remote_file) != 0:
                ret = -1
                error_message("download failed: no such file \"" + remote_file + "\"")
                self.__remove_file(local_file)
            if ret == 0:
                json_str = [""]
                if self.__read_json_file(local_file, json_str) != 0:
                    ret = -1
                else:
                    try:
                        json_obj = json.loads(json_str[0])
                        actual_version_id[0] = "v_" + json_obj[version_id] + "_" + version_id
                    except:
                        ret = -1
                        error_message("json format error: " + local_file)
            if ret == 0:
                info_message("current " + module_name + "`s " + version_id + " version is " + actual_version_id[0])
        return ret

    def __check_module_conflict(self, module):
        ret = 0
        curr_module = module.module_name
        # check recycle depends
        recycle_list = []
        for i in self.downloading_statck:
            if i == curr_module:
                ret = -1
            if ret != 0:
                recycle_list.append(i)
        if ret != 0:
            err_info = "recycle depend: "
            for i in recycle_list:
                err_info += "->" + i
            error_message(err_info)
        # check version conflict
        if ret == 0:
            if curr_module in self.downloaded_map:
                if self.downloaded_map[curr_module].version_id != module.version_id:
                    ret = -1
                    old_module = self.downloaded_map[curr_module]
                    error_message("version conflict: " + \
                        module.parent_module_name + "->" + \
                        curr_module + "_" + module.version_id + ", " + \
                        old_module.parent_module_name + "->" + \
                        old_module.module_name + "_" + old_module.version_id)
        return ret

    def __tar_file(self, file_path, target_path):
        ret = 0
        info_message("tar " + file_path + " ...")
        try:
            tar = tarfile.open(file_path, "r:gz")
            try:
                tar.extractall(target_path)
            except:
                ret = -1
                error_message("tar " + file_path + " failed")
            finally:
                tar.close()
        except:
            ret = -1
            error_message("open " + file_path + " failed")
        return ret

    def __read_json_file(self, json_file, json_str):
        ret = 0
        try:
            file = open(json_file, "r")
            try:
                json_str[0] = file.read()
            except:
                ret = -1
                error_message("read " + json_file + " failed")
            finally:
                file.close()
        except:
            ret = -1
            error_message("open " + json_file + " failed")
        return ret

    def __print_env(self):
        info_message("================ env bengin ================")
        info_message("root path: " + self.__root_path)
        info_message("download path: " + self.__download_path)
        tmp_str = ",".join(self.__download_platforms)
        info_message("download platforms: " + tmp_str)
        info_message("================ env end    ================")

    def __parse_arg(self):
        ret = 0
        try:
            opts, args = getopt.getopt(sys.argv[1:], "R:npqlvahA")
            for op, value in opts:
                if op == "-R":
                    self.__root_path = value
                    self.__download_path = self.__root_path + "/inner-depend"
                elif op == "-n":
                    self.__download_platforms.append("linux-x86_64")
                elif op == "-p":
                    self.__download_platforms.append("linux-aarch64")
                elif op == "-q":
                    self.__download_platforms.append("qnx-aarch64")
                elif op == "-l":
                    self.__download_platforms.append("linaro-aarch64")
                elif op == "-v":
                    self.__download_platforms.append("qnx-x86_64")
                elif op == "-a":
                    self.__download_platforms.append("poky-aarch32")
                elif op == "-A":
                    self.__download_platforms.append("poky-aarch64")
                elif op == "-h":
                    ret = -1
                    break
        except getopt.GetoptError as error:
            ret = -1
            error_message(error)
        # check arguments
        if ret == 0:
            if self.__root_path == "":
                ret = -1
                error_message("argument error: root path is empty")
            if len(self.__download_platforms) == 0:
                ret = -1
                error_message("argument error: download platforms is empty")
        if ret != 0:
            self.__usage()
        return ret

    def __usage(self):
        """print usage"""
        info_message("================ usage bengin ================")
        info_message("-R: root path")
        info_message("-n: platform option, linux-x86_64,  n = native")
        info_message("-p: platform option, linux-aarch64, p = px2")
        info_message("-q: platform option, qnx-aarch64,   q = qnx")
        info_message("-l: platform option, linaro-aarch64,   l = linaro")
        info_message("-v: platform option, qnx-x86_64,    v = vmware")
        info_message("-a: platform option, poky-aarch32,  a = poky")
        info_message("-h: help")
        info_message("example:")
        info_message("    ./download-depend.py -R . -n")
        info_message("    ./download-depend.py -R . -npqv")
        info_message("================ usage end    ================")


# entry
inst = Downloader()
inst.download()


def download_file(self, local_file, remote_file):
    """download file"""
    ret = 0
    urllib.urlretrieve(remote_file, local_file)
    return ret

class Integration(object):
    """integration"""
    def __init__(self):
        """init"""
        self.__version = "1.0.0.0"
    def integrate(self):
        # print tips
        self.__print_tips()
        """integrate"""
        # update
        log_failed = 1
        # download depends
        for moudle_name in g_depends:
            file_path = # TODO: 通过version_depend.json获取
            print "downloading " + g_depends[moudle_name]
            download_file(file_path, g_depends[moudle_name])
            print "done. saved to " + file_path
            target_path = # TODO: 
            print "tar file ..."
            __tar_file(file_path, target_path)
    def __remove_file(self, file_path):
        if os.path.isfile(file_path):
            os.remove(file_path)
    def __make_dir(self, path):
        ret = 0
        try:
            if not os.path.exists(path):
                os.makedirs(path)
        except:
            ret = -1
            error_message("make directory failed: " + path)
        return ret
    def __tar_file(self, file_path, target_path):
        ret = 0
        try:
            tar = tarfile.open(file_path, "r:gz")
            try:
                tar.extractall(target_path)
            except:
                ret = -1
                error_message("tar " + file_path + " failed")
            finally:
                tar.close()
        except:
            ret = -1
            error_message("open " + file_path + " failed")
        return ret
    def __print_tips(self):
        info_message("**** " + g_tool_file + " (" + self.__version + ") ****")
# entry
inst = Integration()
inst.integrate()