From b64ff0c28e393be15eb2b377e6980a502a3e3036 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 25 Aug 2020 09:16:39 +0800 Subject: [PATCH] Imporvement for advisors 1.support auto-build, commit, push, create issue and PR for upgrade 2.get version_exceptions from gitee 3.bugfix for clone repo and spec create 4.set timeout for download src 5.support LTS upgrade with local repo yaml 6.optimize some code struct in gitee.py --- advisors/check_missing_specs.py | 7 +- advisors/gitee.py | 117 +++++++++++------- advisors/helper/version_exceptions.yaml | 25 ++++ advisors/oa_upgradable.py | 13 +- advisors/simple-update-robot.py | 155 ++++++++++++++++++------ 5 files changed, 218 insertions(+), 99 deletions(-) create mode 100644 advisors/helper/version_exceptions.yaml diff --git a/advisors/check_missing_specs.py b/advisors/check_missing_specs.py index 42202a02..1340aec7 100755 --- a/advisors/check_missing_specs.py +++ b/advisors/check_missing_specs.py @@ -48,12 +48,9 @@ if __name__ == "__main__": args = pars.parse_args() my_gitee = gitee.Gitee() - try: - spec_string = my_gitee.get_spec(args.repo) - except urllib.error.HTTPError: - spec_string = "" + spec_string = my_gitee.get_spec(args.repo) - if spec_string == "": + if not spec_string: print("no spec file found for {repo} project".format(repo=args.repo)) if args.push: issues = my_gitee.get_issues(args.repo) diff --git a/advisors/gitee.py b/advisors/gitee.py index 4f7d5460..9732435f 100755 --- a/advisors/gitee.py +++ b/advisors/gitee.py @@ -10,6 +10,7 @@ import urllib.error import argparse import yaml import re +import sys import os.path import json import pprint @@ -35,6 +36,7 @@ class Gitee(object): self.community_url_template = self.gitee_url + "openeuler/community/raw/master/repository/{repository}.yaml" #self.specfile_exception_url = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/helper/specfile_exceptions.yaml" self.specfile_exception_url = self.advisor_url + "advisors/helper/specfile_exceptions.yaml" + self.version_exception_url = self.advisor_url + "advisors/helper/version_exceptions.yaml" self.time_format = "%Y-%m-%dT%H:%M:%S%z" def post_gitee(self, url, values, headers=None): @@ -60,28 +62,51 @@ class Gitee(object): url = "https://gitee.com/api/v5/repos/src-openeuler/{repo}/forks".format(repo=repo) values = {} values["access_token"] = self.token["access_token"] - # headers["User-Agent"] = "curl/7.66.0" + #headers["User-Agent"] = "curl/7.66.0" #headers["Content-Type"] = "application/json;charset=UTF-8" #headers["HOST"] = "gitee.com" #headers["Accept"] = "*/*" return self.post_gitee(url, values) - def create_pr(self, head, repo): + def create_issue(self, repo, version, branch): + """ + Create issue in gitee + """ + title = "Upgrade {pkg} to {ver} in {br}".format(pkg=repo, ver=version, br=branch) + body = """This issue is automatically created by openEuler-Advisor. + Please check the correspond PR is accepted before close it. + Thanks. + Yours openEuler-Advisor.""" + self.post_issue(repo, title, body) + + def get_reviewers(self, repo): + """ + Get reviewers of pkg + """ + url = "https://gitee.com/api/v5/repos/src-openeuler/{pkg}/collaborators".format(pkg=repo) + return self.get_gitee(url) + + def create_pr(self, head, repo, version, branch): """ Create PR in gitee """ - url = "https://gitee.com/api/v5/repos/src-openeuler/{repo}/pulls".format(repo=repo) + assignees = "" + reviewer_info = self.get_reviewers(repo) + if reviewer_info: + reviewer_list = json.loads(reviewer_info) + assignees = ",".join(reviewer["login"] for reviewer in reviewer_list) + url = "https://gitee.com/api/v5/repos/src-openeuler/{pkg}/pulls".format(pkg=repo) values = {} values["access_token"] = self.token["access_token"] - values["title"] = "Upgrade to latest version of {repo}".format(repo=repo) - values["head"] = "{head}:master".format(head=head) - values["base"] = "master" - values["body"] = """This is a (mostly) automatically created PR by openEuler-Advisor. -Please be noted that it's not throughly tested. -Review carefully before accept this PR. -Thanks. -Yours openEuler-Advisor. -""" + values["title"] = "Upgrade {pkg} to {ver}".format(pkg=repo, ver=version) + values["head"] = "{hd}:{br}".format(hd=head, br=branch) + values["base"] = branch + values["assignees"] = assignees + values["body"] = """This is a automatically created PR by openEuler-Advisor. + Please be noted that it's not throughly tested. + Review carefully before accept this PR. + Thanks. + Yours openEuler-Advisor.""" return self.post_gitee(url, values) def get_gitee(self, url, headers=None): @@ -92,8 +117,11 @@ Yours openEuler-Advisor. req = urllib.request.Request(url=url, headers=self.headers) else: req = urllib.request.Request(url=url, headers=headers) - u = urllib.request.urlopen(req) - return u.read().decode("utf-8") + try: + u = urllib.request.urlopen(req) + return u.read().decode("utf-8") + except urllib.error.HTTPError: + return None def get_gitee_json(self, url): """ @@ -110,58 +138,55 @@ Yours openEuler-Advisor. Get well known spec file exceptions """ resp = self.get_gitee(self.specfile_exception_url) - exps = yaml.load(resp, Loader=yaml.Loader) - return exps + if not resp: + print("ERROR: specfile_exceptions.yaml may not exist.") + sys.exit(1) + excpt = yaml.load(resp, Loader=yaml.Loader) + return excpt + + def get_version_exception(self): + """ + Get vertion recommend exceptions + """ + resp = self.get_gitee(self.version_exception_url) + if not resp: + print("ERROR: version_exceptions.yaml may not exist.") + sys.exit(1) + excpt = yaml.load(resp, Loader=yaml.Loader) + return excpt def get_spec(self, pkg, br="master"): """ Get openeuler spec file for specific package """ specurl = self.specfile_url_template.format(branch=br, package=pkg, specfile=pkg + ".spec") - exp = self.get_spec_exception() - if pkg in exp: - dir_name = exp[pkg]["dir"] - file_name = exp[pkg]["file"] + excpt_list = self.get_spec_exception() + if pkg in excpt_list: + dir_name = excpt_list[pkg]["dir"] + file_name = excpt_list[pkg]["file"] specurl = urllib.parse.urljoin(specurl, os.path.join(dir_name, file_name)) - try: - resp = self.get_gitee(specurl) - except urllib.error.HTTPError: - resp = "" + resp = self.get_gitee(specurl) return resp - def get_yaml(self, pkg, br="master"): + def get_yaml(self, pkg): """ Get upstream yaml metadata for specific package """ yamlurl = self.advisor_url_template.format(package=pkg) - try: + resp = self.get_gitee(yamlurl) + if not resp: + yamlurl = self.yamlfile_url_template.format(branch="master", package=pkg) resp = self.get_gitee(yamlurl) - except urllib.error.HTTPError: - resp = "Not found" - print("WARNING: {repo}.yaml can't be found in upstream-info.".format(repo=pkg)) - if re.match("Not found", resp): - yamlurl = self.yamlfile_url_template.format(branch=br, package=pkg) - try: - resp = self.get_gitee(yamlurl) - except urllib.error.HTTPError: - resp = "Not found" - if re.match("Not found", resp): - print("WARNING: {repo}.yaml can't be found in repo on {branch} too".format(repo=pkg, branch=br)) - return False - else: - return resp - else: - return resp + if not resp: + print("WARNING: {repo}.yaml can't be found in upstream-info and repo.".format(repo=pkg)) + return resp def get_community(self, repo): """ Get yaml data from community repo """ yamlurl = self.community_url_template.format(repository=repo) - try: - resp = self.get_gitee(yamlurl) - except urllib.error.HTTPError: - resp = "" + resp = self.get_gitee(yamlurl) return resp def get_issues(self, pkg, prj="src-openeuler"): diff --git a/advisors/helper/version_exceptions.yaml b/advisors/helper/version_exceptions.yaml new file mode 100644 index 00000000..fcf6f7b0 --- /dev/null +++ b/advisors/helper/version_exceptions.yaml @@ -0,0 +1,25 @@ +--- +# version recommend exception list +gegl04: + - '20001120.v002' +gimp: + - '19990910' +nss: + - '334.20030307' +glibc: + - '9000' +kata-shim: + - '20191207' +kata-runtime: + - '20191207' +kata-proxy: + - '20191207' +kata-agent: + - '20191207' +kexec-tools: + - '200' +sombok: + - '2011' +shadow: + - '1999' + - '2000' diff --git a/advisors/oa_upgradable.py b/advisors/oa_upgradable.py index 78104437..4f202743 100644 --- a/advisors/oa_upgradable.py +++ b/advisors/oa_upgradable.py @@ -18,15 +18,6 @@ import check_upstream import version_recommend -def _get_rec_excpt(): - """ - Get except case of version recommend - """ - y_file = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "helper/ver_rec_excpt.yaml")) - excpt = yaml.load(y_file, Loader=yaml.Loader) - return excpt - - def _filter_except(excpts, sources): """ Filter except case in sources @@ -42,7 +33,7 @@ def get_ver_tags(gt, repo, cwd_path=None): """ if cwd_path: try: - repo_yaml = open(os.path.join(cwd_path, repo + ".yaml")).read() + repo_yaml = open(os.path.join(cwd_path, "{pkg}.yaml".format(pkg=repo))) except FileNotFoundError: print("WARNING: {pkg}.yaml can't be found in local path: {path}.".format(pkg=repo, path=cwd_path)) repo_yaml = gt.get_yaml(repo) @@ -82,7 +73,7 @@ def get_ver_tags(gt, repo, cwd_path=None): print("Unsupport version control method {vc}".format(vc=vc_type)) return None - excpt_list = _get_rec_excpt() + excpt_list = gt.get_version_exception() if repo in excpt_list: tags = _filter_except(excpt_list[repo], tags) return tags diff --git a/advisors/simple-update-robot.py b/advisors/simple-update-robot.py index 75338e59..7d4f44c7 100755 --- a/advisors/simple-update-robot.py +++ b/advisors/simple-update-robot.py @@ -37,7 +37,11 @@ def download_source_url(spec, o_ver, n_ver): return False elif source.startswith("http") or source.startswith("ftp"): fn = os.path.basename(source) - subprocess.call(["curl", "-L", source, "-o", fn]) + for time in range(2): + if subprocess.call(["curl", "-m", "600", "-L", source, "-o", fn]): + continue + else: + break return fn else: print("WARNING: Not valid URL for Source code") @@ -56,7 +60,11 @@ def download_upstream_url(gt, repo, n_ver): if rp_yaml["version_control"] == "github": url = "https://github.com/{rp}/archive/{nv}.tar.gz".format(rp=rp_yaml["src_repo"], nv=n_ver) fn = "{rp}.{nv}.tar.gz".format(rp=repo, nv=n_ver) - subprocess.call(["curl", "-L", url, "-o", fn]) + for time in range(2): + if subprocess.call(["curl", "-m", "600", "-L", url, "-o", fn]): + continue + else: + break return fn else: print("Handling {vc} is still under developing".format(vc=rp_yaml["version_control"])) @@ -89,6 +97,7 @@ def fork_clone_repo(gt, repo, br): subprocess.call(["git", "clone", "git@gitee.com:{user}/{pkg}".format(user=name, pkg=repo)]) os.chdir(repo) if subprocess.call(["git", "checkout", "{branch}".format(branch=br)]): + os.chdir(os.pardir) time.sleep(1) else: os.chdir(os.pardir) @@ -126,7 +135,7 @@ def create_spec(repo, spec_str, o_ver, n_ver, src_fn=None): in_changelog = False for l in spec_str.splitlines(): if l.startswith("Release:"): - fn.write(re.sub(r"\d", "1", l) + "\n") + fn.write(re.sub(r"\d+", "1", l) + "\n") continue if l.startswith("Source:") or l.startswith("Source0:"): if src_fn: @@ -150,7 +159,48 @@ def create_spec(repo, spec_str, o_ver, n_ver, src_fn=None): os.chdir(os.pardir) -def auto_update_pkg(gt, u_pkg, u_branch): +def build_pkg(u_pkg, u_branch): + """ + Auto build upgrade pkg on obs + """ + build_result = True + if (u_branch == "master"): + project = "openEuler:Mainline" + elif (u_branch == "openEuler-20.03-LTS"): + project = "openEuler:20.03:LTS" + else: + print("WARNING: Please check branch to be upgrade.") + sys.exit(1) + + subprocess.call(["osc", "branch", "{prj}".format(prj=project), "{pkg}".format(pkg=u_pkg)]) + user_info = subprocess.getoutput(["osc user"]) + user = user_info.split(':')[0] + subprocess.call(["osc", "co", "home:{usr}:branches:{prj}/{pkg}".format(usr=user, prj=project, pkg=u_pkg)]) + os.chdir("home:{usr}:branches:{prj}/{pkg}".format(usr=user, prj=project, pkg=u_pkg)) + subprocess.call(["rm * -rf"], shell=True) + subprocess.call(["cp ../../{pkg}/* .".format(pkg=u_pkg)], shell=True) + if subprocess.call(["osc", "build", "standard_aarch64"]): + build_result = False + os.chdir("../../") + return build_result + + +def push_create_pr_issue(gt, u_pkg, o_ver, u_ver, u_branch): + """ + Auto push update repo, create upgrade PR and issue. + """ + os.chdir(u_pkg) + subprocess.call(["git rm *{old_ver}.* -rf".format(old_ver=o_ver)], shell=True) + subprocess.call(["rm *_old.spec -f"], shell=True) + subprocess.call(["git add *"], shell=True) + subprocess.call(["git commit -m \"upgrade {pkg} to {ver}\"".format(pkg=u_pkg, ver=u_ver)], shell=True) + subprocess.call(["git push origin"], shell=True) + gt.create_pr(gt.token["user"], u_pkg, u_ver, u_branch) + gt.create_issue(u_pkg, u_ver, u_branch) + os.chdir(os.pardir) + + +def auto_update_pkg(gt, u_pkg, u_branch, u_ver=None): """ Auto upgrade based on given branch for single package """ @@ -161,36 +211,60 @@ def auto_update_pkg(gt, u_pkg, u_branch): sys.exit(1) pkg_spec = Spec.from_string(spec_str) pkg_ver = replace_macros(pkg_spec.version, pkg_spec) - - pkg_tags = oa_upgradable.get_ver_tags(gt, u_pkg) - if pkg_tags is None: + + if (u_branch == "master"): + pkg_tags = oa_upgradable.get_ver_tags(gt, u_pkg) + if pkg_tags is None: + sys.exit(1) + ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0) + u_ver = ver_rec.latest_version + elif re.search(r"LTS", u_branch): + if not u_ver: + print("WARNING: Please specify upgrade version in LTS upgrade.") + sys.exit(1) + else: + print("WARNING: Please check branch to upgrade.") sys.exit(1) - ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0) - rec_up_ver = ver_rec.latest_version - + fork_clone_repo(gt, u_pkg, u_branch) - if not update_ver_check(u_pkg, pkg_ver, rec_up_ver): + if not update_ver_check(u_pkg, pkg_ver, u_ver): sys.exit(1) - if not download_src(gt, pkg_spec, pkg_ver, rec_up_ver): + if not download_src(gt, pkg_spec, pkg_ver, u_ver): sys.exit(1) - create_spec(u_pkg, spec_str, pkg_ver, rec_up_ver) + create_spec(u_pkg, spec_str, pkg_ver, u_ver) if len(pkg_spec.patches) >= 1: print("WARNING: {repo} has multiple patches, please analyse it.".format(repo=u_pkg)) sys.exit(1) + + if not build_pkg(u_pkg, u_branch): + sys.exit(1) + + push_create_pr_issue(gt, u_pkg, pkg_ver, u_ver, u_branch) def auto_update_repo(gt, u_repo, u_branch): """ Auto upgrade based on given branch for packages in given repository """ - repo_yaml = gt.get_community(u_repo) - if not repo_yaml: - print("WARNING: {repo}.yaml in community is empty.".format(repo=u_repo)) + if (u_branch == "master"): + repo_yaml = gt.get_community(u_repo) + if not repo_yaml: + print("WARNING: {repo}.yaml in community is empty.".format(repo=u_repo)) + sys.exit(1) + elif re.search(r"LTS", u_branch): + try: + repo_yaml = open(os.path.join(os.getcwd(), "{repo}.yaml".format(repo=u_repo))) + except FileNotFoundError: + print("WARNING: {repo}.yaml can't be found in current working directory.".format(repo=u_repo)) + sys.exit(1) + else: + print("WARNING: Please check branch to upgrade.") sys.exit(1) + pkg_info = yaml.load(repo_yaml, Loader=yaml.Loader) pkg_list = pkg_info.get("repositories") for pkg in pkg_list: @@ -203,26 +277,34 @@ def auto_update_repo(gt, u_repo, u_branch): continue pkg_spec = Spec.from_string(spec_str) pkg_ver = replace_macros(pkg_spec.version, pkg_spec) - - pkg_tags = oa_upgradable.get_ver_tags(gt, pkg_name) - if pkg_tags is None: - continue - ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0) - rec_up_ver = ver_rec.latest_version + + if (u_branch == "master"): + pkg_tags = oa_upgradable.get_ver_tags(gt, pkg_name) + if pkg_tags is None: + continue + ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0) + u_ver = ver_rec.latest_version + else: + u_ver = pkg.get("u_ver") fork_clone_repo(gt, pkg_name, u_branch) - if not update_ver_check(pkg_name, pkg_ver, rec_up_ver): + if not update_ver_check(pkg_name, pkg_ver, u_ver): continue - if not download_src(gt, pkg_spec, pkg_ver, rec_up_ver): + if not download_src(gt, pkg_spec, pkg_ver, u_ver): continue - create_spec(pkg_name, spec_str, pkg_ver, rec_up_ver) + create_spec(pkg_name, spec_str, pkg_ver, u_ver) if len(pkg_spec.patches) >= 1: print("WARNING: {repo} has multiple patches, please analyse it.".format(repo=pkg_name)) continue + + if not build_pkg(pkg_name, u_branch): + continue + + push_create_pr_issue(gt, pkg_name, pkg_ver, u_ver, u_branch) if __name__ == "__main__": @@ -232,26 +314,21 @@ if __name__ == "__main__": pars.add_argument("-u", "--update", type=str, help="Auto upgrade for packages in repository or single package", choices=["repo", "pkg"]) pars.add_argument("-n", "--new_version", type=str, help="New upstream version of package will be upgrade to") - pars.add_argument("-s", "--create_spec", help="Create spec file", action="store_true") pars.add_argument("-d", "--download", help="Download upstream source code", action="store_true") + pars.add_argument("-s", "--create_spec", help="Create spec file", action="store_true") pars.add_argument("-fc", "--fork_then_clone", help="Fork src-openeuler repo into users, then clone to local", action="store_true") - pars.add_argument("-p", "--PR", help="Create upgrade PR", action="store_true") + pars.add_argument("-b", "--build_pkg", help="Build package in local", action="store_true") + pars.add_argument("-pcpi", "--push_create_pr_issue", help="Push update repo, create PR and issue", action="store_true") args = pars.parse_args() user_gitee = gitee.Gitee() if args.update: - if not args.branch == "master": - print("WARNING: Now only support master version auto-upgrade.") - print("WARNING: You can try manually upgrade with specified version, command as follow:") - print("python3 simple-update-robot.py {pkg} {br} -fc -d -s -n upgrade_ver".format( - pkg=args.repo_pkg, br=args.branch)) - sys.exit(1) if args.update == "repo": auto_update_repo(user_gitee, args.repo_pkg, args.branch) else: - auto_update_pkg(user_gitee, args.repo_pkg, args.branch) + auto_update_pkg(user_gitee, args.repo_pkg, args.branch, args.new_version) else: spec_string = user_gitee.get_spec(args.repo_pkg, args.branch) if not spec_string: @@ -263,7 +340,7 @@ if __name__ == "__main__": if args.fork_then_clone: fork_clone_repo(user_gitee, args.repo_pkg, args.branch) - if args.download or args.create_spec: + if args.download or args.create_spec or args.push_create_pr_issue: if not args.new_version: print("Please specify the upgraded version of the {repo}".format(repo=args.repo_pkg)) sys.exit(1) @@ -280,6 +357,10 @@ if __name__ == "__main__": if len(spec_file.patches) >= 1: print("WARNING: {repo} has multiple patches, please analyse it.".format(repo=args.repo_pkg)) sys.exit(1) + + if args.build_pkg: + if not build_pkg(args.repo_pkg, args.branch): + sys.exit(1) - if args.PR: - user_gitee.create_pr(user_gitee.token["user"], args.repo_pkg) + if args.push_create_pr_issue: + push_create_pr_issue(user_gitee, args.repo_pkg, cur_version, args.new_version, args.branch) -- GitLab