Add some functions and improvements:

1.Add auto_update for package and repository
2.Package interface of oa_upgradable
3.Imporve some code for user experience
Signed-off-by: NLeo Fang <leofang_94@163.com>

advisors/gitee.py

@@ -26,12 +26,13 @@ class Gitee(object):
 
         self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW 64; rv:50.0) Gecko/20100101 Firefox/50.0'}
         self.gitee_url = "https://gitee.com/"
-        self.src_openeuler_url = self.gitee_url + "src-openeuler/{package}/raw/master/"
+        self.src_openeuler_url = self.gitee_url + "src-openeuler/{package}/raw/{branch}/"
         self.advisor_url = self.gitee_url + "openeuler/openEuler-Advisor/raw/master/"
         self.specfile_url_template = self.src_openeuler_url + "{specfile}"
         self.yamlfile_url_template = self.src_openeuler_url + "{package}.yaml"
         #self.advisor_url_template = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/{package}.yaml"
         self.advisor_url_template = self.advisor_url + "upstream-info/{package}.yaml"
+        self.community_url_template = self.gitee_url + "openeuler/community/raw/master/repository/{repository}.yaml"
         #self.specfile_exception_url = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/helper/specfile_exceptions.yaml"
         self.specfile_exception_url = self.advisor_url + "advisors/helper/specfile_exceptions.yaml"
         self.time_format = "%Y-%m-%dT%H:%M:%S%z"
@@ -96,7 +97,7 @@ Yours openEuler-Advisor.
 
     def get_gitee_json(self, url):
         """
-        get and load gitee json response
+        Get and load gitee json response
         """
         headers = self.headers.copy()
         #headers = {}
@@ -106,31 +107,29 @@ Yours openEuler-Advisor.
 
     def get_spec_exception(self):
         """
-        get well known spec file exceptions
+        Get well known spec file exceptions
         """
         resp = self.get_gitee(self.specfile_exception_url)
         exps = yaml.load(resp, Loader=yaml.Loader)
         return exps
 
-    def get_spec(self, pkg):
+    def get_spec(self, pkg, br="master"):
         """
-        get openeuler spec file for specific package
+        Get openeuler spec file for specific package
         """
-        specurl = self.specfile_url_template.format(package=pkg, specfile=pkg + ".spec")
+        specurl = self.specfile_url_template.format(branch=br, package=pkg, specfile=pkg + ".spec")
         exp = self.get_spec_exception()
         if pkg in exp:
             dir_name = exp[pkg]["dir"]
             file_name = exp[pkg]["file"]
             specurl = urllib.parse.urljoin(specurl, os.path.join(dir_name, file_name))
-
         try:
             resp = self.get_gitee(specurl)
         except urllib.error.HTTPError:
             resp = ""
-
         return resp
 
-    def get_yaml(self, pkg):
+    def get_yaml(self, pkg, br="master"):
         """
         get upstream yaml metadata for specific package
         """
@@ -140,7 +139,7 @@ Yours openEuler-Advisor.
         except urllib.error.HTTPError:
             resp = "Not found"
         if re.match("Not found", resp):
-            yamlurl = self.yamlfile_url_template.format(package=pkg)
+            yamlurl = self.yamlfile_url_template.format(branch=br, package=pkg)
             try:
                 resp = self.get_gitee(yamlurl)
             except urllib.error.HTTPError:
@@ -153,6 +152,17 @@ Yours openEuler-Advisor.
         else:
             return resp
 
+    def get_community(self, repo):
+        """
+        Get yaml data from community repo
+        """
+        yamlurl = self.community_url_template.format(repository=repo)
+        try:
+            resp = self.get_gitee(yamlurl)
+        except urllib.error.HTTPError:
+            resp = ""
+        return resp
+
     def get_issues(self, pkg, prj="src-openeuler"):
         """
         List all open issues of pkg

advisors/oa_upgradable.py

@@ -12,10 +12,54 @@ import os
 import argparse
 
 import urllib.error
+
 import gitee
 import check_upstream
 import version_recommend
 
+
+def get_ver_tags(gt, repo, d_path=None):
+    """
+    Get version tags of given package
+    """
+    repo_yaml = ""
+    if d_path:
+        try:
+            repo_yaml = open(os.path.join(d_path, repo + ".yaml")).read()
+        except FileNotFoundError:
+            print("Failed to get YAML info from default path.")
+
+    if not repo_yaml:
+        try:
+            repo_yaml = gt.get_yaml(repo)
+        except urllib.error.HTTPError:
+            print("Failed to get YAML info for {pkg}".format(pkg=repo))
+            return None
+
+    pkg_info = yaml.load(repo_yaml, Loader=yaml.Loader)
+
+    vc_type = pkg_info["version_control"]
+    if vc_type == "hg":
+        tags = check_upstream.check_hg(pkg_info)
+    elif vc_type == "github":
+        tags = check_upstream.check_github(pkg_info)
+    elif vc_type == "git":
+        tags = check_upstream.check_git(pkg_info)
+    elif vc_type == "gitlab.gnome":
+        tags = check_upstream.check_gnome(pkg_info)
+    elif vc_type == "svn":
+        tags = check_upstream.check_svn(pkg_info)
+    elif vc_type == "metacpan":
+        tags = check_upstream.check_metacpan(pkg_info)
+    elif vc_type == "pypi":
+        tags = check_upstream.check_pypi(pkg_info)
+    else:
+        print("Unsupport version control method {vc}".format(vc=vc_type))
+        return None
+    
+    return tags
+
+
 if __name__ == "__main__":
     parameters = argparse.ArgumentParser()
     parameters.add_argument("-p", "--push", action="store_true",
@@ -27,56 +71,23 @@ if __name__ == "__main__":
 
     args = parameters.parse_args()
 
-    gitee = gitee.Gitee()
-    prj_name = args.repo
-    spec_string = gitee.get_spec(prj_name)
+    user_gitee = gitee.Gitee()
+    spec_string = user_gitee.get_spec(args.repo)
     if not spec_string:
-        print("{repo} seems to be an empty repository".format(repo=args.repo))
+        print("{pkg}.spec can't be found on the master branch".format(pkg=args.repo))
         sys.exit(1)
 
-    s_spec = Spec.from_string(spec_string)
-
-    current_version = replace_macros(s_spec.version, s_spec)
-
-    print("Checking ", prj_name)
-    print("current version is ", current_version)
+    spec_file = Spec.from_string(spec_string)
+    cur_version = replace_macros(spec_file.version, spec_file)
 
-    try:
-        prj_info_string = open(os.path.join(args.default, prj_name + ".yaml")).read()
-    except FileNotFoundError:
-        prj_info_string = ""
+    print("Checking ", args.repo)
+    print("current version is ", cur_version)
 
-    if not prj_info_string:
-        print("Get YAML info from gitee")
-        try:
-            prj_info_string = gitee.get_yaml(prj_name)
-        except urllib.error.HTTPError:
-            print("Failed to get YAML info for {pkg}".format(pkg=prj_name))
-            sys.exit(1)
-
-
-    prj_info = yaml.load(prj_info_string, Loader=yaml.Loader)
-
-    vc_type = prj_info["version_control"]
-    if vc_type == "hg":
-        tags = check_upstream.check_hg(prj_info)
-    elif vc_type == "github":
-        tags = check_upstream.check_github(prj_info)
-    elif vc_type == "git":
-        tags = check_upstream.check_git(prj_info)
-    elif vc_type == "gitlab.gnome":
-        tags = check_upstream.check_gnome(prj_info)
-    elif vc_type == "svn":
-        tags = check_upstream.check_svn(prj_info)
-    elif vc_type == "metacpan":
-        tags = check_upstream.check_metacpan(prj_info)
-    elif vc_type == "pypi":
-        tags = check_upstream.check_pypi(prj_info)
-    else:
-        print("Unsupport version control method {vc}".format(vc=vc_type))
+    pkg_tags = get_ver_tags(user_gitee, args.repo, args.default)
+    if pkg_tags is None:
         sys.exit(1)
+    ver_rec = version_recommend.VersionRecommend(pkg_tags, cur_version, 0)
 
-    print("known release tags :", tags)
-    v = version_recommend.VersionRecommend(tags, current_version, 0)
-    print("Latest version is ", v.latest_version)
-    print("Maintain version is", v.maintain_version)
+    print("known release tags :", pkg_tags)
+    print("Latest version is ", ver_rec.latest_version)
+    print("Maintain version is", ver_rec.maintain_version)

advisors/simple-update-robot.py

@@ -22,6 +22,7 @@ import os.path
 import re
 import datetime
 
+import oa_upgradable
 import version_recommend
 
 
@@ -74,14 +75,14 @@ def update_check(spec, o_ver, n_ver):
     if(ver_type.compare(n_ver, o_ver) == 1):
         return True
     else:
-        print("Please check version of {pkg} will upgrade to, it's current version is {version}.".format(
-            pkg=spec.name, version=o_ver))
+        print("Update failed >> [{pkg}: current_ver:{cur_ver}, upgraded_ver:{upd_ver}]".format(
+            pkg=spec.name, cur_ver=o_ver, upd_ver=n_ver))
         return False
 
 
 def fork_clone_repo(gt, repo):
     """
-    Fork repo from src-openEuler to private repository and clone it to local
+    Fork repo from src-openEuler to private repository, then clone it to local
     """
     if not gt.fork_repo(repo):
         print("The repo of {pkg} seems to have been forked.".format(pkg=repo))
@@ -138,41 +139,132 @@ def create_spec(repo, spec_str, o_ver, n_ver, src_fn=None):
     fn.close()
 
 
+def auto_update_pkg(gt, u_branch, u_pkg):
+    """
+    Auto upgrade based on given branch for single package
+    """
+    spec_str = gt.get_spec(u_pkg, u_branch)
+    if not spec_str:
+        print("{pkg}.spec can't be found on the {br} branch. ".format(
+            pkg=u_pkg, br=u_branch))
+        sys.exit(1)
+    pkg_spec = Spec.from_string(spec_str)
+    pkg_ver = replace_macros(pkg_spec.version, pkg_spec)
+
+    pkg_tags = oa_upgradable.get_ver_tags(gt, u_pkg)
+    if pkg_tags is None:
+        sys.exit(1)
+    ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0)
+    rec_up_ver = pkg_ver
+    if re.search("master", u_branch):
+        rec_up_ver = ver_rec.latest_version
+    elif re.search("LTS", u_branch):
+        rec_up_ver = ver_rec.maintain_version
+    else:
+        print("Only support master and LTS version upgrade.")
+        sys.exit(1)
+
+    fork_clone_repo(gt, u_pkg)
+
+    if not update_check(pkg_spec, pkg_ver, rec_up_ver):
+        sys.exit(1)
+
+    if not download_src(gt, pkg_spec, pkg_ver, rec_up_ver):
+        sys.exit(1)
+
+    create_spec(u_pkg, spec_str, pkg_ver, rec_up_ver)
+
+
+def auto_update_repo(gt, u_branch, u_repo):
+    """
+    Auto upgrade based on given branch for packages in given repository
+    """
+    repo_yaml = gt.get_community(u_repo)
+    if not repo_yaml:
+        print("{repo}.yaml in community is empty.".format(repo=u_repo))
+        sys.exit(1)
+
+    pkg_info = yaml.load(repo_yaml, Loader=yaml.loader)
+    pkg_list = pkg_info.get("repositories")
+    for pkg in pkg_list:
+        pkg_name = pkg.get("name")
+        spec_str = gt.get_spec(pkg_name, u_branch)
+        if not spec_str:
+            print("{pkg}.spec can't be found on the {br} branch. ".format(
+                pkg=pkg_name, br=u_branch))
+            continue
+        pkg_spec = Spec.from_string(spec_str)
+        pkg_ver = replace_macros(pkg_spec.version, pkg_spec)
+
+        pkg_tags = oa_upgradable.get_ver_tags(gt, pkg_name)
+        if pkg_tags is None:
+            continue
+        ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0)
+        rec_up_ver = pkg_ver
+        if re.search("master", u_branch):
+            rec_up_ver = ver_rec.latest_version
+        elif re.search("LTS", u_branch):
+            rec_up_ver = ver_rec.maintain_version
+        else:
+            print("Only support master and LTS version upgrade.")
+            sys.exit(1)
+
+        fork_clone_repo(gt, pkg_name)
+
+        if not update_check(pkg_spec, pkg_ver, rec_up_ver):
+            continue
+
+        if not download_src(gt, pkg_spec, pkg_ver, rec_up_ver):
+            continue
+
+        create_spec(pkg_name, spec_str, pkg_ver, rec_up_ver)
+
+
 if __name__ == "__main__":
     pars = argparse.ArgumentParser()
-    pars.add_argument("pkg", type=str, help="The package to be upgraded")
+    pars.add_argument("repo_pkg", type=str, help="The repository or package to be upgraded")
+    pars.add_argument("branch", type=str, help="The branch that upgrade based")
+    pars.add_argument("-u", "--update", type=str, help="Auto upgrade for packages in repository or single package",
+            choices=["repo", "pkg"])
     pars.add_argument("-n", "--new_version", type=str, help="New upstream version of package will be upgrade to")
     pars.add_argument("-s", "--create_spec", help="Create spec file", action="store_true")
     pars.add_argument("-d", "--download", help="Download upstream source code", action="store_true")
-    pars.add_argument("-fc", "--FC", help="fork src-openeuler repo into users and clone to local", action="store_true")
+    pars.add_argument("-fc", "--fork_then_clone", help="Fork src-openeuler repo into users, then clone to local",
+            action="store_true")
     pars.add_argument("-p", "--PR", help="Create upgrade PR", action="store_true")
     args = pars.parse_args()
-
-    if not args.pkg:
-        print("Please specify the package will upgrade.")
-        sys.exit(1)
     
     user_gitee = gitee.Gitee()
-    spec_string = user_gitee.get_spec(args.pkg)
-    spec_file = Spec.from_string(spec_string)
-    cur_ver = replace_macros(spec_file.version, spec_file)
 
-    if args.FC:
-        fork_clone_repo(user_gitee, args.pkg)
-    
-    if args.download or args.create_spec:
-        if not args.new_version:
-            print("Please specify the upgraded version of the {repo}".format(repo=args.pkg))
-            sys.exit(1)
-        elif not update_check(spec_file, cur_ver, args.new_version):
+    if args.update:
+        if args.update == "repo":
+            auto_update_repo(user_gitee, args.branch, args.repo_pkg)
+        else:
+            auto_update_pkg(user_gitee, args.branch, args.repo_pkg)
+    else:
+        spec_string = user_gitee.get_spec(args.repo_pkg, args.branch)
+        if not spec_string:
+            print("{pkg}.spec can't be found on the {br} branch. ".format(pkg=args.repo_pkg, br=args.branch))
             sys.exit(1)
+        spec_file = Spec.from_string(spec_string)
+        cur_version = replace_macros(spec_file.version, spec_file)
 
-    if args.download:
-        if not download_src(user_gitee, spec_file, cur_ver, args.new_version):
-            sys.exit(1)
+        if args.fork_then_clone:
+            fork_clone_repo(user_gitee, args.repo_pkg)
+
+        if args.download or args.create_spec:
+            if not args.new_version:
+                print("Please specify the upgraded version of the {repo}".format(repo=args.repo_pkg))
+                sys.exit(1)
+            elif not update_check(spec_file, cur_version, args.new_version):
+                sys.exit(1)
+
+        if args.download:
+            if not download_src(user_gitee, spec_file, cur_version, args.new_version):
+                sys.exit(1)
 
-    if args.create_spec:
-        create_spec(args.pkg, spec_string, cur_ver, args.new_version)
+        if args.create_spec:
+            create_spec(args.repo_pkg, spec_string, cur_version, args.new_version)
 
-    if args.PR:
-        user_gitee.create_pr(user_gitee.token["user"], args.pkg)
+        if args.PR:
+            user_gitee.create_pr(user_gitee.token["user"], args.repo_pkg)