!52 Bugfix and enhancement

Merge pull request !52 from LeoFang/master

advisors/check_upstream.py

@@ -36,8 +36,8 @@ def load_last_query_result(info, force_reload=False):
         else:
             return ""
 
-def clean_tags(tags, info):
 
+def clean_tags(tags, info):
     if info.get("tag_pattern", "") != "" and info.get("tag_pattern", "") is not None:
         pattern_regex = re.compile(info["tag_pattern"])
         result_list = [pattern_regex.sub("\\1", x) for x in tags]
@@ -50,7 +50,7 @@ def clean_tags(tags, info):
     if info.get("separator", ".") != "." and info.get("separator", ".") is not None:
         separator_regex = re.compile(info["separator"])
         result_list = [separator_regex.sub(".", x) for x in result_list]
-
+    
     # Xinwei used to mis-spell 'separator'. 
     # Followings are kept for compatability until all yaml files are fixed.
     if info.get("seperator", ".") != "." and info.get("seperator", ".") is not None:
@@ -58,7 +58,7 @@ def clean_tags(tags, info):
         result_list = [separator_regex.sub(".", x) for x in result_list]
 
     result_list = [x for x in result_list if x[0].isdigit()]
-
+    
     return result_list
 
 

advisors/gitee.py

@@ -26,12 +26,13 @@ class Gitee(object):
 
         self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW 64; rv:50.0) Gecko/20100101 Firefox/50.0'}
         self.gitee_url = "https://gitee.com/"
-        self.src_openeuler_url = self.gitee_url + "src-openeuler/{package}/raw/master/"
+        self.src_openeuler_url = self.gitee_url + "src-openeuler/{package}/raw/{branch}/"
         self.advisor_url = self.gitee_url + "openeuler/openEuler-Advisor/raw/master/"
         self.specfile_url_template = self.src_openeuler_url + "{specfile}"
         self.yamlfile_url_template = self.src_openeuler_url + "{package}.yaml"
         #self.advisor_url_template = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/{package}.yaml"
         self.advisor_url_template = self.advisor_url + "upstream-info/{package}.yaml"
+        self.community_url_template = self.gitee_url + "openeuler/community/raw/master/repository/{repository}.yaml"
         #self.specfile_exception_url = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/helper/specfile_exceptions.yaml"
         self.specfile_exception_url = self.advisor_url + "advisors/helper/specfile_exceptions.yaml"
         self.time_format = "%Y-%m-%dT%H:%M:%S%z"
@@ -96,7 +97,7 @@ Yours openEuler-Advisor.
 
     def get_gitee_json(self, url):
         """
-        get and load gitee json response
+        Get and load gitee json response
         """
         headers = self.headers.copy()
         #headers = {}
@@ -106,33 +107,31 @@ Yours openEuler-Advisor.
 
     def get_spec_exception(self):
         """
-        get well known spec file exceptions
+        Get well known spec file exceptions
         """
         resp = self.get_gitee(self.specfile_exception_url)
         exps = yaml.load(resp, Loader=yaml.Loader)
         return exps
 
-    def get_spec(self, pkg):
+    def get_spec(self, pkg, br="master"):
         """
-        get openeuler spec file for specific package
+        Get openeuler spec file for specific package
         """
-        specurl = self.specfile_url_template.format(package=pkg, specfile=pkg + ".spec")
+        specurl = self.specfile_url_template.format(branch=br, package=pkg, specfile=pkg + ".spec")
         exp = self.get_spec_exception()
         if pkg in exp:
             dir_name = exp[pkg]["dir"]
             file_name = exp[pkg]["file"]
             specurl = urllib.parse.urljoin(specurl, os.path.join(dir_name, file_name))
-
         try:
             resp = self.get_gitee(specurl)
         except urllib.error.HTTPError:
             resp = ""
-
         return resp
 
-    def get_yaml(self, pkg):
+    def get_yaml(self, pkg, br="master"):
         """
-        get upstream yaml metadata for specific package
+        Get upstream yaml metadata for specific package
         """
         yamlurl = self.advisor_url_template.format(package=pkg)
         try:
@@ -140,19 +139,30 @@ Yours openEuler-Advisor.
         except urllib.error.HTTPError:
             resp = "Not found"
         if re.match("Not found", resp):
-            yamlurl = self.yamlfile_url_template.format(package=pkg)
+            yamlurl = self.yamlfile_url_template.format(branch=br, package=pkg)
             try:
                 resp = self.get_gitee(yamlurl)
             except urllib.error.HTTPError:
                 resp = "Not found"
             if re.match("Not found", resp):
-                print("Cannot find upstream metadata")
+                print("Cann't find yaml metadata for {package} from upstream-info.".format(package=pkg))
                 return False
             else:
                 return resp
         else:
             return resp
 
+    def get_community(self, repo):
+        """
+        Get yaml data from community repo
+        """
+        yamlurl = self.community_url_template.format(repository=repo)
+        try:
+            resp = self.get_gitee(yamlurl)
+        except urllib.error.HTTPError:
+            resp = ""
+        return resp
+
     def get_issues(self, pkg, prj="src-openeuler"):
         """
         List all open issues of pkg

advisors/helper/ver_rec_excpt.yaml

+---
+# version recommend exception list
+gegl04:
+    - '20001120.v002'
+gimp:
+    - '19990910'
+nss:
+    - '334.20030307'
+glibc:
+    - '9000'

advisors/oa_upgradable.py

@@ -12,10 +12,72 @@ import os
 import argparse
 
 import urllib.error
+
 import gitee
 import check_upstream
 import version_recommend
 
+
+def _get_rec_excpt():
+    """
+    Get except case of version recommend
+    """
+    y_file = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "helper/ver_rec_excpt.yaml"))
+    excpt = yaml.load(y_file, Loader=yaml.Loader)
+    return excpt
+
+
+def _filter_except(excpts, sources):
+    """
+    Filter except case in sources
+    """
+    for e in excpts:
+        sources = [s for s in sources if e not in s]
+    return sources
+    
+
+def get_ver_tags(gt, repo, cwd_path=None):
+    """
+    Get version tags of given package
+    """
+    if cwd_path:
+        try:
+            repo_yaml = open(os.path.join(cwd_path, repo + ".yaml")).read()
+        except FileNotFoundError:
+            print("Cann't find yaml metadata for {pkg} from current working directory.".format(pkg=repo))
+            repo_yaml = gt.get_yaml(repo)
+    else:
+        repo_yaml = gt.get_yaml(repo)
+
+    if repo_yaml:
+        pkg_info = yaml.load(repo_yaml, Loader=yaml.Loader)
+    else:
+        return None
+
+    vc_type = pkg_info["version_control"]
+    if vc_type == "hg":
+        tags = check_upstream.check_hg(pkg_info)
+    elif vc_type == "github":
+        tags = check_upstream.check_github(pkg_info)
+    elif vc_type == "git":
+        tags = check_upstream.check_git(pkg_info)
+    elif vc_type == "gitlab.gnome":
+        tags = check_upstream.check_gnome(pkg_info)
+    elif vc_type == "svn":
+        tags = check_upstream.check_svn(pkg_info)
+    elif vc_type == "metacpan":
+        tags = check_upstream.check_metacpan(pkg_info)
+    elif vc_type == "pypi":
+        tags = check_upstream.check_pypi(pkg_info)
+    else:
+        print("Unsupport version control method {vc}".format(vc=vc_type))
+
+    excpt_list = _get_rec_excpt()
+    if repo in excpt_list:
+        tags = _filter_except(excpt_list[repo], tags) 
+    return tags
+
+
 if __name__ == "__main__":
     parameters = argparse.ArgumentParser()
     parameters.add_argument("-p", "--push", action="store_true",
@@ -27,56 +89,24 @@ if __name__ == "__main__":
 
     args = parameters.parse_args()
 
-    gitee = gitee.Gitee()
-    prj_name = args.repo
-    spec_string = gitee.get_spec(prj_name)
+    user_gitee = gitee.Gitee()
+    spec_string = user_gitee.get_spec(args.repo)
     if not spec_string:
-        print("{repo} seems to be an empty repository".format(repo=args.repo))
+        print("{pkg}.spec can't be found on the master branch".format(pkg=args.repo))
         sys.exit(1)
 
-    s_spec = Spec.from_string(spec_string)
-
-    current_version = replace_macros(s_spec.version, s_spec)
-
-    print("Checking ", prj_name)
-    print("current version is ", current_version)
-
-    try:
-        prj_info_string = open(os.path.join(args.default, prj_name + ".yaml")).read()
-    except FileNotFoundError:
-        prj_info_string = ""
-
-    if not prj_info_string:
-        print("Get YAML info from gitee")
-        try:
-            prj_info_string = gitee.get_yaml(prj_name)
-        except urllib.error.HTTPError:
-            print("Failed to get YAML info for {pkg}".format(pkg=prj_name))
-            sys.exit(1)
+    spec_file = Spec.from_string(spec_string)
+    cur_version = replace_macros(spec_file.version, spec_file)
 
+    print("Checking ", args.repo)
+    print("current version is ", cur_version)
 
-    prj_info = yaml.load(prj_info_string, Loader=yaml.Loader)
+    pkg_tags = get_ver_tags(user_gitee, args.repo, args.default)
+    print("known release tags:", pkg_tags)
 
-    vc_type = prj_info["version_control"]
-    if vc_type == "hg":
-        tags = check_upstream.check_hg(prj_info)
-    elif vc_type == "github":
-        tags = check_upstream.check_github(prj_info)
-    elif vc_type == "git":
-        tags = check_upstream.check_git(prj_info)
-    elif vc_type == "gitlab.gnome":
-        tags = check_upstream.check_gnome(prj_info)
-    elif vc_type == "svn":
-        tags = check_upstream.check_svn(prj_info)
-    elif vc_type == "metacpan":
-        tags = check_upstream.check_metacpan(prj_info)
-    elif vc_type == "pypi":
-        tags = check_upstream.check_pypi(prj_info)
-    else:
-        print("Unsupport version control method {vc}".format(vc=vc_type))
+    if pkg_tags is None:
         sys.exit(1)
+    ver_rec = version_recommend.VersionRecommend(pkg_tags, cur_version, 0)
 
-    print("known release tags :", tags)
-    v = version_recommend.VersionRecommend(tags, current_version, 0)
-    print("Latest version is ", v.latest_version)
-    print("Maintain version is", v.maintain_version)
+    print("Latest version is", ver_rec.latest_version)
+    print("Maintain version is", ver_rec.maintain_version)

advisors/simple-update-robot.py

@@ -21,8 +21,11 @@ import subprocess
 import os.path
 import re
 import datetime
+
+import oa_upgradable
 import version_recommend
 
+
 def download_source_url(spec, o_ver, n_ver):
     """
     Download source file from Source or Source0 URL
@@ -59,10 +62,61 @@ def download_upstream_url(gt, repo, n_ver):
         return False
 
 
+def update_check(spec, o_ver, n_ver):
+    """
+    Requirements check for upgraded package
+    """
+    if len(spec.patches) >= 1:
+        print("I'm too naive to handle complicated package.")
+        print("This package has multiple in-house patches.")
+        return False
+
+    ver_type = version_recommend.VersionType()
+    if(ver_type.compare(n_ver, o_ver) == 1):
+        return True
+    else:
+        print("Update failed >> [{pkg}: current_ver:{cur_ver}, upgraded_ver:{upd_ver}]".format(
+            pkg=spec.name, cur_ver=o_ver, upd_ver=n_ver))
+        return False
+
+
+def fork_clone_repo(gt, repo):
+    """
+    Fork repo from src-openEuler to private repository, then clone it to local
+    """
+    if not gt.fork_repo(repo):
+        print("The repo of {pkg} seems to have been forked.".format(pkg=repo))
+    
+    name = gt.token["user"]
+    subprocess.call(["git", "clone", "git@gitee.com:{user}/{pkg}".format(user=name, pkg=repo)])
+    os.chdir(repo)
+
+
+def download_src(gt, spec, o_ver, n_ver):
+    """
+    Download source code for upgraded package
+    """
+    source_file = download_source_url(spec, o_ver, n_ver)
+    if source_file:
+        print(source_file)
+        return True
+    else:
+        source_file = download_upstream_url(gt, spec.name, n_ver)
+        if source_file:
+            print(source_file)
+            return True
+        else:
+            print("Failed to download the latest source code.")
+            return False
+
+
 def create_spec(repo, spec_str, o_ver, n_ver, src_fn=None):
     """
     Create new spec file for upgraded package
     """
+    fn = open(repo + "_old.spec", "w")
+    fn.write(spec_str)
+    fn.close()
     fn = open(repo + ".spec", "w")
     in_changelog = False
     for l in spec_str.splitlines():
@@ -89,55 +143,133 @@ def create_spec(repo, spec_str, o_ver, n_ver, src_fn=None):
             fn.write("\n")
     fn.close()
 
+
+def auto_update_pkg(gt, u_branch, u_pkg):
+    """
+    Auto upgrade based on given branch for single package
+    """
+    spec_str = gt.get_spec(u_pkg, u_branch)
+    if not spec_str:
+        print("{pkg}.spec can't be found on the {br} branch. ".format(
+            pkg=u_pkg, br=u_branch))
+        sys.exit(1)
+    pkg_spec = Spec.from_string(spec_str)
+    pkg_ver = replace_macros(pkg_spec.version, pkg_spec)
+
+    pkg_tags = oa_upgradable.get_ver_tags(gt, u_pkg)
+    if pkg_tags is None:
+        sys.exit(1)
+    ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0)
+    rec_up_ver = pkg_ver
+    if re.search("master", u_branch):
+        rec_up_ver = ver_rec.latest_version
+    elif re.search("LTS", u_branch):
+        rec_up_ver = ver_rec.maintain_version
+    else:
+        print("Only support master and LTS version upgrade.")
+        sys.exit(1)
+
+    fork_clone_repo(gt, u_pkg)
+
+    if not update_check(pkg_spec, pkg_ver, rec_up_ver):
+        sys.exit(1)
+
+    if not download_src(gt, pkg_spec, pkg_ver, rec_up_ver):
+        sys.exit(1)
+
+    create_spec(u_pkg, spec_str, pkg_ver, rec_up_ver)
+
+
+def auto_update_repo(gt, u_branch, u_repo):
+    """
+    Auto upgrade based on given branch for packages in given repository
+    """
+    repo_yaml = gt.get_community(u_repo)
+    if not repo_yaml:
+        print("{repo}.yaml in community is empty.".format(repo=u_repo))
+        sys.exit(1)
+
+    pkg_info = yaml.load(repo_yaml, Loader=yaml.loader)
+    pkg_list = pkg_info.get("repositories")
+    for pkg in pkg_list:
+        pkg_name = pkg.get("name")
+        spec_str = gt.get_spec(pkg_name, u_branch)
+        if not spec_str:
+            print("{pkg}.spec can't be found on the {br} branch. ".format(
+                pkg=pkg_name, br=u_branch))
+            continue
+        pkg_spec = Spec.from_string(spec_str)
+        pkg_ver = replace_macros(pkg_spec.version, pkg_spec)
+
+        pkg_tags = oa_upgradable.get_ver_tags(gt, pkg_name)
+        if pkg_tags is None:
+            continue
+        ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0)
+        rec_up_ver = pkg_ver
+        if re.search("master", u_branch):
+            rec_up_ver = ver_rec.latest_version
+        elif re.search("LTS", u_branch):
+            rec_up_ver = ver_rec.maintain_version
+        else:
+            print("Only support master and LTS version upgrade.")
+            sys.exit(1)
+
+        fork_clone_repo(gt, pkg_name)
+
+        if not update_check(pkg_spec, pkg_ver, rec_up_ver):
+            continue
+
+        if not download_src(gt, pkg_spec, pkg_ver, rec_up_ver):
+            continue
+
+        create_spec(pkg_name, spec_str, pkg_ver, rec_up_ver)
+
+
 if __name__ == "__main__":
     pars = argparse.ArgumentParser()
-    pars.add_argument("pkg", type=str, help="The package to be upgraded")
+    pars.add_argument("repo_pkg", type=str, help="The repository or package to be upgraded")
+    pars.add_argument("branch", type=str, help="The branch that upgrade based")
+    pars.add_argument("-u", "--update", type=str, help="Auto upgrade for packages in repository or single package",
+            choices=["repo", "pkg"])
     pars.add_argument("-n", "--new_version", type=str, help="New upstream version of package will be upgrade to")
     pars.add_argument("-s", "--create_spec", help="Create spec file", action="store_true")
     pars.add_argument("-d", "--download", help="Download upstream source code", action="store_true")
-    pars.add_argument("-f", "--fork", help="fork src-openeuler repo into users", action="store_true")
-    pars.add_argument("-c", "--clone", help="clone privatge repo to local", action="store_true")
+    pars.add_argument("-fc", "--fork_then_clone", help="Fork src-openeuler repo into users, then clone to local",
+            action="store_true")
     pars.add_argument("-p", "--PR", help="Create upgrade PR", action="store_true")
     args = pars.parse_args()
+    
+    user_gitee = gitee.Gitee()
 
-    my_gitee = gitee.Gitee()
-    my_version = version_recommend.VersionType()
-    spec_string= my_gitee.get_spec(args.pkg)
-
-    s_spec = Spec.from_string(spec_string)
-    cur_ver = replace_macros(s_spec.version, s_spec)
+    if args.update:
+        if args.update == "repo":
+            auto_update_repo(user_gitee, args.branch, args.repo_pkg)
+        else:
+            auto_update_pkg(user_gitee, args.branch, args.repo_pkg)
+    else:
+        spec_string = user_gitee.get_spec(args.repo_pkg, args.branch)
+        if not spec_string:
+            print("{pkg}.spec can't be found on the {br} branch. ".format(pkg=args.repo_pkg, br=args.branch))
+            sys.exit(1)
+        spec_file = Spec.from_string(spec_string)
+        cur_version = replace_macros(spec_file.version, spec_file)
 
-    if args.fork:
-        if not my_gitee.fork_repo(args.pkg):
-            print("The repo of {pkg} seems to have been forked.".format(pkg=args.pkg))
+        if args.fork_then_clone:
+            fork_clone_repo(user_gitee, args.repo_pkg)
 
-    if args.clone:
-        user=my_gitee.token["user"]
-        subprocess.call(["git", "clone", "git@gitee.com:{user}/{pkg}".format(user=user, pkg=args.pkg)])
-        os.chdir(args.pkg)
+        if args.download or args.create_spec:
+            if not args.new_version:
+                print("Please specify the upgraded version of the {repo}".format(repo=args.repo_pkg))
+                sys.exit(1)
+            elif not update_check(spec_file, cur_version, args.new_version):
+                sys.exit(1)
 
-    if args.download:
-        source_file = download_source_url(s_spec, cur_ver, args.new_version)
-        if source_file:
-            print(source_file)
-        else:
-            source_file = download_upstream_url(my_gitee, args.pkg, args.new_version)
-            if source_file:
-                print(source_file)
-            else:
-                print("Failed to download the latest source code.")
+        if args.download:
+            if not download_src(user_gitee, spec_file, cur_version, args.new_version):
                 sys.exit(1)
 
-    if args.create_spec:
-        if len(s_spec.patches) >= 1:
-            print("I'm too naive to handle complicated package.")
-            print("This package has multiple in-house patches.")
-            sys.exit(1)
-        if(my_version.compare(args.new_version, cur_ver) ==1):
-            create_spec(args.pkg, spec_string, cur_ver, args.new_version)
-        else:
-            print("Please check version of {pkg} will upgrade to, it's current version is {version}.".format(
-				pkg=args.pkg, version=cur_ver))
+        if args.create_spec:
+            create_spec(args.repo_pkg, spec_string, cur_version, args.new_version)
 
-    if args.PR:
-        my_gitee.create_pr(my_gitee.token["user"], args.pkg)
+        if args.PR:
+            user_gitee.create_pr(user_gitee.token["user"], args.repo_pkg)

advisors/version_recommend.py

@@ -72,7 +72,7 @@ class VersionType(object):
         :return 0: z1 equal then z2
         :raises: None
         """
-        return self._compare(self, z1, z2)
+        return self._compare(z1, z2)
 
     def _compare(self, z1, z2):
         """
@@ -91,11 +91,15 @@ class VersionType(object):
         len2 = len(d2)
         length = min(len1, len2)
         for index in range(length):
-            if d1[index].isdigit() and d1[index].isdigit():
+            if d1[index].isdigit() and d2[index].isdigit():
                 if int(d1[index]) > int(d2[index]):
                     return 1
                 elif int(d1[index]) < int(d2[index]):
                     return -1
+            elif d1[index].isdigit():
+                return 1
+            elif d2[index].isdigit():
+                return -1
             else:
                 if d1[index] > d2[index]:
                     return 1
@@ -126,7 +130,7 @@ class VersionType(object):
         :returns: The split result
         :raises: None
         """
-        for f, s in re.findall(r'([\d]+)|([^\d.]+)', x):
+        for f, s in re.findall(r'([\d]+)|([^\d.-]+)', x):
             if f:
                 float(f)
                 yield f
@@ -1085,6 +1089,10 @@ class VersionRecommend(object):
         if m is None:  # 版本号应该是数字开头
             return False
 
+        m = re.search(r'[ab]\d', version)
+        if not m is None:
+            return False
+
         if 'rc' in version \
                 or 'RC' in version \
                 or 'dev' in version \

upstream-info/python2.yaml

+---
+version_control: github
+src_repo: python/cpython
+tag_prefix: "^v?3*|^v"
+seperator: "."
+url: https://github.com/python/cpython.git

upstream-info/python3.yaml

+---
+version_control: github
+src_repo: python/cpython
+tag_prefix: "^v"
+seperator: "."
+url: https://github.com/python/cpython.git