!80 修复yaml错误以及工具处理逻辑问题

Merge pull request !80 from 木得感情的openEuler机器人/yaml_fix

advisors/check_upstream.py

@@ -41,6 +41,9 @@ def clean_tags(tags, info):
     if info.get("tag_pattern", "") != "" and info.get("tag_pattern", "") is not None:
         pattern_regex = re.compile(info["tag_pattern"])
         result_list = [pattern_regex.sub("\\1", x) for x in tags]
+    elif info.get("tag_reorder", "") != "" and info.get("tag_reorder", "") is not None:
+        pattern_regex = re.compile(info["tag_reorder"])
+        result_list = [pattern_regex.sub(info["tag_neworder"], x) for x in tags]
     elif info.get("tag_prefix", "") != "" and info.get("tag_prefix", "") is not None:
         prefix_regex = re.compile(info["tag_prefix"])
         result_list = [prefix_regex.sub("", x) for x in tags]
@@ -81,8 +84,40 @@ def dirty_redirect_tricks(url, resp):
     if "" in cookie: cookie.remove("") 
     return need_trick, new_url, list(cookie)
 
+def check_hg_raw(info):
+    eprint("{repo} > Using hg raw-tags".format(repo=info["src_repo"]+"/raw-tags"))
+    resp = load_last_query_result(info)
+    if resp == "":
+        headers = {
+                'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)'
+                }
+        url = urljoin(info["src_repo"] + "/", "raw-tags")
+        resp = requests.get(url, headers=headers)
+        resp = resp.text
+        need_trick, url, cookie = dirty_redirect_tricks(url, resp)
+        if need_trick:
+            # I dont want to introduce another dependency on requests
+            # but urllib handling cookie is outragely complex
+            c_dict = {}
+            for c in cookie:
+                k, v = c.split('=')
+                c_dict[k] = v
+            resp = requests.get(url, headers=headers, cookies=c_dict)
+            resp = resp.text
+
+    last_query = {}
+    last_query["time_stamp"] = datetime.now()
+    last_query["raw_data"] = resp
+    info["last_query"] = last_query
+    tags = []
+    for l in resp.splitlines():
+        tags.append(l.split()[0])
+    result_list = clean_tags(tags, info)
+    return result_list
+
 
 def check_hg(info):
+    eprint("{repo} > Using hg json-tags".format(repo=info["src_repo"]+"/json-tags"))
     resp = load_last_query_result(info)
     if resp == "":
         headers = {
@@ -114,6 +149,7 @@ def check_hg(info):
     result_list = clean_tags(result_list, info)
     return result_list
 
+
 def check_metacpan(info):
     resp = load_last_query_result(info)
     if resp == "":
@@ -140,6 +176,7 @@ def check_metacpan(info):
     last_query["time_stamp"] = datetime.now()
     last_query["raw_data"] = resp
     info["last_query"] = last_query
+    tags = clean_tags(tags, info)
     return tags
 
 def check_pypi(info):
@@ -233,9 +270,31 @@ def check_github(info):
     tags = clean_tags(tags, info)
     return tags
 
+def check_gnu_ftp(info):
+    headers = {
+            'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)'
+            }
+    url = urljoin("https://ftp.gnu.org/gnu/", info["src_repo"] + "/")
+    eprint("{repo} > List ftp directory".format(repo=url))
+    resp = requests.get(url, headers=headers)
+    resp = resp.text
+    re_pattern = re.compile("href=\"(.*)\">(\\1)</a>")
+    tags = []
+    for l in resp.splitlines():
+        m = re_pattern.search(l)
+        if m:
+            tags.append(m[1])
+    tags = clean_tags(tags, info)
+    return tags
+
+
 def check_gnome(info):
     resp = load_last_query_result(info)
-    repo_url = "https://gitlab.gnome.org/GNOME/"+info["src_repo"]+".git"
+    src_repos = info["src_repo"].split("/")
+    if len(src_repos) == 1:
+        repo_url = "https://gitlab.gnome.org/GNOME/" + info["src_repo"] + ".git"
+    else:
+        repo_url = "https://gitlab.gnome.org/" + info["src_repo"] + ".git"
 
     if resp == "":
         resp = __check_git_helper(repo_url)
@@ -264,7 +323,8 @@ def check_gitee(info):
 
 def check_svn(info):
     resp = load_last_query_result(info)
-    repo_url = info["src_repo"] + "/tags"
+    tag_dir = info.get("tag_dir", "tags")
+    repo_url = info["src_repo"] + "/" + tag_dir
     if resp == "":
         resp = __check_svn_helper(repo_url)
         last_query = {}

advisors/oa_upgradable.py

@@ -60,6 +60,8 @@ def get_ver_tags(gt, repo, cwd_path=None):
         return None
     if vc_type == "hg":
         tags = check_upstream.check_hg(pkg_info)
+    elif vc_type == "hg-raw":
+        tags = check_upstream.check_hg_raw(pkg_info)
     elif vc_type == "github":
         tags = check_upstream.check_github(pkg_info)
     elif vc_type == "git":
@@ -74,6 +76,8 @@ def get_ver_tags(gt, repo, cwd_path=None):
         tags = check_upstream.check_pypi(pkg_info)
     elif vc_type == "gitee":
         tags = check_upstream.check_gitee(pkg_info)
+    elif vc_type == "gnu-ftp":
+        tags = check_upstream.check_gnu_ftp(pkg_info)
     else:
         print("Unsupport version control method {vc}".format(vc=vc_type))
         return None

upstream-info/CUnit.yaml

 version_control: svn                                                                                              
-src_repo: https://svn.code.sf.net/p/cunit/code/trunk                                                                                              
+src_repo: https://svn.code.sf.net/p/cunit/code
 tag_prefix:                                                                                              
 separator: 

upstream-info/SDL.yaml

-version_control: hg
+version_control: hg-raw
 src_repo: http://hg.libsdl.org/SDL
-tag_prefix: "release-"
+tag_prefix: "release-(2)?"
 separator: "."

upstream-info/SDL2.yaml

-version_control: hg
+version_control: hg-raw
 src_repo: http://hg.libsdl.org/SDL
 tag_prefix: ^release-
 separator: .

upstream-info/acpica-tools.yaml

 version_control: github
 src_repo: acpica/acpica
-tag_prefix: "^R"
+tag_reorder: "^R(\\d\\d)_?(\\d\\d)_?(\\d\\d)"
+tag_neworder: "20\\3\\1\\2"
 separator: "_"

upstream-info/apr-util.yaml

 version_control: svn
-src_repo: https://svn.apache.org/viewvc/apr/apr-util
+src_repo: https://svn.apache.org/repos/asf/apr/apr-util
 tag_prefix: ^
 separator: .

upstream-info/apr.yaml

 version_control: svn
-src_repo: https://svn.apache.org/viewvc/apr/apr
+src_repo: https://svn.apache.org/repos/asf/apr/apr
 tag_prefix: ^
 separator: .

upstream-info/atk.yaml

 ---
 version_control: gitlab.gnome
 src_repo: atk
-tag_prefix: "^v"
-separator: "."
+tag_prefix: "^ATK_"
+separator: "_"
 last_query:
   time_stamp: 2020-04-24 13:50:12.668396540 +00:00
   raw_data: |

upstream-info/bc.yaml

+version_control: gnu-ftp
+src_repo: bc
+tag_pattern: bc-(.*).tar.gz(.sig)?
+seperator: "."

upstream-info/bridge-utils.yaml

+version_control: git
+src_repo: https://git.kernel.org/pub/scm/linux/kernel/git/shemminger/bridge-utils.git
+tag_prefix: '^v'
+seperator: .

upstream-info/celt051.yaml

+version_control: NA
+rpm_repo: https://git.centos.org/rpms/celt051.git
+tag_prefix: "^celt051-"
+seperator: "."

upstream-info/ctags.yaml

 version_control: svn                                                                                              
-src_repo: https://svn.code.sf.net/p/ctags/code/trunk                                                                                             
+src_repo: https://svn.code.sf.net/p/ctags/code
 tag_prefix: "^Ctags-"                                                                                              
 separator: "_"

upstream-info/ed.yaml

+version_control: gnu-ftp
+src_repo: ed
+tag_pattern: ed-(.*).(tar.*)
+seperator: "."

upstream-info/edac-utils.yaml

 version_control: svn                                                                                              
-src_repo: https://svn.code.sf.net/p/edac-utils/code/trunk                                                                                              
+src_repo: https://svn.code.sf.net/p/edac-utils/code
 tag_prefix:                                                                                              
 separator: "."

upstream-info/emacs.yaml

+version_control: git
+src_repo: https://git.savannah.gnu.org/git/emacs.git
+tag_prefix: ^emacs-
+seperator: "."

upstream-info/epydoc.yaml

 version_control: svn
-src_repo: https://epydoc.svn.sourceforge.net/svnroot/epydoc/trunk/epydoc
+src_repo: https://svn.code.sf.net/p/epydoc/code
 tag_prefix: ^epydoc-
 separator: .

upstream-info/fabtests.yaml

+version_control: github
+src_repo: ofiwg/fabtests
+tag_prefix: ^v
+seperator: .

upstream-info/gdbm.yaml

 version_control: git 
-src_repo: git://gnu.org.ua/gdbm.git
-tag_prefix: gdbm-
+src_repo: https://git.savannah.gnu.org/git/gdbm.git
+tag_prefix: ^v
 separator: .

upstream-info/gnu-free-fonts.yaml

 version_control: svn
-src_repo: https://svn.savannah.gnu.org/viewvc/freefont
-tag_repo: RELEASE_
+src_repo: svn://svn.savannah.gnu.org/freefont
+tag_prefix: "^RELEASE_"
 separator: .

upstream-info/gperftools.yaml

 version_control: github
 src_repo:  gperftools/gperftools
-tag_prefix: ^v
+tag_prefix: ^gperftools-
 separator: .

upstream-info/guile.yaml

+version_control: git
+src_repo: https://git.savannah.gnu.org/git/guile.git
+tag_prefix: ^v
+seperator: "."

upstream-info/iptables.yaml

+version_control: git
+src_repo: git://git.netfilter.org/iptables.git
+tag_prefix: "^v"
+seperator: .

upstream-info/irrlicht.yaml

 version_control: svn
-src_repo: https://svn.code.sf.net/p/irrlicht/code/trunk
+src_repo: https://svn.code.sf.net/p/irrlicht/code
 tag_prefix: release-
 separator: .

upstream-info/kurdit-unikurd-web-fonts.yaml

-version_control: github
+version_control: NA
 src_repo:  gooselinux/kurdit-unikurd-web-fonts
+note: src_repo here is not the actual repo for font
 tag_prefix: ^
 separator: .

upstream-info/lame.yaml

 version_control: svn
-src_repo: https://svn.code.sf.net/p/lame/svn/trunk/lame
-tag_prefix: RELEASE_
+src_repo: https://svn.code.sf.net/p/lame/svn
+tag_prefix: RELEASE__
 separator: _

upstream-info/libavc1394.yaml

 version_control: svn
-src_repo: https://svn.code.sf.net/p/libavc1394/code/trunk
-tag_prefix: "^v"
-separator: "."
+src_repo: https://svn.code.sf.net/p/libavc1394/code
+tag_prefix: "^V_"
+separator: "_"

upstream-info/libdvdnav.yaml

-version_control: github
-src_repo: mirror/libdvdnav
-tag_pattern: ^v
+version_control: git
+src_repo: https://code.videolan.org/videolan/libdvdnav.git
+tag_prefix: ^v
 separator: .

upstream-info/libdwarf.yaml

 git_url: https://github.com/Distrotech/libdwarf.git
 version_control: github
 src_repo: Distrotech/libdwarf
-tag_prefix: "v"
+tag_prefix: "^"
 separator: ""

upstream-info/libgphoto2.yaml

 version_control: github
 src_repo: gphoto/libgphoto2
-tag_pattern: libgphoto2-
+tag_prefix: libgphoto2-
 separator: _

upstream-info/libiec61883.yaml

 version_control: github
 src_repo: Distrotech/libiec61883
-tag_pattern: ^v
+tag_prefix: ^v
 separator: .

upstream-info/libieee1284.yaml

 version_control: github
 src_repo: twaugh/libieee1284
-tag_pattern: ^v
+tag_prefix: ^v
 separator: _

upstream-info/libisofs.yaml

 version_control: git
-src_repo: https://dev.lovelyhq.com/libburnia
-tag_pattern: release-
+src_repo: https://dev.lovelyhq.com/libburnia/libisofs.git
+tag_prefix: release-
 separator: .

upstream-info/libmodman.yaml

-version_control: svn
-src_repo: https://code.google.com/archive/p/libmodman/source/default/source
+version_control: NA
+svn_url: https://code.google.com/archive/p/libmodman/source/default/source
 tag_prefix: ^libmodman-
 separator: .

upstream-info/libmpeg2.yaml

+version_control: CVS
+src_repo: libmpeg2.cvs.sourceforge.net:/cvsroot/libmpeg2
+tag_prefix: NA
+seperator: NA

upstream-info/liboauth.yaml

-version_control: svn
-src_repo: https://svn.code.sf.net/p/liboauth/code/trunk
-tag_prefix: liboauth-code
-separator: ^liboauth-
+version_control: github
+src_repo: x42/liboauth
+tag_prefix: "^v"
+separator: "."

upstream-info/libpwquality.yaml

 version_control: github
 src_repo:  libpwquality/libpwquality
-tag_prefix: ^v
+tag_prefix: ^libpwquality-
 separator: .

upstream-info/libsigc++20.yaml → upstream-info/libsigcpp20.yaml

 version_control: github
 src_repo: libsigcplusplus/libsigcplusplus
-tag_pattern: ^v
+tag_prefix: "^v?2.99|^3"
 separator: .

upstream-info/libtalloc.yaml

-version_control: github
-src_repo: nccgroup/libtalloc
-tag_prefix: talloc-
+version_control: git
+src_repo: git://git.samba.org/samba.git
+tag_prefix: "^talloc-"
 separator: "."

upstream-info/libwacom.yaml

 version_control: github
 src_repo:  linuxwacom/libwacom
-tag_prefix: ^v
+tag_prefix: ^libwacom-
 separator: .

upstream-info/linux-firmware.yaml

+version_control: git
+src_repo: https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git
+tag_prefix: 
+seperator: 

upstream-info/lshw.yaml

+version_control: git
+src_repo: https://ezix.org/src/pkg/lshw.git 
+tag_prefix: "B."
+seperator: "."

upstream-info/lutok.yaml

+version_control: github
+src_repo: jmmv/lutok
+tag_prefix: lutok-
+separator: "."

upstream-info/mallard-rng.yaml

+version_control: github
+src_repo: projectmallard/projectmallard.org
+tag_prefix: mallard-rng-
+seperator: .

upstream-info/mcpp.yaml

-version_control: svn
-src_repo: https://svn.code.sf.net/p/mcpp/code/trunk/src
-tag_prefix: ^V
+version_control: github
+src_repo: zeroc-ice/mcpp
+tag_prefix: ^v
 separator: .

upstream-info/mod_auth_openidc.yaml

+version_control: github
+src_repo: zmartzone/mod_auth_openidc
+tag_prefix: "^v"
+seperator: .

upstream-info/nss-pem.yaml

 version_control: github
 src_repo:  kdudka/nss-pem
-tag_prefix: ^v
+tag_prefix: ^nss-pem-
 separator: .

upstream-info/nss.yaml

 version_control: hg
 src_repo: https://hg.mozilla.org/projects/nss
-tag_prefix: ^NSS_
+tag_pattern: ^NSS_(.*)_RTM
 separator: _

upstream-info/java-1.8.0-openjdk.yaml → upstream-info/openjdk-1.8.0.yaml

 ---
-version_control: hg
+version_control: hg-raw
 src_repo: http://hg.openjdk.java.net/aarch64-port/jdk8u-shenandoah
 tag_prefix: jdk8u
 separator: "-"

upstream-info/openssl-pkcs11.yaml

 version_control: github
 src_repo:  OpenSC/libp11
-tag_prefix: ^v
+tag_prefix: ^libp11-
 separator: .

upstream-info/python-which.yaml

+git_url: https://github.com/trentm/which.git
+version_control: NA
+src_repo: trentm/which
+tag_prefix: ^
+seperator: .

upstream-info/rapidjson.yaml

+version_control: github
+src_repo: Tencent/rapidjson
+tag_prefix: "^v"
+separator: "."

upstream-info/sharutils.yaml

+version_control: gnu-ftp
+src_repo: sharutils
+tag_pattern: sharutils-(.*).tar.gz(.sig)?
+seperator: "."

upstream-info/spock.yaml

 version_control: github
 src_repo:  spockframework/spock
-tag_prefix: ^v
-separator: .
+tag_prefix: "spock-"
+separator: "-"

upstream-info/syscontainer-tools.yaml

+git_url: https:/gitee.com/openeuler/syscontainer-tools
+version_control: NA
+src_repo: openeuler/syscontainer-tools
+tag_prefix: "^v"
+separator: "."

upstream-info/tar.yaml

+version_control: git
+src_repo: https://git.savannah.gnu.org/git/tar.git
+tag_prefix: "release_"
+separator: "_"

upstream-info/texamator.yaml

+git_url: https://github.com/alexisflesch/texamator.git
+version_control: NA
+src_repo: alexisflesch/texamator
+tag_prefix: "^v"
+separator: "."

upstream-info/trace-cmd.yaml

+version_control: git
+src_repo: https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/trace-cmd.git
+tag_prefix: trace-cmd-v
+seperator: .

upstream-info/which.yaml

+version_control: gnu-ftp
+src_repo: which
+tag_pattern: which-(.*).(tar.*)
+seperator: "."

upstream-info/xmlrpc-c.yaml

+version_control: svn
+src_repo: https://svn.code.sf.net/p/xmlrpc-c/code
+tag_dir: release_number
+tag_prefix: ""
+separator: "."