complete checking methods

advisors/check_upstream.py

@@ -91,47 +91,167 @@ def check_hg(info):
                 k, v = c.split('=')
                 c_dict[k] = v
             resp = requests.get(url, headers=headers, cookies=c_dict)
+            resp = resp.text
 
     last_query = {}
     last_query["time_stamp"] = datetime.now()
-    last_query["raw_data"] = resp.text
+    last_query["raw_data"] = resp
     info["last_query"] = last_query
     # try and except ?
-    tags_json = json.loads(resp.text)
+    tags_json = json.loads(resp)
     sort_tags = tags_json["tags"]
     sort_tags.sort(reverse=True, key=lambda x: x['date'][0])
     result_list = [tag['tag'] for tag in sort_tags]
     result_list = clean_tags(result_list, info)
     return result_list
 
-def check_github(info):
+def check_metacpan(info):
     resp = load_last_query_result(info)
-    if info.get("query_type", "git-ls") != "git-ls":
-        resp = ""
-    cmd_list = ["git", "ls-remote", "--tags", "https://github.com/" + info["src_repo"] + ".git"]
     if resp == "":
-        eprint("{repo} > Using git ls-remote".format(repo=info["src_repo"]))
-        subp = subprocess.Popen(cmd_list, stdout=subprocess.PIPE)
-        resp = subp.stdout.read().decode("utf-8")
-        if subp.wait() != 0:
-            eprint("{repo} > git ls-remote encount errors".format(repo=info["src_repo"]))
+        headers = {
+                'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)'
+                }
+        url = urljoin("https://fastapi.metacpan.org/release/", info["src_repo"])
+        resp = requests.get(url, headers=headers)
+        resp = resp.text
+
+    tags = []
+    result_json = json.loads(resp)
+    if result_json != {}:
+        if "version" not in result_json.keys():
+            eprint("{repo} > ERROR FOUND".format(repo=info["src_repo"]))
             sys.exit(1)
+        else:
+            tags.append(result_json["version"])
+    else:
+        eprint("{repo} found unsorted on cpan.metacpan.org".format(repo=info["src_repo"]))
+        sys.exit(1)
 
-        last_query = {}
-        last_query["time_stamp"] = datetime.now()
-        last_query["raw_data"] = resp
-        info["last_query"] = last_query
-        info["query_type"] = "git-ls"
+    last_query = {}
+    last_query["time_stamp"] = datetime.now()
+    last_query["raw_data"] = resp
+    info["last_query"] = last_query
+    return tags
 
+def check_pypi(info):
+    resp = load_last_query_result(info)
+    tags = []
+    if resp == "":
+        headers = {
+                'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)'
+                }
+        url = urljoin("https://pypi.org/pypi/", info["src_repo"] + "/json")
+        resp = requests.get(url, headers=headers)
+        resp = resp.text
+
+    result_json = json.loads(resp)
+    if result_json != {}:
+        tags.append(result_json["info"]["version"])
+    else:
+        eprint("{repo} > No Response or JSON parse failed".format(repo=info["src_repo"]))
+        sys.exit(1)
+    return tags
+
+def __check_subprocess(cmd_list):
+    subp = subprocess.Popen(cmd_list, stdout=subprocess.PIPE)
+    resp = subp.stdout.read().decode("utf-8")
+    if subp.wait() != 0:
+        eprint("{cmd} > encount errors".format(cmd=" ".join(cmd_list)))
+        sys.exit(1)
+    return resp
+
+def __check_svn_helper(repo_url):
+    eprint("{repo} > Using svn ls".format(repo=repo_url))
+    cmd_list = ["/usr/bin/svn", "ls", "-v", repo_url]
+    return __check_subprocess(cmd_list)
+
+def __check_git_helper(repo_url):
+    eprint("{repo} > Using git ls-remote".format(repo=repo_url))
+    cmd_list = ["git", "ls-remote", "--tags", repo_url]
+    return __check_subprocess(cmd_list)
+
+def __svn_resp_to_tags(resp):
+    tags = []
+    for line in resp.splitlines():
+        items = line.split()
+        for item in items:
+            if item[-1] == "/":
+                tags.append(item[:-1])
+                break
+    return tags
+
+def __git_resp_to_tags(resp):
     tags = []
     pattern = re.compile("^([^ \t]*)[ \t]*refs\/tags\/([^ \t]*)")
     for line in resp.splitlines():
         m = pattern.match(line)
         if m:
             tags.append(m.group(2))
+    return tags
+
+def check_git (info):
+    resp = load_last_query_result(info)
+    if resp == "":
+        resp = __check_git_helper(info["src_repo"])
+        last_query={}
+        last_query["time_stamp"] = datetime.now()
+        last_query["raw_data"] = resp
+        info["last_query"] = last_query
+
+    tags = __git_resp_to_tags(resp)
+    tags = clean_tags(tags, info)
+
+    return tags
+
+def check_github(info):
+    resp = load_last_query_result(info)
+    if info.get("query_type", "git-ls") != "git-ls":
+        resp = ""
+
+    repo_url = "https://github.com/" + info["src_repo"] + ".git"
+
+    if resp == "":
+        resp = __check_git_helper(repo_url)
+        last_query = {}
+        last_query["time_stamp"] = datetime.now()
+        last_query["raw_data"] = resp
+        info["last_query"] = last_query
+        info["query_type"] = "git-ls"
+
+    tags = __git_resp_to_tags(resp)
+    tags = clean_tags(tags, info)
+    return tags
+
+def check_gnome(info):
+    resp = load_last_query_result(info)
+    repo_url = "https://gitlab.gnome.org/GNOME/"+info["src_repo"]+".git"
+
+    if resp == "":
+        resp = __check_git_helper(repo_url)
+        last_query={}
+        last_query["time_stamp"] = datetime.now()
+        last_query["raw_data"] = resp
+        info["last_query"] = last_query
+
+    tags = __git_resp_to_tags(resp)
     tags = clean_tags(tags, info)
     return tags
 
+def check_svn(info):
+    resp = load_last_query_result(info)
+    repo_url = info["src_repo"] + "/tags"
+    if resp == "":
+        resp = __check_svn_helper(repo_url)
+        last_query = {}
+        last_query["time_stamp"] = datetime.now()
+        last_query["raw_data"] = resp
+        info["last_query"] = last_query
+
+    tags = __svn_resp_to_tags(resp)
+    tags = clean_tags(tags, info)
+    return tags
+
+
 if __name__ == "__main__":
     pass
 """
@@ -159,5 +279,4 @@ def sort_tags (tags)
 	return tags
 end
 
-
 """

advisors/oa_upgradable.py

@@ -32,8 +32,8 @@ if __name__ == "__main__":
 
     current_version = s_spec.version
 
-    print(prj_name)
-    print(current_version)
+    print("Checking ", prj_name)
+    print("current version is ",current_version)
 
     try:
         prj_info_string = gitee.get_yaml(prj_name)
@@ -55,133 +55,21 @@ if __name__ == "__main__":
         tags = check_upstream.check_hg(prj_info)
     elif vc_type == "github":
         tags = check_upstream.check_github(prj_info)
-    else:
-        pass
-
-    print("tags :", tags)
-    v = version_recommend.VersionRecommend(tags, current_version, 0)
-    print("Latest version is ", v.latest_version)
-    print("Maintain version is", v.maintain_version)
-"""
-    if vc_type == "svn":
-        tags = check_upstream_svn(prj_info)
     elif vc_type == "git":
-        tags = check_upstream_git(prj_info)
-	tags = clean_tags(tags.lines)
-    elif vc_type == "metacpan":
-	tags = check_upstream_metacpan(prj_info)
-	tags = clean_tags(tags.lines)
+        tags = check_upstream.check_git(prj_info)
     elif vc_type == "gitlab.gnome":
-	tags = check_upstream_gnome(prj_info)
-	tags = clean_tags(tags.lines)
+        tags = check_upstream.check_gnome(prj_info)
+    elif vc_type == "svn":
+        tags = check_upstream.check_svn(prj_info)
+    elif vc_type == "metacpan":
+        tags = check_upstream.check_metacpan(prj_info)
     elif vc_type == "pypi":
-	tags = check_upstream_pypi(prj_info)
-	tags = clean_tags(tags.lines)
+        tags = check_upstream.check_pypi(prj_info)
     else:
         print("Unsupport version control method {vc}".format(vc=vc_type))
         sys.exit(1)
-"""
-"""
-def compare_tags (a, b)
-	arr_a = a.split(".")
-	arr_b = b.split(".")
-	len = [arr_a.length, arr_b.length].min
-	idx = 0
-	while idx < len do
-		res1 = arr_a[idx].to_i <=> arr_b[idx].to_i
-		return res1 if res1 != 0
-		res2 = arr_a[idx].length <=> arr_b[idx].length
-		return -res2 if res2 != 0
-		res3 = arr_a[idx][-1].to_i <=> arr_b[idx][-1].to_i
-		return res3 if res3 != 0
-		idx = idx + 1
-	end
-	return arr_a.length <=> arr_b.length
-end
-
-def sort_tags (tags)
-	tags.sort! { |a, b|
-		compare_tags(a,b)
-	}
-	return tags
-end
-
-def clean_tags(tags)
-	new_tags = []
-	tags.each{|line|
-		new_tags = new_tags.append clean_tag(line, Prj_info)
-	}
-	return new_tags
-end
 
-def upgrade_recommend(tags_param, cur_tag, policy)
-	tags = tags_param.reverse
-	tag1 = cur_tag
-	tag2 = cur_tag
-	if policy == "latest" then
-		return tags[0]
-	elsif policy == "latest-stable" then
-		tags.each { |tag|
-			if tag.split(".").count {|f| f.to_i != 0 } >= 3 then
-				tag1 = tag
-				break
-			end
-		}
-		tags.each { |tag|
-			if tag.split(".").count {|f| f.to_i != 0} >= 2 then
-				tag2 = tag
-				break
-			end
-		}
-		if tag2[0].to_i > tag1[0].to_i then
-			return tag2
-		else
-			return tag1
-		end
-	elsif policy == "perfer-stable" then
-		tags.each { |tag|
-			if tag.start_with?(cur_tag) then
-				return tag
-			end
-		}
-		if cur_tag.split(".").length >= 3 then
-			search_tag = cur_tag.split(".")[0..1].join(".")
-			tags.each { |tag|
-				if tag.start_with?(search_tag) then
-					return tag
-				end
-			}
-		end
-		return cur_tag
-	else
-		return cur_tag
-	end
-
-end
-
-print Prj_name, ":\n"
-
-tags = sort_tags(tags)
-print "Latest upstream is ", tags[-1], "\n"
-#print "Recommended is     ", upgrade_recommend(tags, Cur_ver, "latest-stable"), "\n"
-print "Current version is ", Cur_ver, "\n"
-
-puts "This package has #{spec_struct.get_diverse} patches"
-
-if tags.length == 0 or compare_tags(tags[-1], Cur_ver) < 0 then
-	STDERR.puts "DEBUG #{Prj_name} > tags are #{tags}"
-	File.delete("upstream-info/"+Prj_name+".yaml") if File.exist?("upstream-info/"+Prj_name+".yaml")
-	File.open("known-issues/"+Prj_name+".yaml", "w") { |file| file.write(Prj_info.to_yaml) }
-else
-	File.open("upstream-info/"+Prj_name+".yaml", "w") { |file| file.write(Prj_info.to_yaml) }
-end
-File.delete(specfile) if specfile != ""
-
-if options[:push] then
-	puts "Push to gitee\n"
-	ad = Advisor.new
-	ad.new_issue("src-openeuler", Prj_name, "Upgrade to Latest Release", "Dear #{Prj_name} maintainer:\n\n  We found the latst version of #{Prj_name} is #{tags[-1]}, while the current version in openEuler is #{Cur_ver}.\n\n  Please consider upgrading.\n\n\nYours openEuler Advisor.")
-else
-	puts "keep it to us\n"
-end
-"""
+    print("known release tags :", tags)
+    v = version_recommend.VersionRecommend(tags, current_version, 0)
+    print("Latest version is ", v.latest_version)
+    print("Maintain version is", v.maintain_version)