提交 66d9e086 编写于 作者: O openeuler-ci-bot 提交者: Gitee

!57 refactor patch_tracking_cli.py

Merge pull request !57 from chenyanpanHW/master
......@@ -16,6 +16,7 @@ requests = "*"
werkzeug = "*"
flask-httpauth = "*"
sqlalchemy = "*"
pandas = "*"
[requires]
python_version = "3.7"
{
"_meta": {
"hash": {
"sha256": "d1f395f5adb9429b23b6eecefc56c0c65c91d42c7668e2b89f8280a2ce6cbbcd"
"sha256": "69f670800c1dbbc64632f716294e7acfb72b3be7bee88a2701745239b39d9935"
},
"pipfile-spec": 6,
"requires": {
......@@ -139,6 +139,60 @@
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.1.1"
},
"numpy": {
"hashes": [
"sha256:082f8d4dd69b6b688f64f509b91d482362124986d98dc7dc5f5e9f9b9c3bb983",
"sha256:1bc0145999e8cb8aed9d4e65dd8b139adf1919e521177f198529687dbf613065",
"sha256:309cbcfaa103fc9a33ec16d2d62569d541b79f828c382556ff072442226d1968",
"sha256:3673c8b2b29077f1b7b3a848794f8e11f401ba0b71c49fbd26fb40b71788b132",
"sha256:480fdd4dbda4dd6b638d3863da3be82873bba6d32d1fc12ea1b8486ac7b8d129",
"sha256:56ef7f56470c24bb67fb43dae442e946a6ce172f97c69f8d067ff8550cf782ff",
"sha256:5a936fd51049541d86ccdeef2833cc89a18e4d3808fe58a8abeb802665c5af93",
"sha256:5b6885c12784a27e957294b60f97e8b5b4174c7504665333c5e94fbf41ae5d6a",
"sha256:667c07063940e934287993366ad5f56766bc009017b4a0fe91dbd07960d0aba7",
"sha256:7ed448ff4eaffeb01094959b19cbaf998ecdee9ef9932381420d514e446601cd",
"sha256:8343bf67c72e09cfabfab55ad4a43ce3f6bf6e6ced7acf70f45ded9ebb425055",
"sha256:92feb989b47f83ebef246adabc7ff3b9a59ac30601c3f6819f8913458610bdcc",
"sha256:935c27ae2760c21cd7354402546f6be21d3d0c806fffe967f745d5f2de5005a7",
"sha256:aaf42a04b472d12515debc621c31cf16c215e332242e7a9f56403d814c744624",
"sha256:b12e639378c741add21fbffd16ba5ad25c0a1a17cf2b6fe4288feeb65144f35b",
"sha256:b1cca51512299841bf69add3b75361779962f9cee7d9ee3bb446d5982e925b69",
"sha256:b8456987b637232602ceb4d663cb34106f7eb780e247d51a260b84760fd8f491",
"sha256:b9792b0ac0130b277536ab8944e7b754c69560dac0415dd4b2dbd16b902c8954",
"sha256:c9591886fc9cbe5532d5df85cb8e0cc3b44ba8ce4367bd4cf1b93dc19713da72",
"sha256:cf1347450c0b7644ea142712619533553f02ef23f92f781312f6a3553d031fc7",
"sha256:de8b4a9b56255797cbddb93281ed92acbc510fb7b15df3f01bd28f46ebc4edae",
"sha256:e1b1dc0372f530f26a03578ac75d5e51b3868b9b76cd2facba4c9ee0eb252ab1",
"sha256:e45f8e981a0ab47103181773cc0a54e650b2aef8c7b6cd07405d0fa8d869444a",
"sha256:e4f6d3c53911a9d103d8ec9518190e52a8b945bab021745af4939cfc7c0d4a9e",
"sha256:ed8a311493cf5480a2ebc597d1e177231984c818a86875126cfd004241a73c3e",
"sha256:ef71a1d4fd4858596ae80ad1ec76404ad29701f8ca7cdcebc50300178db14dfc"
],
"markers": "python_version >= '3.6'",
"version": "==1.19.1"
},
"pandas": {
"hashes": [
"sha256:02f1e8f71cd994ed7fcb9a35b6ddddeb4314822a0e09a9c5b2d278f8cb5d4096",
"sha256:13f75fb18486759da3ff40f5345d9dd20e7d78f2a39c5884d013456cec9876f0",
"sha256:35b670b0abcfed7cad76f2834041dcf7ae47fd9b22b63622d67cdc933d79f453",
"sha256:4c73f373b0800eb3062ffd13d4a7a2a6d522792fa6eb204d67a4fad0a40f03dc",
"sha256:5759edf0b686b6f25a5d4a447ea588983a33afc8a0081a0954184a4a87fd0dd7",
"sha256:5a7cf6044467c1356b2b49ef69e50bf4d231e773c3ca0558807cdba56b76820b",
"sha256:69c5d920a0b2a9838e677f78f4dde506b95ea8e4d30da25859db6469ded84fa8",
"sha256:8778a5cc5a8437a561e3276b85367412e10ae9fff07db1eed986e427d9a674f8",
"sha256:9871ef5ee17f388f1cb35f76dc6106d40cb8165c562d573470672f4cdefa59ef",
"sha256:9c31d52f1a7dd2bb4681d9f62646c7aa554f19e8e9addc17e8b1b20011d7522d",
"sha256:ab8173a8efe5418bbe50e43f321994ac6673afc5c7c4839014cf6401bbdd0705",
"sha256:ae961f1f0e270f1e4e2273f6a539b2ea33248e0e3a11ffb479d757918a5e03a9",
"sha256:b3c4f93fcb6e97d993bf87cdd917883b7dab7d20c627699f360a8fb49e9e0b91",
"sha256:c9410ce8a3dee77653bc0684cfa1535a7f9c291663bd7ad79e39f5ab58f67ab3",
"sha256:f69e0f7b7c09f1f612b1f8f59e2df72faa8a6b41c5a436dde5b615aaf948f107",
"sha256:faa42a78d1350b02a7d2f0dbe3c80791cf785663d6997891549d0f86dc49125e"
],
"index": "pypi",
"version": "==1.0.5"
},
"python-dateutil": {
"hashes": [
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
......
......@@ -183,7 +183,7 @@ systemctl start patch-tracking
例如:
```shell script
patch-tracking-cli --server 127.0.0.1:5001 --user admin --password Test@123 --version_control github --repo testPatchTrack/testPatch1 --branch master --scm_repo BJMX/testPatch01 --scm_branch test --enable true
patch-tracking-cli add --server 127.0.0.1:5001 --user admin --password Test@123 --version_control github --repo testPatchTrack/testPatch1 --branch master --scm_repo BJMX/testPatch01 --scm_branch test --enable true
```
#### 4.1.2 指定文件添加
......@@ -198,7 +198,7 @@ patch-tracking-cli --server 127.0.0.1:5001 --user admin --password Test@123 --ve
例如:
```shell script
patch-tracking-cli --server 127.0.0.1:5001 --user admin --password Test@123 --file tracking.yaml
patch-tracking-cli add --server 127.0.0.1:5001 --user admin --password Test@123 --file tracking.yaml
```
yaml内容格式如下,冒号左边的内容不可修改,右边内容根据实际情况填写。
......@@ -230,27 +230,35 @@ enable 是否自动跟踪该仓库
--dir :存放yaml文件目录的路径
```shell script
patch-tracking-cli --server 127.0.0.1:5001 --user admin --password Test@123 --dir /home/Work/test_yaml/
patch-tracking-cli add --server 127.0.0.1:5001 --user admin --password Test@123 --dir /home/Work/test_yaml/
```
### 4.2 查询跟踪项
参数含义:
>--server :必选参数,启动Patch Tracking服务的URL,例如:127.0.0.1:5001 \
--table :必选参数,需要查询的表 \
--repo :可选参数,需要查询的repo;如果没有该参数查询表中所有内容 \
--branch :可选参数,需要查询的branch,必须和--repo同时查询,没有--repo不允许单独查询该参数
#### 4.2.1 查询tracking表
```shell script
curl -k https://<LISTEN>/tracking
patch-tracking-cli query --server <LISTEN> --table tracking
```
例如:
```shell script
curl -k https://127.0.0.1:5001/tracking
patch-tracking-cli query --server 127.0.0.1:5001 --table tracking
```
### 4.3 查询生成的 Issue 列表
```shell script
curl -k https://<LISTEN>/issue
patch-tracking-cli query --server <LISTEN> --table issue
```
例如:
```shell script
curl -k https://127.0.0.1:5001/issue
patch-tracking-cli query --server 127.0.0.1:5001 --table issue
```
### 4.4 码云查看 issue 及 PR
......
......@@ -37,12 +37,14 @@ def update_tracking(data):
db.session.commit()
def delete_tracking(id_):
def delete_tracking(repo_, branch_=None):
"""
delete tracking
"""
post = Tracking.query.filter(Tracking.id == id_).one()
db.session.delete(post)
if branch_:
Tracking.query.filter(Tracking.repo == repo_, Tracking.branch == branch_).delete()
else:
Tracking.query.filter(Tracking.repo == repo_).delete()
db.session.commit()
......
......@@ -40,7 +40,7 @@ class ResponseCode:
}
@classmethod
def gen_dict(cls, code, data=None):
def ret_message(cls, code, data=None):
"""
generate response dictionary
"""
......
......@@ -19,16 +19,16 @@ def get():
if not request.args:
issues = Issue.query.all()
else:
required_params = ['repo', 'branch']
allowed_key = ['repo', 'branch']
input_params = request.args
data = dict()
for k, param in input_params.items():
if k in required_params:
if k in allowed_key:
data[k] = param
else:
return ResponseCode.gen_dict(ResponseCode.INPUT_PARAMETERS_ERROR)
return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)
issues = Issue.query.filter_by(**data).all()
resp_data = list()
for item in issues:
resp_data.append(item.to_json())
return ResponseCode.gen_dict(code=ResponseCode.SUCCESS, data=resp_data)
return ResponseCode.ret_message(code=ResponseCode.SUCCESS, data=resp_data)
......@@ -5,7 +5,7 @@ import logging
from flask import request, Blueprint
from sqlalchemy.exc import SQLAlchemyError
from patch_tracking.database.models import Tracking
from patch_tracking.api.business import create_tracking, update_tracking
from patch_tracking.api.business import create_tracking, update_tracking, delete_tracking
from patch_tracking.api.constant import ResponseCode
from patch_tracking.util.auth import auth
......@@ -13,6 +13,35 @@ logger = logging.getLogger(__name__)
tracking = Blueprint('tracking', __name__)
@tracking.route('', methods=["DELETE"])
@auth.login_required
def delete():
"""
Delete tracking(s).
"""
input_params = request.args
keys = list(input_params.keys())
if not keys or "repo" not in keys:
return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)
if len(set(keys) - {"repo", "branch"}) != 0:
return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)
try:
if "branch" in keys:
if Tracking.query.filter(Tracking.repo == input_params['repo'], Tracking.branch == input_params['branch']):
delete_tracking(input_params['repo'], input_params['branch'])
logger.info('Delete tracking repo: %s, branch: %s', input_params['repo'], input_params['branch'])
else:
if Tracking.query.filter(Tracking.repo == input_params['repo']):
delete_tracking(input_params['repo'])
logger.info('Delete tracking repo: %s', input_params['repo'])
return ResponseCode.ret_message(code=ResponseCode.SUCCESS)
except SQLAlchemyError as err:
return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_ERROR, data=err)
@tracking.route('', methods=["GET"])
def get():
"""
......@@ -21,34 +50,30 @@ def get():
if not request.args:
trackings = Tracking.query.all()
else:
required_params = ['repo', 'branch', 'enabled']
allowed_key = ['repo', 'branch', 'enabled']
input_params = request.args
data = dict()
for k, param in input_params.items():
if k in required_params:
if k in allowed_key:
if k == 'enabled':
param = bool(param == 'true')
data[k] = param
required_params.remove(k)
else:
return ResponseCode.gen_dict(ResponseCode.INPUT_PARAMETERS_ERROR)
if 'repo' in required_params and 'branch' not in required_params:
return ResponseCode.gen_dict(ResponseCode.INPUT_PARAMETERS_ERROR)
return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)
trackings = Tracking.query.filter_by(**data).all()
resp_data = list()
for item in trackings:
resp_data.append(item.to_json())
return ResponseCode.gen_dict(code=ResponseCode.SUCCESS, data=resp_data)
return ResponseCode.ret_message(code=ResponseCode.SUCCESS, data=resp_data)
@tracking.route('', methods=["POST"])
@auth.login_required
def post():
"""
Creates os update a tracking.
Creates or update a tracking.
"""
required_params = ['version_control', 'scm_repo', 'scm_branch', 'scm_commit', 'repo', 'branch', 'enabled']
input_params = request.json
......@@ -58,15 +83,13 @@ def post():
data[item] = input_params[item]
required_params.remove(item)
else:
return ResponseCode.gen_dict(ResponseCode.INPUT_PARAMETERS_ERROR)
return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)
if len(required_params) > 1 or (len(required_params) == 1 and required_params[0] != 'scm_commit'):
return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)
if required_params:
if len(required_params) == 1 and required_params[0] == 'scm_commit':
pass
else:
return ResponseCode.gen_dict(ResponseCode.INPUT_PARAMETERS_ERROR)
if data['version_control'] != 'github':
return ResponseCode.gen_dict(ResponseCode.INPUT_PARAMETERS_ERROR)
return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)
track = Tracking.query.filter_by(repo=data['repo'], branch=data['branch']).first()
if track:
......@@ -74,11 +97,11 @@ def post():
update_tracking(data)
logger.info('Update tracking. Data: %s.', data)
except SQLAlchemyError as err:
return ResponseCode.gen_dict(code=ResponseCode.INSERT_DATA_ERROR, data=err)
return ResponseCode.ret_message(code=ResponseCode.INSERT_DATA_ERROR, data=err)
else:
try:
create_tracking(data)
logger.info('Create tracking. Data: %s.', data)
except SQLAlchemyError as err:
return ResponseCode.gen_dict(code=ResponseCode.INSERT_DATA_ERROR, data=err)
return ResponseCode.gen_dict(code=ResponseCode.SUCCESS, data=request.json)
return ResponseCode.ret_message(code=ResponseCode.INSERT_DATA_ERROR, data=err)
return ResponseCode.ret_message(code=ResponseCode.SUCCESS, data=request.json)
......@@ -14,27 +14,26 @@ logging.config.fileConfig('logging.conf', disable_existing_loggers=False)
app = Flask(__name__)
logger = logging.getLogger(__name__)
app.config.from_pyfile("settings.conf")
def check_settings_conf():
"""
check settings.conf
"""
flag = 0
setting_error = False
required_settings = ['LISTEN', 'GITHUB_ACCESS_TOKEN', 'GITEE_ACCESS_TOKEN', 'SCAN_DB_INTERVAL', 'USER', 'PASSWORD']
for setting in required_settings:
if setting in app.config:
if not app.config[setting]:
logger.error('%s is empty in settings.conf.', setting)
flag = 1
setting_error = True
else:
logger.error('%s not configured in settings.conf.', setting)
flag = 1
if flag:
setting_error = True
if setting_error:
sys.exit()
app.config.from_pyfile("settings.conf")
check_settings_conf()
GITHUB_ACCESS_TOKEN = app.config['GITHUB_ACCESS_TOKEN']
......@@ -53,7 +52,7 @@ app.register_blueprint(tracking, url_prefix="/tracking")
db.init_app(app)
task.job_init(app)
task.init(app)
if __name__ == "__main__":
app.run(ssl_context="adhoc")
......@@ -3,60 +3,123 @@
command line of creating tracking item
"""
import argparse
import sys
import os
import sys
import pandas
import requests
from requests.auth import HTTPBasicAuth
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
USAGE = """
patch-tracking-cli --help
patch-tracking-cli --server SERVER --version_control github --scm_repo SCM_REPO --scm_branch SCM_BRANCH \\
--repo REPO --branch BRANCH --enabled True --user USER --password PWD
patch-tracking-cli --server SERVER --file FILE --user USER --password PWD
patch-tracking-cli --server SERVER --dir DIR --user USER --password PWD
"""
parser = argparse.ArgumentParser(
usage=USAGE, allow_abbrev=False, description="command line to create/update patch tracking item"
)
def query_table(args):
"""
query table
"""
server = args.server
parser.add_argument("--server", help="patch tracking daemon server")
if args.table == "tracking":
url = '/'.join(['https:/', server, 'tracking'])
if args.branch and args.repo:
params = {'repo': args.repo, 'branch': args.branch}
else:
params = {'repo': args.repo}
try:
ret = requests.get(url, params=params, verify=False)
if ret.status_code == 200 and ret.json()['code'] == '2001':
return 'success', ret
return 'error', ret
except Exception as exception:
return 'error', 'Connect server error: ' + str(exception)
elif args.table == "issue":
url = '/'.join(['https:/', server, 'issue'])
params = {'repo': args.repo, 'branch': args.branch}
try:
ret = requests.get(url, params=params, verify=False)
if ret.status_code == 200 and ret.json()['code'] == '2001':
return 'success', ret
return 'error', ret
except Exception as exception:
return 'error', 'Connect server error: ' + str(exception)
return 'error', 'table ' + args.table + ' not found'
def add_param_check_url(params, file_path=None):
"""
check url
"""
scm_url = f"https://github.com/{params['scm_repo']}/tree/{params['scm_branch']}"
url = f"https://gitee.com/{params['repo']}/tree/{params['branch']}"
patch_tracking_url = f"https://{params['server']}"
server_ret = server_check(patch_tracking_url)
if server_ret[0] != 'success':
return 'error'
parser.add_argument("--version_control", choices=['github'], help="upstream version control system")
parser.add_argument("--scm_repo", help="upstream scm repository")
parser.add_argument("--scm_branch", help="upstream scm branch")
parser.add_argument("--repo", help="source package repository")
parser.add_argument("--branch", help="source package branch")
parser.add_argument("--enabled", choices=["True", "true", "False", "false"], help="whether tracing is enabled")
scm_ret = repo_branch_check(scm_url)
if scm_ret[0] != 'success':
if file_path:
print(
f"scm_repo: {params['scm_repo']} and scm_branch: {params['scm_branch']} check failed. \n"
f"Error in {file_path}. {scm_ret[1]}"
)
else:
print(f"scm_repo: {params['scm_repo']} and scm_branch: {params['scm_branch']} check failed. {scm_ret[1]}")
return 'error'
ret = repo_branch_check(url)
if ret[0] != 'success':
if file_path:
print(f"repo: {params['repo']} and branch: {params['branch']} check failed. {ret[1]}. Error in {file_path}")
else:
print(f"repo: {params['repo']} and branch: {params['branch']} check failed. {ret[1]}.")
return 'error'
return None
parser.add_argument('--file', help='import patch tracking from file')
parser.add_argument('--dir', help='import patch tracking from files in directory')
parser.add_argument('--user', help='Authentication username')
parser.add_argument('--password', help='Authentication password')
def server_check(url):
"""
check if patch_tracking server start
"""
try:
ret = requests.head(url=url, verify=False)
except Exception as exception:
print(f"Error: Cannot connect to {url}, please make sure patch-tracking service is running.")
return 'error', exception
if ret.status_code == 200 or ret.status_code == 404:
return 'success', ret
args = parser.parse_args()
print(f"Unexpected Error: {ret.text}")
return 'error', ret.text
style1 = args.version_control or args.repo or args.branch or args.scm_repo or args.scm_branch or args.enabled
style2 = bool(args.file)
style3 = bool(args.dir)
if str([style1, style2, style3]).count('True') >= 2:
print("mix different usage style")
parser.print_usage()
sys.exit(-1)
def repo_branch_check(url):
"""
check if repo/branch exist
"""
headers = {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) " +
"Ubuntu Chromium/83.0.4103.61 Chrome/83.0.4103.61 Safari/537.36"
}
try:
ret = requests.get(url=url, headers=headers)
except Exception as exception:
return 'error', exception
if ret.status_code == 404:
return 'error', f'{url} not exist.'
if ret.status_code == 200:
return 'success', ret
return 'error', ret.text
def single_input_track(params, file_path=None):
def params_input_track(params, file_path=None):
"""
load tracking from ommand lcine arguments
load tracking from command line arguments
"""
if param_check(params, file_path) == 'error':
return 'error', 'Check input params error.'
if param_check_url(params, file_path) == 'error':
if add_param_check_url(params, file_path) == 'error':
return 'error', 'Check input params error.'
repo = params['repo']
......@@ -88,19 +151,98 @@ def single_input_track(params, file_path=None):
return 'error', 'Authenticate Error. Please make sure user and password are correct.'
if ret.status_code == 200 and ret.json()['code'] == '2001':
return 'success', 'created'
print("status_code: {}, return text: {}".format(ret.status_code, ret.text))
return 'error', 'Unexpected Error.'
def add(args):
"""
add tracking
"""
style1 = bool(args.version_control) or bool(args.repo) or bool(args.branch) or bool(args.scm_repo) or bool(
args.scm_branch
) or bool(args.enabled)
style2 = bool(args.file)
style3 = bool(args.dir)
if str([style1, style2, style3]).count('True') >= 2:
print("mix different usage style")
print(add_usage)
return
if style2:
file_input_track(args.file, args)
elif style3:
dir_input_track(args.dir, args)
else:
params = {
'repo': args.repo,
'branch': args.branch,
'scm_repo': args.scm_repo,
'scm_branch': args.scm_branch,
'version_control': args.version_control,
'enabled': args.enabled,
'server': args.server,
'user': args.user,
'password': args.password
}
ret = params_input_track(params)
if ret[0] == 'success':
print('Tracking successfully.')
else:
print(ret[1])
def delete(args):
"""
delete tracking
"""
server = args.server
user = args.user
password = args.password
url = '/'.join(['https:/', server, 'tracking'])
if args.branch:
params = {'repo': args.repo, 'branch': args.branch}
else:
params = {'repo': args.repo}
try:
ret = requests.delete(url, params=params, verify=False, auth=HTTPBasicAuth(user, password))
if ret.status_code == 200 and ret.json()['code'] == '2001':
print('Tracking delete successfully.')
return
print("Tracking delete failed. Error: %s", ret)
except Exception as exception:
print('Error: Connect server error: %s', str(exception))
def query(args):
"""
query table data
"""
if args.branch and not args.repo:
print(query_usage)
return
status, ret = query_table(args)
if status == "success":
df = pandas.DataFrame.from_dict(ret.json()["data"], orient="columns")
df.index = range(1, len(df) + 1)
print(df)
else:
print("status_code: {}, return text: {}".format(ret.status_code, ret.text))
return 'error', 'Unexpected Error.'
print(ret)
def file_input_track(file_path):
def file_input_track(file_path, args):
"""
load tracking from file
"""
if os.path.exists(file_path) and os.path.isfile(file_path):
if os.path.splitext(file_path)[-1] != ".yaml":
print('Please input yaml file. Error in {}'.format(file_path))
return None
return
with open(file_path) as file:
content = file.readlines()
params = dict()
......@@ -110,7 +252,7 @@ def file_input_track(file_path):
value = item.split(':')[1].strip(' ').strip('\n')
params.update({k: value})
params.update({'server': args.server, 'user': args.user, 'password': args.password})
ret = single_input_track(params, file_path)
ret = params_input_track(params, file_path)
if ret[0] == 'success':
print('Tracking successfully {} for {}'.format(ret[1], file_path))
else:
......@@ -119,7 +261,7 @@ def file_input_track(file_path):
print('yaml path error. Params error in {}'.format(file_path))
def dir_input_track(dir_path):
def dir_input_track(dir_path, args):
"""
load tracking from dir
"""
......@@ -127,157 +269,77 @@ def dir_input_track(dir_path):
for root, _, files in os.walk(dir_path):
if not files:
print('error: dir path empty')
return None
return
for file in files:
if os.path.splitext(file)[-1] == ".yaml":
file_path = os.path.join(root, file)
file_input_track(file_path)
file_input_track(file_path, args)
else:
print('Please input yaml file. Error in {}'.format(file))
else:
print('error: dir path error. Params error in {}'.format(dir_path))
def patch_tracking_server_check(url):
"""
check if patch_tracking server start
"""
try:
ret = requests.head(url=url, verify=False)
except Exception as exception:
print(f"Error: Cannot connect to {url}, please make sure patch-tracking service is running.")
return 'error', exception
if ret.status_code == 200 or ret.status_code == 404:
return 'success', ret
print(f"Unexpected Error: {ret.text}")
return 'error', ret.text
def repo_branch_check(url):
"""
check if repo/branch exist
"""
headers = {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) " +
"Ubuntu Chromium/83.0.4103.61 Chrome/83.0.4103.61 Safari/537.36"
}
try:
ret = requests.get(url=url, headers=headers)
except Exception as exception:
return 'error', exception
if ret.status_code == 404:
return 'error', f'{url} not exist.'
if ret.status_code == 200:
return 'success', ret
return 'error', ret.text
def command_default_param_check():
flag = 0
if not args.server:
print("Error: --server not configure.")
flag = 1
if not args.user:
print("Error: --user not configure.")
flag = 1
if not args.password:
print("Error: --password not configure.")
flag = 1
if flag == 1:
return 'error'
else:
return 'success'
def param_check(params, file_path=None):
"""
check if param is valid
"""
flag = 0
required_param = ['version_control', 'scm_repo', 'scm_branch', 'repo', 'branch', 'enabled', 'user', 'password']
for req in required_param:
if req not in params:
if file_path:
print(f'param: --{req} must be configured. Error in {file_path}')
else:
print(f'param: --{req} must be configured.')
flag = 1
for k, value in params.items():
if not value:
if file_path:
print(f'param: --{k} must be configured. Error in {file_path}')
else:
print(f'param: --{k} cannot be empty.')
flag = 1
if flag:
return 'error'
return None
def param_check_url(params, file_path=None):
"""
check url
"""
scm_url = f"https://github.com/{params['scm_repo']}/tree/{params['scm_branch']}"
url = f"https://gitee.com/{params['repo']}/tree/{params['branch']}"
patch_tracking_url = f"https://{params['server']}"
server_ret = patch_tracking_server_check(patch_tracking_url)
if server_ret[0] != 'success':
return 'error'
scm_ret = repo_branch_check(scm_url)
if scm_ret[0] != 'success':
if file_path:
print(
f"scm_repo: {params['scm_repo']} and scm_branch: {params['scm_branch']} check failed. \n"
f"Error in {file_path}. {scm_ret[1]}"
)
else:
print(f"scm_repo: {params['scm_repo']} and scm_branch: {params['scm_branch']} check failed. {scm_ret[1]}")
return 'error'
ret = repo_branch_check(url)
if ret[0] != 'success':
if file_path:
print(f"repo: {params['repo']} and branch: {params['branch']} check failed. {ret[1]}. Error in {file_path}")
parser = argparse.ArgumentParser(
prog='patch_tracking_cli',
allow_abbrev=False,
description="command line tool for manipulating patch tracking information"
)
subparsers = parser.add_subparsers(description=None, dest='subparser_name', help='additional help')
# common argument
common_parser = argparse.ArgumentParser(add_help=False)
common_parser.add_argument("--server", required=True, help="patch tracking daemon server")
# authentication argument
authentication_parser = argparse.ArgumentParser(add_help=False)
authentication_parser.add_argument('--user', required=True, help='authentication username')
authentication_parser.add_argument('--password', required=True, help='authentication password')
# add
add_usage = """
%(prog)s --server SERVER --user USER --password PASSWORD
--version_control github --scm_repo SCM_REPO --scm_branch SCM_BRANCH
--repo REPO --branch BRANCH --enabled True
%(prog)s --server SERVER --user USER --password PASSWORD --file FILE
%(prog)s --server SERVER --user USER --password PASSWORD --dir DIR"""
parser_add = subparsers.add_parser(
'add', parents=[common_parser, authentication_parser], help="add tracking", usage=add_usage
)
parser_add.set_defaults(func=add)
parser_add.add_argument("--version_control", choices=['github'], help="upstream version control system")
parser_add.add_argument("--scm_repo", help="upstream scm repository")
parser_add.add_argument("--scm_branch", help="upstream scm branch")
parser_add.add_argument("--repo", help="source package repository")
parser_add.add_argument("--branch", help="source package branch")
parser_add.add_argument("--enabled", choices=["True", "true", "False", "false"], help="whether tracing is enabled")
parser_add.add_argument('--file', help='import patch tracking from file')
parser_add.add_argument('--dir', help='import patch tracking from files in directory')
# delete
del_usage = """
%(prog)s --server SERVER --table TABLE --repo REPO [--branch BRANCH]"""
parser_delete = subparsers.add_parser('delete', parents=[common_parser, authentication_parser], help="delete tracking")
parser_delete.set_defaults(func=delete)
parser_delete.add_argument("--repo", required=True, help="source package repository")
parser_delete.add_argument("--branch", help="source package branch")
# query
query_usage = """
%(prog)s --server SERVER --table {tracking,issue} [--repo REPO] [--branch BRANCH]"""
parser_query = subparsers.add_parser('query', parents=[common_parser], help="query tracking/issue")
parser_query.set_defaults(func=query)
parser_query.add_argument("--table", required=True, choices=["tracking", "issue"], help="query tracking or issue")
parser_query.add_argument("--repo", help="source package repository")
parser_query.add_argument("--branch", help="source package branch")
if __name__ == "__main__":
args_ = parser.parse_args()
if args_.subparser_name:
if args_.func(args_) != "success":
sys.exit(1)
else:
print(f"repo: {params['repo']} and branch: {params['branch']} check failed. {ret[1]}.")
return 'error'
return None
def main():
"""
main
"""
if command_default_param_check() == 'error':
return None
if style2:
file_input_track(args.file)
elif style3:
dir_input_track(args.dir)
sys.exit(0)
else:
params = {
'repo': args.repo,
'branch': args.branch,
'scm_repo': args.scm_repo,
'scm_branch': args.scm_branch,
'version_control': args.version_control,
'enabled': args.enabled,
'server': args.server,
'user': args.user,
'password': args.password
}
ret = single_input_track(params)
if ret[0] == 'success':
print('Tracking successfully.')
else:
print(ret[1])
if __name__ == '__main__':
main()
parser.print_help()
sys.exit(1)
......@@ -4,42 +4,39 @@ load job/task of tracking
import datetime
import logging
from patch_tracking.task import scheduler
from patch_tracking.task import task_apscheduler
from patch_tracking.database.models import Tracking
from patch_tracking.util.github_api import GitHubApi
from patch_tracking.api.business import update_tracking
logger = logging.getLogger(__name__)
def job_init(app):
def init(app):
"""
jobs init
scheduler jobs init
"""
scan_db_interval = app.config['SCAN_DB_INTERVAL']
scheduler.init_app(app)
scheduler.add_job(
id='Add Tracking job - Update DB',
func=patch_tracking_task,
trigger='interval',
args=(app, ),
seconds=int(scan_db_interval),
next_run_time=datetime.datetime.now()
)
with app.app_context():
new_track = task_apscheduler.get_track_from_db()
scheduler.init_app(app)
scheduler.add_job(
id='Add Tracking job - Update DB',
func=load,
trigger='interval',
args=(new_track, ),
seconds=int(scan_db_interval),
next_run_time=datetime.datetime.now()
)
scheduler.add_job(
id=str("Check empty commitID"),
func=get_commit_id_empty,
trigger='interval',
args=(new_track, app),
seconds=600,
next_run_time=datetime.datetime.now(),
misfire_grace_time=300,
)
scheduler.add_job(
id=str("Check empty commitID"),
func=check_empty_commit_id,
trigger='interval',
args=(app, ),
seconds=600,
next_run_time=datetime.datetime.now(),
misfire_grace_time=300,
)
scheduler.start()
scheduler.start()
def add_job(job_id, func, args):
......@@ -52,11 +49,12 @@ def add_job(job_id, func, args):
)
def get_commit_id_empty(new_track, flask_app):
def check_empty_commit_id(flask_app):
"""
check commit ID for empty tracking
"""
with flask_app.app_context():
new_track = get_track_from_db()
github_api = GitHubApi()
for item in new_track:
if item.scm_commit:
......@@ -81,17 +79,29 @@ def get_commit_id_empty(new_track, flask_app):
)
def load(all_track):
def get_track_from_db():
"""
load trackings to jobs
query all trackings from database
"""
all_job_id = list()
for item in scheduler.get_jobs():
all_job_id.append(item.id)
for track in all_track:
if track.branch.split('/')[0] != 'patch-tracking':
job_id = str(track.repo + ":" + track.branch)
if job_id not in all_job_id:
add_job(
job_id=job_id, func='patch_tracking.task.task_apscheduler:upload_patch_to_gitee', args=(track, )
)
all_track = Tracking.query.filter_by(enabled=True)
return all_track
def patch_tracking_task(flask_app):
"""
add patch trackings to jobs
"""
with flask_app.app_context():
all_track = get_track_from_db()
all_job_id = list()
for item in scheduler.get_jobs():
all_job_id.append(item.id)
for track in all_track:
if track.branch.split('/')[0] != 'patch-tracking':
job_id = str(track.repo + ":" + track.branch)
if job_id not in all_job_id:
add_job(
job_id=job_id,
func='patch_tracking.task.task_apscheduler:upload_patch_to_gitee',
args=(track, )
)
......@@ -4,8 +4,8 @@ tracking job
import logging
import base64
import time
from patch_tracking.util.gitee_api import post_create_branch, post_upload_patch, post_create_issue, \
post_create_pull_request, get_path_content, put_upload_spec, post_create_spec
from patch_tracking.util.gitee_api import create_branch, upload_patch, create_gitee_issue
from patch_tracking.util.gitee_api import create_pull_request, get_path_content, upload_spec, create_spec
from patch_tracking.util.github_api import GitHubApi
from patch_tracking.database.models import Tracking
from patch_tracking.api.business import update_tracking, create_issue
......@@ -15,29 +15,22 @@ from patch_tracking.util.spec import Spec
logger = logging.getLogger(__name__)
def get_track_from_db():
"""
query all trackings from database
"""
all_track = Tracking.query.filter_by(enabled=True)
return all_track
def upload_patch_to_gitee(track):
"""
upload a patch file to Gitee
"""
cur_time = time.strftime("%Y%m%d%H%M%S", time.localtime())
with scheduler.app.app_context():
logger.info('[Patch Tracking %s] track.scm_commit_id: %s.', cur_time, track.scm_commit)
patch = get_scm_patch(track)
if patch:
issue = upload_patch(patch, cur_time)
issue = create_patch_issue_pr(patch, cur_time)
if issue:
create_issue_db(issue)
else:
logger.debug('No issue need to create.')
logger.info('[Patch Tracking %s] No issue need to create.', cur_time)
else:
logger.debug('No new commit.')
logger.debug('[Patch Tracking %s] No new commit.', cur_time)
def get_all_commit_info(scm_repo, db_commit, latest_commit):
......@@ -63,10 +56,16 @@ def get_all_commit_info(scm_repo, db_commit, latest_commit):
latest_commit = result['parent']
else:
logger.info('Successful get scm commit from %s to %s ID/message/time/patch.', db_commit, latest_commit)
logger.info(
'[Patch Tracking] Successful get scm commit from %s to %s ID/message/time/patch.', db_commit,
latest_commit
)
break
else:
logger.error('Get scm: %s commit: %s ID/message/time failed. Result: %s', scm_repo, latest_commit, result)
logger.error(
'[Patch Tracking] Get scm: %s commit: %s ID/message/time failed. Result: %s', scm_repo, latest_commit,
result
)
return commit_list
......@@ -77,19 +76,20 @@ def get_scm_patch(track):
Different warehouse has different acquisition methods
:return:
"""
scm_dict = dict()
github_api = GitHubApi()
scm_dict['scm_repo'] = track.scm_repo
scm_dict['scm_branch'] = track.scm_branch
scm_dict['scm_commit'] = track.scm_commit
scm_dict['enabled'] = track.enabled
scm_dict['repo'] = track.repo
scm_dict['branch'] = track.branch
scm_dict['version_control'] = track.version_control
scm_dict = dict(
scm_repo=track.scm_repo,
scm_branch=track.scm_branch,
scm_commit=track.scm_commit,
enabled=track.enabled,
repo=track.repo,
branch=track.branch,
version_control=track.version_control
)
status, result = github_api.get_latest_commit(scm_dict['scm_repo'], scm_dict['scm_branch'])
logger.debug('get_latest_commit: %s %s', status, result)
logger.debug(
'repo: %s branch: %s. get_latest_commit: %s %s', scm_dict['scm_repo'], scm_dict['scm_branch'], status, result
)
if status == 'success':
commit_id = result['latest_commit']
......@@ -104,24 +104,28 @@ def get_scm_patch(track):
'scm_repo': scm_dict['scm_repo']
}
update_tracking(data)
logger.info(
'[Patch Tracking] Scm_repo: %s Scm_branch: %s.Get latest commit ID: %s From commit ID: None.',
scm_dict['scm_repo'], scm_dict['scm_branch'], result['latest_commit']
)
else:
if commit_id != scm_dict['scm_commit']:
commit_list = get_all_commit_info(scm_dict['scm_repo'], scm_dict['scm_commit'], commit_id)
scm_dict['commit_list'] = commit_list
return scm_dict
logger.info(
'Latest commit id not change of scm_repo: %s scm_branch: %s. Nothing need to do.', scm_dict['scm_repo'],
scm_dict['scm_branch']
'[Patch Tracking] Scm_repo: %s Scm_branch: %s.Get latest commit ID: %s From commit ID: %s. Nothing need to do.',
scm_dict['scm_repo'], scm_dict['scm_branch'], commit_id, scm_dict['scm_commit']
)
else:
logger.error(
'Fail to get latest commit id of scm_repo: %s scm_branch: %s. Return val: %s', scm_dict['scm_repo'],
scm_dict['scm_branch'], result
'[Patch Tracking] Fail to get latest commit id of scm_repo: %s scm_branch: %s. Return val: %s',
scm_dict['scm_repo'], scm_dict['scm_branch'], result
)
return None
def upload_patch(patch, cur_time):
def create_patch_issue_pr(patch, cur_time):
"""
Create temporary branches, submit files, and create PR and issue
:return:
......@@ -133,11 +137,11 @@ def upload_patch(patch, cur_time):
issue_dict['repo'] = patch['repo']
issue_dict['branch'] = patch['branch']
new_branch = 'patch-tracking/' + cur_time
result = post_create_branch(patch['repo'], patch['branch'], new_branch)
result = create_branch(patch['repo'], patch['branch'], new_branch)
if result == 'success':
logger.info('Successful create branch: %s', new_branch)
logger.info('[Patch Tracking %s] Successful create branch: %s', cur_time, new_branch)
else:
logger.error('Fail to create branch: %s', new_branch)
logger.error('[Patch Tracking %s] Fail to create branch: %s', cur_time, new_branch)
patch_lst = list()
# 表格格式会导致 Gitee 敏感词,先去掉
issue_table = ""
......@@ -146,8 +150,8 @@ def upload_patch(patch, cur_time):
issue_table += '[{}]({}) | {} | {}'.format(
latest_commit['commit_id'][0:7], scm_commit_url, latest_commit['time'], latest_commit['message']
) + '\n'
patch_file_content = latest_commit['patch_content']
patch_file_content = latest_commit['patch_content']
post_data = {
'repo': patch['repo'],
'branch': new_branch,
......@@ -156,26 +160,30 @@ def upload_patch(patch, cur_time):
'cur_time': cur_time,
'commit_url': scm_commit_url
}
result = post_upload_patch(post_data)
result = upload_patch(post_data)
if result == 'success':
logger.info('Successfully upload patch file of commit: %s', latest_commit['commit_id'])
logger.info(
'[Patch Tracking %s] Successfully upload patch file of commit: %s', cur_time, latest_commit['commit_id']
)
else:
logger.error('Fail to upload patch file of commit: %s', latest_commit['commit_id'])
logger.error(
'[Patch Tracking %s] Fail to upload patch file of commit: %s', cur_time, latest_commit['commit_id']
)
patch_lst.append(str(latest_commit['commit_id']))
logger.debug(issue_table)
result = post_create_issue(patch['repo'], issue_table, cur_time)
result = create_gitee_issue(patch['repo'], issue_table, cur_time)
if result[0] == 'success':
issue_num = result[1]
logger.info('Successfully create issue: %s', issue_num)
ret = post_create_pull_request(patch['repo'], patch['branch'], new_branch, issue_num, cur_time)
logger.info('[Patch Tracking %s] Successfully create issue: %s', cur_time, issue_num)
ret = create_pull_request(patch['repo'], patch['branch'], new_branch, issue_num, cur_time)
if ret == 'success':
logger.info('Successfully create PR of issue: %s.', issue_num)
logger.info('[Patch Tracking %s] Successfully create PR of issue: %s.', cur_time, issue_num)
else:
logger.error('Fail to create PR of issue: %s. Result: %s', issue_num, ret)
logger.error('[Patch Tracking %s] Fail to create PR of issue: %s. Result: %s', cur_time, issue_num, ret)
issue_dict['issue'] = issue_num
upload_spec(patch, patch_lst, cur_time)
upload_spec_to_repo(patch, patch_lst, cur_time)
data = {
'version_control': patch['version_control'],
......@@ -188,12 +196,12 @@ def upload_patch(patch, cur_time):
}
update_tracking(data)
else:
logger.error('Fail to create issue: %s. Result: %s', issue_table, result[1])
logger.error('[Patch Tracking %s] Fail to create issue: %s. Result: %s', cur_time, issue_table, result[1])
return issue_dict
def upload_spec(patch, patch_lst, cur_time):
def upload_spec_to_repo(patch, patch_lst, cur_time):
"""
update and upload spec file
"""
......@@ -212,14 +220,14 @@ def upload_spec(patch, patch_lst, cur_time):
spec_content = str(base64.b64decode(ret['content']), encoding='utf-8')
spec_sha = ret['sha']
new_spec = modify_spec(log_title, log_content, patch_file_lst, spec_content)
update_spec(patch['repo'], new_branch, cur_time, new_spec, spec_sha)
update_spec_to_repo(patch['repo'], new_branch, cur_time, new_spec, spec_sha)
else:
if 'message' in ret and 'File Not Found' in ret['message']:
spec_content = ''
new_spec = modify_spec(log_title, log_content, patch_file_lst, spec_content)
create_spec(patch['repo'], new_branch, cur_time, new_spec)
create_spec_to_repo(patch['repo'], new_branch, cur_time, new_spec)
else:
logger.error('Fail to update spec: %s. Result: %s', spec_file, ret)
logger.error('[Patch Tracking %s] Fail to update spec: %s. Result: %s', cur_time, spec_file, ret)
def modify_spec(log_title, log_content, patch_file_lst, spec_content):
......@@ -230,26 +238,26 @@ def modify_spec(log_title, log_content, patch_file_lst, spec_content):
return spec.update(log_title, log_content, patch_file_lst)
def update_spec(repo, branch, cur_time, spec_content, spec_sha):
def update_spec_to_repo(repo, branch, cur_time, spec_content, spec_sha):
"""
update spec file
"""
ret = put_upload_spec(repo, branch, cur_time, spec_content, spec_sha)
ret = upload_spec(repo, branch, cur_time, spec_content, spec_sha)
if ret == 'success':
logger.info('Successfully update spec file.')
logger.info('[Patch Tracking %s] Successfully update spec file.', cur_time)
else:
logger.error('Fail to update spec file. Result: %s', ret)
logger.error('[Patch Tracking %s] Fail to update spec file. Result: %s', cur_time, ret)
def create_spec(repo, branch, cur_time, spec_content):
def create_spec_to_repo(repo, branch, cur_time, spec_content):
"""
create new spec file
"""
ret = post_create_spec(repo, branch, spec_content, cur_time)
ret = create_spec(repo, branch, spec_content, cur_time)
if ret == 'success':
logger.info('Successfully create spec file.')
logger.info('[Patch Tracking %s] Successfully create spec file.', cur_time)
else:
logger.error('Fail to create spec file. Result: %s', ret)
logger.error('[Patch Tracking %s] Fail to create spec file. Result: %s', cur_time, ret)
def create_issue_db(issue):
......
......@@ -23,7 +23,7 @@ def get_path_content(repo, branch, path):
return ret
def post_create_branch(repo, branch, new_branch):
def create_branch(repo, branch, new_branch):
"""
create branch
"""
......@@ -37,7 +37,7 @@ def post_create_branch(repo, branch, new_branch):
return response.json()
def post_upload_patch(data):
def upload_patch(data):
"""
upload patch
"""
......@@ -54,7 +54,7 @@ def post_upload_patch(data):
return response.json()
def post_create_spec(repo, branch, spec_content, cur_time):
def create_spec(repo, branch, spec_content, cur_time):
"""
create spec
"""
......@@ -72,7 +72,7 @@ def post_create_spec(repo, branch, spec_content, cur_time):
return response.json()
def put_upload_spec(repo, branch, cur_time, spec_content, spec_sha):
def upload_spec(repo, branch, cur_time, spec_content, spec_sha):
"""
upload spec
"""
......@@ -99,7 +99,7 @@ def put_upload_spec(repo, branch, cur_time, spec_content, spec_sha):
return response.json()
def post_create_issue(repo, issue_body, cur_time):
def create_gitee_issue(repo, issue_body, cur_time):
"""
create issue
"""
......@@ -114,7 +114,7 @@ def post_create_issue(repo, issue_body, cur_time):
return 'error', response.json()
def post_create_pull_request(repo, branch, patch_branch, issue_num, cur_time):
def create_pull_request(repo, branch, patch_branch, issue_num, cur_time):
"""
create pull request
"""
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册