From 4417a79b8601b197ddf064f1ba3eb0eac3a17919 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Wed, 22 Jul 2020 04:05:46 +0300 Subject: [PATCH] Update scripts (#12650) --- utils/github/backport.py | 114 +++++++++++++++--------------- utils/github/cherrypick.py | 137 +++++++++++++++++++------------------ utils/github/local.py | 11 ++- utils/github/query.py | 13 ++-- 4 files changed, 138 insertions(+), 137 deletions(-) diff --git a/utils/github/backport.py b/utils/github/backport.py index b860b3e93c..a90b79a01e 100644 --- a/utils/github/backport.py +++ b/utils/github/backport.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -from query import Query as RemoteRepo -from local import BareRepository as LocalRepo -import cherrypick +from clickhouse.utils.github.cherrypick import CherryPick +from clickhouse.utils.github.query import Query as RemoteRepo +from clickhouse.utils.github.local import Repository as LocalRepo import argparse import logging @@ -12,83 +12,78 @@ import sys class Backport: def __init__(self, token, owner, name, team): - ''' - `refs` is a list of (ref_path, base_commit) sorted by ancestry starting from the least recent ref. - ''' self._gh = RemoteRepo(token, owner=owner, name=name, team=team, max_page_size=30) + self._token = token self.default_branch_name = self._gh.default_branch + self.ssh_url = self._gh.ssh_url def getPullRequests(self, from_commit): return self._gh.get_pull_requests(from_commit) + def execute(self, repo, til, number, run_cherrypick): + repo = LocalRepo(repo, 'origin', self.default_branch_name) + branches = repo.get_release_branches()[-number:] # [(branch_name, base_commit)] -def run(token, repo_bare, til, number, run_cherrypick): - bp = Backport(token, 'ClickHouse', 'ClickHouse', 'core') - repo = LocalRepo(repo_bare, bp.default_branch_name) + if not branches: + logging.info('No release branches found!') + return - branches = repo.get_release_branches()[-number:] # [(branch_name, base_commit)] - - if not branches: - logging.info('No release branches found!') - return - - for branch in branches: - logging.info('Found release branch: %s', branch[0]) + for branch in branches: + logging.info('Found release branch: %s', branch[0]) - if not til: - til = branches[0][1] - prs = bp.getPullRequests(til) + if not til: + til = branches[0][1] + prs = self.getPullRequests(til) - backport_map = {} + backport_map = {} - RE_MUST_BACKPORT = re.compile(r'^v(\d+\.\d+)-must-backport$') - RE_NO_BACKPORT = re.compile(r'^v(\d+\.\d+)-no-backport$') + RE_MUST_BACKPORT = re.compile(r'^v(\d+\.\d+)-must-backport$') + RE_NO_BACKPORT = re.compile(r'^v(\d+\.\d+)-no-backport$') - # pull-requests are sorted by ancestry from the least recent. - for pr in prs: - while repo.comparator(branches[-1][1]) >= repo.comparator(pr['mergeCommit']['oid']): - branches.pop() + # pull-requests are sorted by ancestry from the least recent. + for pr in prs: + while repo.comparator(branches[-1][1]) >= repo.comparator(pr['mergeCommit']['oid']): + branches.pop() - assert len(branches) + assert len(branches) - branch_set = set([branch[0] for branch in branches]) + branch_set = set([branch[0] for branch in branches]) - # First pass. Find all must-backports - for label in pr['labels']['nodes']: - if label['name'].startswith('pr-') and label['color'] == 'ff0000': - backport_map[pr['number']] = branch_set.copy() - continue - m = RE_MUST_BACKPORT.match(label['name']) - if m: - if pr['number'] not in backport_map: - backport_map[pr['number']] = set() - backport_map[pr['number']].add(m.group(1)) + # First pass. Find all must-backports + for label in pr['labels']['nodes']: + if label['name'].startswith('pr-') and label['color'] == 'ff0000': + backport_map[pr['number']] = branch_set.copy() + continue + m = RE_MUST_BACKPORT.match(label['name']) + if m: + if pr['number'] not in backport_map: + backport_map[pr['number']] = set() + backport_map[pr['number']].add(m.group(1)) - # Second pass. Find all no-backports - for label in pr['labels']['nodes']: - if label['name'] == 'pr-no-backport' and pr['number'] in backport_map: - del backport_map[pr['number']] - break - m = RE_NO_BACKPORT.match(label['name']) - if m and pr['number'] in backport_map and m.group(1) in backport_map[pr['number']]: - backport_map[pr['number']].remove(m.group(1)) + # Second pass. Find all no-backports + for label in pr['labels']['nodes']: + if label['name'] == 'pr-no-backport' and pr['number'] in backport_map: + del backport_map[pr['number']] + break + m = RE_NO_BACKPORT.match(label['name']) + if m and pr['number'] in backport_map and m.group(1) in backport_map[pr['number']]: + backport_map[pr['number']].remove(m.group(1)) - for pr, branches in backport_map.items(): - logging.info('PR #%s needs to be backported to:', pr) - for branch in branches: - logging.info('\t%s %s', branch, run_cherrypick(token, pr, branch)) + for pr, branches in backport_map.items(): + logging.info('PR #%s needs to be backported to:', pr) + for branch in branches: + logging.info('\t%s %s', branch, run_cherrypick(self._token, pr, branch)) - # print API costs - logging.info('\nGitHub API total costs per query:') - for name, value in bp._gh.api_costs.items(): - logging.info('%s : %s', name, value) + # print API costs + logging.info('\nGitHub API total costs per query:') + for name, value in self._gh.api_costs.items(): + logging.info('%s : %s', name, value) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--token', type=str, required=True, help='token for Github access') - parser.add_argument('--repo-bare', type=str, required=True, help='path to bare repository', metavar='PATH') - parser.add_argument('--repo-full', type=str, required=True, help='path to full repository', metavar='PATH') + parser.add_argument('--repo', type=str, required=True, help='path to full repository', metavar='PATH') parser.add_argument('--til', type=str, help='check PRs from HEAD til this commit', metavar='COMMIT') parser.add_argument('-n', type=int, dest='number', help='number of last release branches to consider') parser.add_argument('--dry-run', action='store_true', help='do not create or merge any PRs', default=False) @@ -100,5 +95,6 @@ if __name__ == "__main__": else: logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.INFO) - cherrypick_run = lambda token, pr, branch: cherrypick.run(token, pr, branch, args.repo_full, args.dry_run) - run(args.token, args.repo_bare, args.til, args.number, cherrypick_run) + cherrypick_run = lambda token, pr, branch: CherryPick(token, 'ClickHouse', 'ClickHouse', 'core', pr, branch).execute(args.repo, args.dry_run) + bp = Backport(args.token, 'ClickHouse', 'ClickHouse', 'core') + bp.execute(args.repo, args.til, args.number, cherrypick_run) diff --git a/utils/github/cherrypick.py b/utils/github/cherrypick.py index 0694516da1..ead86a5bfb 100644 --- a/utils/github/cherrypick.py +++ b/utils/github/cherrypick.py @@ -14,12 +14,13 @@ Second run checks PR from previous run to be merged or at least being mergeable. Third run creates PR from backport branch (with merged previous PR) to release branch. ''' -from query import Query as RemoteRepo +from clickhouse.utils.github.query import Query as RemoteRepo import argparse from enum import Enum import logging import os +import subprocess import sys @@ -33,10 +34,15 @@ class CherryPick: SECOND_CONFLICTS = 'conflicts on 2nd stage' MERGED = 'backported' + def _run(self, args): + logging.info(subprocess.check_output(args)) + def __init__(self, token, owner, name, team, pr_number, target_branch): self._gh = RemoteRepo(token, owner=owner, name=name, team=team) self._pr = self._gh.get_pull_request(pr_number) + self.ssh_url = self._gh.ssh_url + # TODO: check if pull-request is merged. self.merge_commit_oid = self._pr['mergeCommit']['oid'] @@ -60,25 +66,26 @@ class CherryPick: ) # FIXME: replace with something better than os.system() - git_prefix = 'git -C {} -c "user.email=robot-clickhouse@yandex-team.ru" -c "user.name=robot-clickhouse" '.format(repo_path) + git_prefix = ['git', '-C', repo_path, '-c', 'user.email=robot-clickhouse@yandex-team.ru', '-c', 'user.name=robot-clickhouse'] base_commit_oid = self._pr['mergeCommit']['parents']['nodes'][0]['oid'] # Create separate branch for backporting, and make it look like real cherry-pick. - os.system(git_prefix + 'checkout -f ' + self.target_branch) - os.system(git_prefix + 'checkout -B ' + self.backport_branch) - os.system(git_prefix + 'merge -s ours --no-edit ' + base_commit_oid) + self._run(git_prefix + ['checkout', '-f', self.target_branch]) + self._run(git_prefix + ['checkout', '-B', self.backport_branch]) + self._run(git_prefix + ['merge', '-s', 'ours', '--no-edit', base_commit_oid]) # Create secondary branch to allow pull request with cherry-picked commit. - os.system(git_prefix + 'branch -f {} {}'.format(self.cherrypick_branch, self.merge_commit_oid)) + self._run(git_prefix + ['branch', '-f', self.cherrypick_branch, self.merge_commit_oid]) - os.system(git_prefix + 'push -f origin {branch}:{branch}'.format(branch=self.backport_branch)) - os.system(git_prefix + 'push -f origin {branch}:{branch}'.format(branch=self.cherrypick_branch)) + self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.backport_branch)]) + self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.cherrypick_branch)]) # Create pull-request like a local cherry-pick pr = self._gh.create_pull_request(source=self.cherrypick_branch, target=self.backport_branch, - title='Cherry pick #{number} to {target}: {title}'.format( - number=self._pr['number'], target=self.target_branch, title=self._pr['title'].replace('"', '\\"')), - description='Original pull-request #{}\n\n{}'.format(self._pr['number'], DESCRIPTION)) + title='Cherry pick #{number} to {target}: {title}'.format( + number=self._pr['number'], target=self.target_branch, + title=self._pr['title'].replace('"', '\\"')), + description='Original pull-request #{}\n\n{}'.format(self._pr['number'], DESCRIPTION)) # FIXME: use `team` to leave a single eligible assignee. self._gh.add_assignee(pr, self._pr['author']) @@ -102,18 +109,20 @@ class CherryPick: 'Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n' ) - git_prefix = 'git -C {} -c "user.email=robot-clickhouse@yandex-team.ru" -c "user.name=robot-clickhouse" '.format(repo_path) + git_prefix = ['git', '-C', repo_path, '-c', 'user.email=robot-clickhouse@yandex-team.ru', '-c', 'user.name=robot-clickhouse'] + + pr_title = 'Backport #{number} to {target}: {title}'.format( + number=self._pr['number'], target=self.target_branch, + title=self._pr['title'].replace('"', '\\"')) - os.system(git_prefix + 'checkout -f ' + self.backport_branch) - os.system(git_prefix + 'pull --ff-only origin ' + self.backport_branch) - os.system(git_prefix + 'reset --soft `{git} merge-base {target} {backport}`'.format(git=git_prefix, target=self.target_branch, backport=self.backport_branch)) - os.system(git_prefix + 'commit -a -m "Squash backport branch"') - os.system(git_prefix + 'push -f origin {branch}:{branch}'.format(branch=self.backport_branch)) + self._run(git_prefix + ['checkout', '-f', self.backport_branch]) + self._run(git_prefix + ['pull', '--ff-only', 'origin', self.backport_branch]) + self._run(git_prefix + ['reset', '--soft', self._run(git_prefix + ['merge-base', self.target_branch, self.backport_branch])]) + self._run(git_prefix + ['commit', '-a', '-m', pr_title]) + self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.backport_branch)]) - pr = self._gh.create_pull_request(source=self.backport_branch, target=self.target_branch, - title='Backport #{number} to {target}: {title}'.format( - number=self._pr['number'], target=self.target_branch, title=self._pr['title'].replace('"', '\\"')), - description='Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}'.format(self._pr['number'], cherrypick_pr['number'], DESCRIPTION)) + pr = self._gh.create_pull_request(source=self.backport_branch, target=self.target_branch, title=pr_title, + description='Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}'.format(self._pr['number'], cherrypick_pr['number'], DESCRIPTION)) # FIXME: use `team` to leave a single eligible assignee. self._gh.add_assignee(pr, self._pr['author']) @@ -123,53 +132,50 @@ class CherryPick: return pr - -def run(token, pr, branch, repo, dry_run=False): - cp = CherryPick(token, 'ClickHouse', 'ClickHouse', 'core', pr, branch) - - pr1 = cp.getCherryPickPullRequest() - if not pr1: - if not dry_run: - pr1 = cp.createCherryPickPullRequest(repo) - logging.debug('Created PR with cherry-pick of %s to %s: %s', pr, branch, pr1['url']) + def execute(self, repo_path, dry_run=False): + pr1 = self.getCherryPickPullRequest() + if not pr1: + if not dry_run: + pr1 = self.createCherryPickPullRequest(repo_path) + logging.debug('Created PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + else: + return CherryPick.Status.NOT_INITIATED else: - return CherryPick.Status.NOT_INITIATED - else: - logging.debug('Found PR with cherry-pick of %s to %s: %s', pr, branch, pr1['url']) - - if not pr1['merged'] and pr1['mergeable'] == 'MERGEABLE' and not pr1['closed']: - if not dry_run: - pr1 = cp.mergeCherryPickPullRequest(pr1) - logging.debug('Merged PR with cherry-pick of %s to %s: %s', pr, branch, pr1['url']) - - if not pr1['merged']: - logging.debug('Waiting for PR with cherry-pick of %s to %s: %s', pr, branch, pr1['url']) - - if pr1['closed']: - return CherryPick.Status.DISCARDED - elif pr1['mergeable'] == 'CONFLICTING': - return CherryPick.Status.FIRST_CONFLICTS + logging.debug('Found PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + + if not pr1['merged'] and pr1['mergeable'] == 'MERGEABLE' and not pr1['closed']: + if not dry_run: + pr1 = self.mergeCherryPickPullRequest(pr1) + logging.debug('Merged PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + + if not pr1['merged']: + logging.debug('Waiting for PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + + if pr1['closed']: + return CherryPick.Status.DISCARDED + elif pr1['mergeable'] == 'CONFLICTING': + return CherryPick.Status.FIRST_CONFLICTS + else: + return CherryPick.Status.FIRST_MERGEABLE + + pr2 = self.getBackportPullRequest() + if not pr2: + if not dry_run: + pr2 = self.createBackportPullRequest(pr1, repo_path) + logging.debug('Created PR with backport of %s to %s: %s', self._pr['number'], self.target_branch, pr2['url']) + else: + return CherryPick.Status.FIRST_MERGEABLE else: - return CherryPick.Status.FIRST_MERGEABLE + logging.debug('Found PR with backport of %s to %s: %s', self._pr['number'], self.target_branch, pr2['url']) - pr2 = cp.getBackportPullRequest() - if not pr2: - if not dry_run: - pr2 = cp.createBackportPullRequest(pr1, repo) - logging.debug('Created PR with backport of %s to %s: %s', pr, branch, pr2['url']) + if pr2['merged']: + return CherryPick.Status.MERGED + elif pr2['closed']: + return CherryPick.Status.DISCARDED + elif pr2['mergeable'] == 'CONFLICTING': + return CherryPick.Status.SECOND_CONFLICTS else: - return CherryPick.Status.FIRST_MERGEABLE - else: - logging.debug('Found PR with backport of %s to %s: %s', pr, branch, pr2['url']) - - if pr2['merged']: - return CherryPick.Status.MERGED - elif pr2['closed']: - return CherryPick.Status.DISCARDED - elif pr2['mergeable'] == 'CONFLICTING': - return CherryPick.Status.SECOND_CONFLICTS - else: - return CherryPick.Status.SECOND_MERGEABLE + return CherryPick.Status.SECOND_MERGEABLE if __name__ == "__main__": @@ -182,4 +188,5 @@ if __name__ == "__main__": parser.add_argument('--repo', '-r', type=str, required=True, help='path to full repository', metavar='PATH') args = parser.parse_args() - run(args.token, args.pr, args.branch, args.repo) + cp = CherryPick(args.token, 'ClickHouse', 'ClickHouse', 'core', args.pr, args.branch) + cp.execute(args.repo) diff --git a/utils/github/local.py b/utils/github/local.py index 60d9f8ab1e..a997721bc7 100644 --- a/utils/github/local.py +++ b/utils/github/local.py @@ -1,11 +1,5 @@ # -*- coding: utf-8 -*- -try: - import git # `pip install gitpython` -except ImportError: - import sys - sys.exit("Package 'gitpython' not found. Try run: `pip install [--user] gitpython`") - import functools import logging import os @@ -14,6 +8,8 @@ import re class RepositoryBase(object): def __init__(self, repo_path): + import git + self._repo = git.Repo(repo_path, search_parent_directories=(not repo_path)) # commit comparator @@ -34,10 +30,12 @@ class RepositoryBase(object): for commit in self._repo.iter_commits(rev_range, first_parent=True): yield commit + class Repository(RepositoryBase): def __init__(self, repo_path, remote_name, default_branch_name): super(Repository, self).__init__(repo_path) self._remote = self._repo.remotes[remote_name] + self._remote.fetch() self._default = self._remote.refs[default_branch_name] def get_release_branches(self): @@ -63,6 +61,7 @@ class Repository(RepositoryBase): return sorted(release_branches, key=lambda x : self.comparator(x[1])) + class BareRepository(RepositoryBase): def __init__(self, repo_path, default_branch_name): super(BareRepository, self).__init__(repo_path) diff --git a/utils/github/query.py b/utils/github/query.py index bb39493cf5..e7afef8d6a 100644 --- a/utils/github/query.py +++ b/utils/github/query.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import requests -import time class Query: @@ -136,7 +135,7 @@ class Query: ''' query = _QUERY.format(owner=self._owner, name=self._name, number=number, - pull_request_data = self._PULL_REQUEST, min_page_size=self._min_page_size) + pull_request_data=self._PULL_REQUEST, min_page_size=self._min_page_size) return self._run(query)['repository']['pullRequest'] def find_pull_request(self, base, head): @@ -152,7 +151,7 @@ class Query: ''' query = _QUERY.format(owner=self._owner, name=self._name, base=base, head=head, - pull_request_data = self._PULL_REQUEST, min_page_size=self._min_page_size) + pull_request_data=self._PULL_REQUEST, min_page_size=self._min_page_size) result = self._run(query)['repository']['pullRequests'] if result['totalCount'] > 0: return result['nodes'][0] @@ -257,7 +256,7 @@ class Query: query = _QUERY.format(target=target, source=source, id=self._id, title=title, body=description, draft="true" if draft else "false", modify="true" if can_modify else "false", - pull_request_data = self._PULL_REQUEST) + pull_request_data=self._PULL_REQUEST) return self._run(query, is_mutation=True)['createPullRequest']['pullRequest'] def merge_pull_request(self, id): @@ -271,7 +270,7 @@ class Query: }} ''' - query = _QUERY.format(id=id, pull_request_data = self._PULL_REQUEST) + query = _QUERY.format(id=id, pull_request_data=self._PULL_REQUEST) return self._run(query, is_mutation=True)['mergePullRequest']['pullRequest'] # FIXME: figure out how to add more assignees at once @@ -340,10 +339,10 @@ class Query: if not labels: return - query = _SET_LABEL.format(pr_id = pull_request['id'], label_id = labels[0]['id']) + query = _SET_LABEL.format(pr_id=pull_request['id'], label_id=labels[0]['id']) self._run(query, is_mutation=True) - ### OLD METHODS + # OLD METHODS # _LABELS = ''' # repository(owner: "ClickHouse" name: "ClickHouse") {{ -- GitLab