Update scripts (#12650)

utils/github/backport.py

 # -*- coding: utf-8 -*-
 
-from query import Query as RemoteRepo
-from local import BareRepository as LocalRepo
-import cherrypick
+from clickhouse.utils.github.cherrypick import CherryPick
+from clickhouse.utils.github.query import Query as RemoteRepo
+from clickhouse.utils.github.local import Repository as LocalRepo
 
 import argparse
 import logging
@@ -12,83 +12,78 @@ import sys
 
 class Backport:
     def __init__(self, token, owner, name, team):
-        '''
-        `refs` is a list of (ref_path, base_commit) sorted by ancestry starting from the least recent ref.
-        '''
         self._gh = RemoteRepo(token, owner=owner, name=name, team=team, max_page_size=30)
+        self._token = token
         self.default_branch_name = self._gh.default_branch
+        self.ssh_url = self._gh.ssh_url
 
     def getPullRequests(self, from_commit):
         return self._gh.get_pull_requests(from_commit)
 
+    def execute(self, repo, til, number, run_cherrypick):
+        repo = LocalRepo(repo, 'origin', self.default_branch_name)
+        branches = repo.get_release_branches()[-number:]  # [(branch_name, base_commit)]
 
-def run(token, repo_bare, til, number, run_cherrypick):
-    bp = Backport(token, 'ClickHouse', 'ClickHouse', 'core')
-    repo = LocalRepo(repo_bare, bp.default_branch_name)
+        if not branches:
+            logging.info('No release branches found!')
+            return
 
-    branches = repo.get_release_branches()[-number:]  # [(branch_name, base_commit)]
-
-    if not branches:
-        logging.info('No release branches found!')
-        return
-
-    for branch in branches:
-        logging.info('Found release branch: %s', branch[0])
+        for branch in branches:
+            logging.info('Found release branch: %s', branch[0])
 
-    if not til:
-        til = branches[0][1]
-    prs = bp.getPullRequests(til)
+        if not til:
+            til = branches[0][1]
+        prs = self.getPullRequests(til)
 
-    backport_map = {}
+        backport_map = {}
 
-    RE_MUST_BACKPORT = re.compile(r'^v(\d+\.\d+)-must-backport$')
-    RE_NO_BACKPORT = re.compile(r'^v(\d+\.\d+)-no-backport$')
+        RE_MUST_BACKPORT = re.compile(r'^v(\d+\.\d+)-must-backport$')
+        RE_NO_BACKPORT = re.compile(r'^v(\d+\.\d+)-no-backport$')
 
-    # pull-requests are sorted by ancestry from the least recent.
-    for pr in prs:
-        while repo.comparator(branches[-1][1]) >= repo.comparator(pr['mergeCommit']['oid']):
-            branches.pop()
+        # pull-requests are sorted by ancestry from the least recent.
+        for pr in prs:
+            while repo.comparator(branches[-1][1]) >= repo.comparator(pr['mergeCommit']['oid']):
+                branches.pop()
 
-        assert len(branches)
+            assert len(branches)
 
-        branch_set = set([branch[0] for branch in branches])
+            branch_set = set([branch[0] for branch in branches])
 
-        # First pass. Find all must-backports
-        for label in pr['labels']['nodes']:
-            if label['name'].startswith('pr-') and label['color'] == 'ff0000':
-                backport_map[pr['number']] = branch_set.copy()
-                continue
-            m = RE_MUST_BACKPORT.match(label['name'])
-            if m:
-                if pr['number'] not in backport_map:
-                    backport_map[pr['number']] = set()
-                backport_map[pr['number']].add(m.group(1))
+            # First pass. Find all must-backports
+            for label in pr['labels']['nodes']:
+                if label['name'].startswith('pr-') and label['color'] == 'ff0000':
+                    backport_map[pr['number']] = branch_set.copy()
+                    continue
+                m = RE_MUST_BACKPORT.match(label['name'])
+                if m:
+                    if pr['number'] not in backport_map:
+                        backport_map[pr['number']] = set()
+                    backport_map[pr['number']].add(m.group(1))
 
-        # Second pass. Find all no-backports
-        for label in pr['labels']['nodes']:
-            if label['name'] == 'pr-no-backport' and pr['number'] in backport_map:
-                del backport_map[pr['number']]
-                break
-            m = RE_NO_BACKPORT.match(label['name'])
-            if m and pr['number'] in backport_map and m.group(1) in backport_map[pr['number']]:
-                backport_map[pr['number']].remove(m.group(1))
+            # Second pass. Find all no-backports
+            for label in pr['labels']['nodes']:
+                if label['name'] == 'pr-no-backport' and pr['number'] in backport_map:
+                    del backport_map[pr['number']]
+                    break
+                m = RE_NO_BACKPORT.match(label['name'])
+                if m and pr['number'] in backport_map and m.group(1) in backport_map[pr['number']]:
+                    backport_map[pr['number']].remove(m.group(1))
 
-    for pr, branches in backport_map.items():
-        logging.info('PR #%s needs to be backported to:', pr)
-        for branch in branches:
-            logging.info('\t%s %s', branch, run_cherrypick(token, pr, branch))
+        for pr, branches in backport_map.items():
+            logging.info('PR #%s needs to be backported to:', pr)
+            for branch in branches:
+                logging.info('\t%s %s', branch, run_cherrypick(self._token, pr, branch))
 
-    # print API costs
-    logging.info('\nGitHub API total costs per query:')
-    for name, value in bp._gh.api_costs.items():
-        logging.info('%s : %s', name, value)
+        # print API costs
+        logging.info('\nGitHub API total costs per query:')
+        for name, value in self._gh.api_costs.items():
+            logging.info('%s : %s', name, value)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument('--token',     type=str, required=True, help='token for Github access')
-    parser.add_argument('--repo-bare', type=str, required=True, help='path to bare repository', metavar='PATH')
-    parser.add_argument('--repo-full', type=str, required=True, help='path to full repository', metavar='PATH')
+    parser.add_argument('--repo',      type=str, required=True, help='path to full repository', metavar='PATH')
     parser.add_argument('--til',       type=str,                help='check PRs from HEAD til this commit', metavar='COMMIT')
     parser.add_argument('-n',          type=int, dest='number', help='number of last release branches to consider')
     parser.add_argument('--dry-run',   action='store_true',     help='do not create or merge any PRs', default=False)
@@ -100,5 +95,6 @@ if __name__ == "__main__":
     else:
         logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.INFO)
 
-    cherrypick_run = lambda token, pr, branch: cherrypick.run(token, pr, branch, args.repo_full, args.dry_run)
-    run(args.token, args.repo_bare, args.til, args.number, cherrypick_run)
+    cherrypick_run = lambda token, pr, branch: CherryPick(token, 'ClickHouse', 'ClickHouse', 'core', pr, branch).execute(args.repo, args.dry_run)
+    bp = Backport(args.token, 'ClickHouse', 'ClickHouse', 'core')
+    bp.execute(args.repo, args.til, args.number, cherrypick_run)

utils/github/cherrypick.py

@@ -14,12 +14,13 @@ Second run checks PR from previous run to be merged or at least being mergeable.
 Third run creates PR from backport branch (with merged previous PR) to release branch.
 '''
 
-from query import Query as RemoteRepo
+from clickhouse.utils.github.query import Query as RemoteRepo
 
 import argparse
 from enum import Enum
 import logging
 import os
+import subprocess
 import sys
 
 
@@ -33,10 +34,15 @@ class CherryPick:
         SECOND_CONFLICTS = 'conflicts on 2nd stage'
         MERGED = 'backported'
 
+    def _run(self, args):
+        logging.info(subprocess.check_output(args))
+
     def __init__(self, token, owner, name, team, pr_number, target_branch):
         self._gh = RemoteRepo(token, owner=owner, name=name, team=team)
         self._pr = self._gh.get_pull_request(pr_number)
 
+        self.ssh_url = self._gh.ssh_url
+
         # TODO: check if pull-request is merged.
 
         self.merge_commit_oid = self._pr['mergeCommit']['oid']
@@ -60,25 +66,26 @@ class CherryPick:
         )
 
         # FIXME: replace with something better than os.system()
-        git_prefix = 'git -C {} -c "user.email=robot-clickhouse@yandex-team.ru" -c "user.name=robot-clickhouse" '.format(repo_path)
+        git_prefix = ['git', '-C', repo_path, '-c', 'user.email=robot-clickhouse@yandex-team.ru', '-c', 'user.name=robot-clickhouse']
         base_commit_oid = self._pr['mergeCommit']['parents']['nodes'][0]['oid']
 
         # Create separate branch for backporting, and make it look like real cherry-pick.
-        os.system(git_prefix + 'checkout -f ' + self.target_branch)
-        os.system(git_prefix + 'checkout -B ' + self.backport_branch)
-        os.system(git_prefix + 'merge -s ours --no-edit ' + base_commit_oid)
+        self._run(git_prefix + ['checkout', '-f', self.target_branch])
+        self._run(git_prefix + ['checkout', '-B', self.backport_branch])
+        self._run(git_prefix + ['merge', '-s', 'ours', '--no-edit', base_commit_oid])
 
         # Create secondary branch to allow pull request with cherry-picked commit.
-        os.system(git_prefix + 'branch -f {} {}'.format(self.cherrypick_branch, self.merge_commit_oid))
+        self._run(git_prefix + ['branch', '-f', self.cherrypick_branch, self.merge_commit_oid])
 
-        os.system(git_prefix + 'push -f origin {branch}:{branch}'.format(branch=self.backport_branch))
-        os.system(git_prefix + 'push -f origin {branch}:{branch}'.format(branch=self.cherrypick_branch))
+        self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.backport_branch)])
+        self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.cherrypick_branch)])
 
         # Create pull-request like a local cherry-pick
         pr = self._gh.create_pull_request(source=self.cherrypick_branch, target=self.backport_branch,
-            title='Cherry pick #{number} to {target}: {title}'.format(
-                number=self._pr['number'], target=self.target_branch, title=self._pr['title'].replace('"', '\\"')),
-            description='Original pull-request #{}\n\n{}'.format(self._pr['number'], DESCRIPTION))
+                                          title='Cherry pick #{number} to {target}: {title}'.format(
+                                              number=self._pr['number'], target=self.target_branch,
+                                              title=self._pr['title'].replace('"', '\\"')),
+                                          description='Original pull-request #{}\n\n{}'.format(self._pr['number'], DESCRIPTION))
 
         # FIXME: use `team` to leave a single eligible assignee.
         self._gh.add_assignee(pr, self._pr['author'])
@@ -102,18 +109,20 @@ class CherryPick:
             'Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n'
         )
 
-        git_prefix = 'git -C {} -c "user.email=robot-clickhouse@yandex-team.ru" -c "user.name=robot-clickhouse" '.format(repo_path)
+        git_prefix = ['git', '-C', repo_path, '-c', 'user.email=robot-clickhouse@yandex-team.ru', '-c', 'user.name=robot-clickhouse']
+
+        pr_title = 'Backport #{number} to {target}: {title}'.format(
+            number=self._pr['number'], target=self.target_branch,
+            title=self._pr['title'].replace('"', '\\"'))
 
-        os.system(git_prefix + 'checkout -f ' + self.backport_branch)
-        os.system(git_prefix + 'pull --ff-only origin ' + self.backport_branch)
-        os.system(git_prefix + 'reset --soft `{git} merge-base {target} {backport}`'.format(git=git_prefix, target=self.target_branch, backport=self.backport_branch))
-        os.system(git_prefix + 'commit -a -m "Squash backport branch"')
-        os.system(git_prefix + 'push -f origin {branch}:{branch}'.format(branch=self.backport_branch))
+        self._run(git_prefix + ['checkout', '-f', self.backport_branch])
+        self._run(git_prefix + ['pull', '--ff-only', 'origin', self.backport_branch])
+        self._run(git_prefix + ['reset', '--soft', self._run(git_prefix + ['merge-base', self.target_branch, self.backport_branch])])
+        self._run(git_prefix + ['commit', '-a', '-m', pr_title])
+        self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.backport_branch)])
 
-        pr = self._gh.create_pull_request(source=self.backport_branch, target=self.target_branch,
-            title='Backport #{number} to {target}: {title}'.format(
-                number=self._pr['number'], target=self.target_branch, title=self._pr['title'].replace('"', '\\"')),
-            description='Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}'.format(self._pr['number'], cherrypick_pr['number'], DESCRIPTION))
+        pr = self._gh.create_pull_request(source=self.backport_branch, target=self.target_branch, title=pr_title,
+                                          description='Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}'.format(self._pr['number'], cherrypick_pr['number'], DESCRIPTION))
 
         # FIXME: use `team` to leave a single eligible assignee.
         self._gh.add_assignee(pr, self._pr['author'])
@@ -123,53 +132,50 @@ class CherryPick:
 
         return pr
 
-
-def run(token, pr, branch, repo, dry_run=False):
-    cp = CherryPick(token, 'ClickHouse', 'ClickHouse', 'core', pr, branch)
-
-    pr1 = cp.getCherryPickPullRequest()
-    if not pr1:
-        if not dry_run:
-            pr1 = cp.createCherryPickPullRequest(repo)
-            logging.debug('Created PR with cherry-pick of %s to %s: %s', pr, branch, pr1['url'])
+    def execute(self, repo_path, dry_run=False):
+        pr1 = self.getCherryPickPullRequest()
+        if not pr1:
+            if not dry_run:
+                pr1 = self.createCherryPickPullRequest(repo_path)
+                logging.debug('Created PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url'])
+            else:
+                return CherryPick.Status.NOT_INITIATED
         else:
-            return CherryPick.Status.NOT_INITIATED
-    else:
-        logging.debug('Found PR with cherry-pick of %s to %s: %s', pr, branch, pr1['url'])
-
-    if not pr1['merged'] and pr1['mergeable'] == 'MERGEABLE' and not pr1['closed']:
-        if not dry_run:
-            pr1 = cp.mergeCherryPickPullRequest(pr1)
-            logging.debug('Merged PR with cherry-pick of %s to %s: %s', pr, branch, pr1['url'])
-
-    if not pr1['merged']:
-        logging.debug('Waiting for PR with cherry-pick of %s to %s: %s', pr, branch, pr1['url'])
-
-        if pr1['closed']:
-            return CherryPick.Status.DISCARDED
-        elif pr1['mergeable'] == 'CONFLICTING':
-            return CherryPick.Status.FIRST_CONFLICTS
+            logging.debug('Found PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url'])
+
+        if not pr1['merged'] and pr1['mergeable'] == 'MERGEABLE' and not pr1['closed']:
+            if not dry_run:
+                pr1 = self.mergeCherryPickPullRequest(pr1)
+                logging.debug('Merged PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url'])
+
+        if not pr1['merged']:
+            logging.debug('Waiting for PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url'])
+
+            if pr1['closed']:
+                return CherryPick.Status.DISCARDED
+            elif pr1['mergeable'] == 'CONFLICTING':
+                return CherryPick.Status.FIRST_CONFLICTS
+            else:
+                return CherryPick.Status.FIRST_MERGEABLE
+
+        pr2 = self.getBackportPullRequest()
+        if not pr2:
+            if not dry_run:
+                pr2 = self.createBackportPullRequest(pr1, repo_path)
+                logging.debug('Created PR with backport of %s to %s: %s', self._pr['number'], self.target_branch, pr2['url'])
+            else:
+                return CherryPick.Status.FIRST_MERGEABLE
         else:
-            return CherryPick.Status.FIRST_MERGEABLE
+            logging.debug('Found PR with backport of %s to %s: %s', self._pr['number'], self.target_branch, pr2['url'])
 
-    pr2 = cp.getBackportPullRequest()
-    if not pr2:
-        if not dry_run:
-            pr2 = cp.createBackportPullRequest(pr1, repo)
-            logging.debug('Created PR with backport of %s to %s: %s', pr, branch, pr2['url'])
+        if pr2['merged']:
+            return CherryPick.Status.MERGED
+        elif pr2['closed']:
+            return CherryPick.Status.DISCARDED
+        elif pr2['mergeable'] == 'CONFLICTING':
+            return CherryPick.Status.SECOND_CONFLICTS
         else:
-            return CherryPick.Status.FIRST_MERGEABLE
-    else:
-        logging.debug('Found PR with backport of %s to %s: %s', pr, branch, pr2['url'])
-
-    if pr2['merged']:
-        return CherryPick.Status.MERGED
-    elif pr2['closed']:
-        return CherryPick.Status.DISCARDED
-    elif pr2['mergeable'] == 'CONFLICTING':
-        return CherryPick.Status.SECOND_CONFLICTS
-    else:
-        return CherryPick.Status.SECOND_MERGEABLE
+            return CherryPick.Status.SECOND_MERGEABLE
 
 
 if __name__ == "__main__":
@@ -182,4 +188,5 @@ if __name__ == "__main__":
     parser.add_argument('--repo',   '-r', type=str, required=True, help='path to full repository', metavar='PATH')
     args = parser.parse_args()
 
-    run(args.token, args.pr, args.branch, args.repo)
+    cp = CherryPick(args.token, 'ClickHouse', 'ClickHouse', 'core', args.pr, args.branch)
+    cp.execute(args.repo)

utils/github/local.py

 # -*- coding: utf-8 -*-
 
-try:
-    import git # `pip install gitpython`
-except ImportError:
-    import sys
-    sys.exit("Package 'gitpython' not found. Try run: `pip install [--user] gitpython`")
-
 import functools
 import logging
 import os
@@ -14,6 +8,8 @@ import re
 
 class RepositoryBase(object):
     def __init__(self, repo_path):
+        import git
+
         self._repo = git.Repo(repo_path, search_parent_directories=(not repo_path))
 
         # commit comparator
@@ -34,10 +30,12 @@ class RepositoryBase(object):
         for commit in self._repo.iter_commits(rev_range, first_parent=True):
             yield commit
 
+
 class Repository(RepositoryBase):
     def __init__(self, repo_path, remote_name, default_branch_name):
         super(Repository, self).__init__(repo_path)
         self._remote = self._repo.remotes[remote_name]
+        self._remote.fetch()
         self._default = self._remote.refs[default_branch_name]
 
     def get_release_branches(self):
@@ -63,6 +61,7 @@ class Repository(RepositoryBase):
 
         return sorted(release_branches, key=lambda x : self.comparator(x[1]))
 
+
 class BareRepository(RepositoryBase):
     def __init__(self, repo_path, default_branch_name):
         super(BareRepository, self).__init__(repo_path)

utils/github/query.py

 # -*- coding: utf-8 -*-
 
 import requests
-import time
 
 
 class Query:
@@ -136,7 +135,7 @@ class Query:
         '''
 
         query = _QUERY.format(owner=self._owner, name=self._name, number=number,
-                              pull_request_data = self._PULL_REQUEST, min_page_size=self._min_page_size)
+                              pull_request_data=self._PULL_REQUEST, min_page_size=self._min_page_size)
         return self._run(query)['repository']['pullRequest']
 
     def find_pull_request(self, base, head):
@@ -152,7 +151,7 @@ class Query:
         '''
 
         query = _QUERY.format(owner=self._owner, name=self._name, base=base, head=head,
-                              pull_request_data = self._PULL_REQUEST, min_page_size=self._min_page_size)
+                              pull_request_data=self._PULL_REQUEST, min_page_size=self._min_page_size)
         result = self._run(query)['repository']['pullRequests']
         if result['totalCount'] > 0:
             return result['nodes'][0]
@@ -257,7 +256,7 @@ class Query:
 
         query = _QUERY.format(target=target, source=source, id=self._id, title=title, body=description,
                               draft="true" if draft else "false", modify="true" if can_modify else "false",
-                              pull_request_data = self._PULL_REQUEST)
+                              pull_request_data=self._PULL_REQUEST)
         return self._run(query, is_mutation=True)['createPullRequest']['pullRequest']
 
     def merge_pull_request(self, id):
@@ -271,7 +270,7 @@ class Query:
             }}
         '''
 
-        query = _QUERY.format(id=id, pull_request_data = self._PULL_REQUEST)
+        query = _QUERY.format(id=id, pull_request_data=self._PULL_REQUEST)
         return self._run(query, is_mutation=True)['mergePullRequest']['pullRequest']
 
     # FIXME: figure out how to add more assignees at once
@@ -340,10 +339,10 @@ class Query:
         if not labels:
             return
 
-        query = _SET_LABEL.format(pr_id = pull_request['id'], label_id = labels[0]['id'])
+        query = _SET_LABEL.format(pr_id=pull_request['id'], label_id=labels[0]['id'])
         self._run(query, is_mutation=True)
 
-    ### OLD METHODS
+    # OLD METHODS
 
     # _LABELS = '''
     #     repository(owner: "ClickHouse" name: "ClickHouse") {{