importer.rb

module Gitlab
  module BitbucketServerImport
    class Importer
      include Gitlab::ShellAdapter
      attr_reader :recover_missing_commits
      attr_reader :project, :project_key, :repository_slug, :client, :errors, :users

      REMOTE_NAME = 'bitbucket_server'.freeze
      BATCH_SIZE = 100

      TempBranch = Struct.new(:name, :sha)

      def self.imports_repository?
        true
      end

      def self.refmap
        [:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head']
      end

      # Unlike GitHub, you can't grab the commit SHAs for pull requests that
      # have been closed but not merged even though Bitbucket has these
      # commits internally. We can recover these pull requests by creating a
      # branch with the Bitbucket REST API, but by default we turn this
      # behavior off.
      def initialize(project, recover_missing_commits: false)
        @project = project
        @recover_missing_commits = recover_missing_commits
        @project_key = project.import_data.data['project_key']
        @repository_slug = project.import_data.data['repo_slug']
        @client = BitbucketServer::Client.new(project.import_data.credentials)
        @formatter = Gitlab::ImportFormatter.new
        @errors = []
        @users = {}
        @temp_branches = []
      end

      def execute
        import_repository
        import_pull_requests
        delete_temp_branches
        handle_errors

        true
      end

      private

      def handle_errors
        return unless errors.any?

        project.update_column(:import_error, {
          message: 'The remote data could not be fully imported.',
          errors: errors
        }.to_json)
      end

      def gitlab_user_id(email)
        find_user_id(email) || project.creator_id
      end

      def find_user_id(email)
        return nil unless email

        return users[email] if users.key?(email)

        user = User.find_by_any_email(email)
        users[email] = user&.id if user

        user&.id
      end

      def repo
        @repo ||= client.repo(project_key, repository_slug)
      end

      def sha_exists?(sha)
        project.repository.commit(sha)
      end

      def temp_branch_name(pull_request, suffix)
        "gitlab/import/pull-request/#{pull_request.iid}/#{suffix}"
      end

      # This method restores required SHAs that GitLab needs to create diffs
      # into branch names as the following:
      #
      # gitlab/import/pull-request/N/{to,from}
      def restore_branches(pull_requests)
        shas_to_restore = []

        pull_requests.each do |pull_request|
          shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :from),
                                            pull_request.source_branch_sha)
          shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :to),
                                            pull_request.target_branch_sha)
        end

        # Create the branches on the Bitbucket Server first
        created_branches = restore_branch_shas(shas_to_restore)

        @temp_branches += created_branches
        # Now sync the repository so we get the new branches
        import_repository unless created_branches.empty?
      end

      def restore_branch_shas(shas_to_restore)
        shas_to_restore.each_with_object([]) do |temp_branch, branches_created|
          branch_name = temp_branch.name
          sha = temp_branch.sha

          next if sha_exists?(sha)

          begin
            client.create_branch(project_key, repository_slug, branch_name, sha)
            branches_created << temp_branch
          rescue BitbucketServer::Connection::ConnectionError => e
            Rails.logger.warn("BitbucketServerImporter: Unable to recreate branch for SHA #{sha}: #{e}")
          end
        end
      end

      def import_repository
        project.ensure_repository
        project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME)
      rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e
        # Expire cache to prevent scenarios such as:
        # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
        # 2. Retried import, repo is broken or not imported but +exists?+ still returns true
        project.repository.expire_content_cache if project.repository_exists?

        raise e.message
      end

      # Bitbucket Server keeps tracks of references for open pull requests in
      # refs/heads/pull-requests, but closed and merged requests get moved
      # into hidden internal refs under stash-refs/pull-requests. Unless the
      # SHAs involved are at the tip of a branch or tag, there is no way to
      # retrieve the server for those commits.
      #
      # To avoid losing history, we use the Bitbucket API to re-create the branch
      # on the remote server. Then we have to issue a `git fetch` to download these
      # branches.
      def import_pull_requests
        pull_requests = client.pull_requests(project_key, repository_slug).to_a

        # Creating branches on the server and fetching the newly-created branches
        # may take a number of network round-trips. Do this in batches so that we can
        # avoid doing a git fetch for every new branch.
        pull_requests.each_slice(BATCH_SIZE) do |batch|
          restore_branches(batch) if recover_missing_commits

          batch.each do |pull_request|
            begin
              import_bitbucket_pull_request(pull_request)
            rescue StandardError => e
              errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw }
            end
          end
        end
      end

      def delete_temp_branches
        @temp_branches.each do |branch|
          begin
            client.delete_branch(project_key, repository_slug, branch.name, branch.sha)
            project.repository.delete_branch(branch.name)
          rescue BitbucketServer::Connection::ConnectionError => e
            @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message }
          end
        end
      end

      def import_bitbucket_pull_request(pull_request)
        description = ''
        description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author_email)
        description += pull_request.description

        source_branch_sha = pull_request.source_branch_sha
        target_branch_sha = pull_request.target_branch_sha
        source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha
        target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha
        author_id = gitlab_user_id(pull_request.author_email)

        project.merge_requests.find_by(iid: pull_request.iid)&.destroy

        attributes = {
          iid: pull_request.iid,
          title: pull_request.title,
          description: description,
          source_project: project,
          source_branch: Gitlab::Git.ref_name(pull_request.source_branch_name),
          source_branch_sha: source_branch_sha,
          target_project: project,
          target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name),
          target_branch_sha: target_branch_sha,
          state: pull_request.state,
          author_id: author_id,
          assignee_id: nil,
          created_at: pull_request.created_at,
          updated_at: pull_request.updated_at
        }

        attributes[:merge_commit_sha] = target_branch_sha if pull_request.merged?
        merge_request = project.merge_requests.create!(attributes)
        import_pull_request_comments(pull_request, merge_request) if merge_request.persisted?
      end

      def import_pull_request_comments(pull_request, merge_request)
        comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?)

        merge_event = other_activities.find(&:merge_event?)
        import_merge_event(merge_request, merge_event) if merge_event

        inline_comments, pr_comments = comments.partition(&:inline_comment?)

        import_inline_comments(inline_comments.map(&:comment), merge_request)
        import_standalone_pr_comments(pr_comments.map(&:comment), merge_request)
      end

      def import_merge_event(merge_request, merge_event)
        committer = merge_event.committer_email

        user_id = gitlab_user_id(committer)
        timestamp = merge_event.merge_timestamp
        metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request)
        metric.update(merged_by_id: user_id, merged_at: timestamp)
      end

      def import_inline_comments(inline_comments, merge_request)
        inline_comments.each do |comment|
          position = build_position(merge_request, comment)
          parent = create_diff_note(merge_request, comment, position)

          next unless parent&.persisted?

          discussion_id = parent.discussion_id

          comment.comments.each do |reply|
            create_diff_note(merge_request, reply, position, discussion_id)
          end
        end
      end

      def create_diff_note(merge_request, comment, position, discussion_id = nil)
        attributes = pull_request_comment_attributes(comment)
        attributes.merge!(position: position, type: 'DiffNote')
        attributes[:discussion_id] = discussion_id if discussion_id

        note = merge_request.notes.build(attributes)

        if note.valid?
          note.save
          return note
        end

        # Bitbucket Server supports the ability to comment on any line, not just the
        # line in the diff. If we can't add the note as a DiffNote, fallback to creating
        # a regular note.
        create_fallback_diff_note(merge_request, comment)
      rescue StandardError => e
        errors << { type: :pull_request, id: comment.id, errors: e.message }
        nil
      end

      def create_fallback_diff_note(merge_request, comment)
        attributes = pull_request_comment_attributes(comment)
        attributes[:note] = "*Comment on file: #{comment.file_path}, old line: #{comment.old_pos}, new line: #{comment.new_pos}*\n\n" + attributes[:note]

        merge_request.notes.create!(attributes)
      end

      def build_position(merge_request, pr_comment)
        params = {
          diff_refs: merge_request.diff_refs,
          old_path: pr_comment.file_path,
          new_path: pr_comment.file_path,
          old_line: pr_comment.old_pos,
          new_line: pr_comment.new_pos
        }

        Gitlab::Diff::Position.new(params)
      end

      def import_standalone_pr_comments(pr_comments, merge_request)
        pr_comments.each do |comment|
          begin
            merge_request.notes.create!(pull_request_comment_attributes(comment))

            comment.comments.each do |replies|
              merge_request.notes.create!(pull_request_comment_attributes(replies))
            end
          rescue StandardError => e
            errors << { type: :pull_request, iid: comment.id, errors: e.message }
          end
        end
      end

      def pull_request_comment_attributes(comment)
        {
          project: project,
          note: comment.note,
          author_id: gitlab_user_id(comment.author_email),
          created_at: comment.created_at,
          updated_at: comment.updated_at
        }
      end
    end
  end
end