Ignore Gitaly errors if cache flushing fails on project destruction

We should just ignore these errors and move along with the deletion
since the repositories are going to be trashed anyway.

Closes https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31164

app/services/projects/destroy_service.rb

@@ -210,11 +210,20 @@ module Projects
     end
 
     def flush_caches(project)
-      project.repository.before_delete
+      ignore_git_errors(repo_path) { project.repository.before_delete }
 
-      Repository.new(wiki_path, project, disk_path: repo_path).before_delete
+      ignore_git_errors(wiki_path) { Repository.new(wiki_path, project, disk_path: repo_path).before_delete }
 
       Projects::ForksCountService.new(project).delete_cache
     end
+
+    # If we get a Gitaly error, the repository may be corrupted. We can
+    # ignore these errors since we're going to trash the repositories
+    # anyway.
+    def ignore_git_errors(disk_path, &block)
+      yield
+    rescue Gitlab::Git::CommandError => e
+      Gitlab::GitLogger.warn(class: self.class.name, project_id: project.id, disk_path: disk_path, message: e.to_s)
+    end
   end
 end

changelogs/unreleased/sh-ignore-git-errors-delete-project.yml

+---
+title: Ignore Gitaly errors if cache flushing fails on project destruction
+merge_request: 31164
+author:
+type: fixed

spec/services/projects/destroy_service_spec.rb

@@ -121,7 +121,22 @@ describe Projects::DestroyService do
     it { expect(Dir.exist?(remove_path)).to be_truthy }
   end
 
-  context 'when flushing caches fail' do
+  context 'when flushing caches fail due to Git errors' do
+    before do
+      allow(project.repository).to receive(:before_delete).and_raise(::Gitlab::Git::CommandError)
+      allow(Gitlab::GitLogger).to receive(:warn).with(
+        class: described_class.name,
+        project_id: project.id,
+        disk_path: project.disk_path,
+        message: 'Gitlab::Git::CommandError').and_call_original
+
+      perform_enqueued_jobs { destroy_project(project, user, {}) }
+    end
+
+    it_behaves_like 'deleting the project'
+  end
+
+  context 'when flushing caches fail due to Redis' do
     before do
       new_user = create(:user)
       project.team.add_user(new_user, Gitlab::Access::DEVELOPER)