Migration to remove pending delete projects with non-existing namespace

There might be some projects where the namespace was removed, but the
project wasn't. For these the projects still have a `namespace_id`
set. So this adds a post-deploy migration remove all projects that are
pending delete, and have a `namespace_id` that is non-existing.

app/workers/namespaceless_project_destroy_worker.rb

@@ -18,7 +18,8 @@ class NamespacelessProjectDestroyWorker
     rescue ActiveRecord::RecordNotFound
       return
     end
-    return unless project.namespace_id.nil?  # Reject doing anything for projects that *do* have a namespace
+
+    return if namespace?(project)  # Reject doing anything for projects that *do* have a namespace
 
     project.team.truncate
 
@@ -29,6 +30,10 @@ class NamespacelessProjectDestroyWorker
 
   private
 
+  def namespace?(project)
+    project.namespace_id && Namespace.exists?(project.namespace_id)
+  end
+
   def unlink_fork(project)
     merge_requests = project.forked_from_project.merge_requests.opened.from_project(project)
 

changelogs/unreleased/tc-remove-nonexisting-namespace-pending-delete-projects.yml

+---
+title: Migration to remove pending delete projects with non-existing namespace
+merge_request: 13598
+author:
+type: other

db/post_migrate/20170816102555_cleanup_nonexisting_namespace_pending_delete_projects.rb

+# Follow up of CleanupNamespacelessPendingDeleteProjects and it cleans
+# all projects with `pending_delete = true` and for which the
+# namespace no longer exists.
+class CleanupNonexistingNamespacePendingDeleteProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    @offset = 0
+
+    loop do
+      ids = pending_delete_batch
+
+      break if ids.empty?
+
+      args = ids.map { |id| Array(id) }
+
+      NamespacelessProjectDestroyWorker.bulk_perform_async(args)
+
+      @offset += 1
+    end
+  end
+
+  def down
+    # noop
+  end
+
+  private
+
+  def pending_delete_batch
+    connection.exec_query(find_batch).map { |row| row['id'].to_i }
+  end
+
+  BATCH_SIZE = 5000
+
+  def find_batch
+    projects = Project.arel_table
+    namespaces = Namespace.arel_table
+
+    namespace_query = namespaces.project(1)
+                        .where(namespaces[:id].eq(projects[:namespace_id]))
+                        .exists.not
+
+    projects.project(projects[:id])
+      .where(projects[:pending_delete].eq(true))
+      .where(namespace_query)
+      .skip(@offset * BATCH_SIZE)
+      .take(BATCH_SIZE)
+      .to_sql
+  end
+end

spec/migrations/cleanup_nonexisting_namespace_pending_delete_projects_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170816102555_cleanup_nonexisting_namespace_pending_delete_projects.rb')
+
+describe CleanupNonexistingNamespacePendingDeleteProjects do
+  before do
+    # Stub after_save callbacks that will fail when Project has invalid namespace
+    allow_any_instance_of(Project).to receive(:ensure_storage_path_exist).and_return(nil)
+    allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil)
+  end
+
+  describe '#up' do
+    set(:some_project) { create(:project) }
+
+    it 'only cleans up when namespace does not exist' do
+      create(:project, pending_delete: true)
+      project = build(:project, pending_delete: true, namespace: nil, namespace_id: Namespace.maximum(:id).to_i.succ)
+      project.save(validate: false)
+
+      expect(NamespacelessProjectDestroyWorker).to receive(:bulk_perform_async).with([[project.id]])
+
+      described_class.new.up
+    end
+
+    it 'does nothing when no pending delete projects without namespace found' do
+      create(:project, pending_delete: true, namespace: create(:namespace))
+
+      expect(NamespacelessProjectDestroyWorker).not_to receive(:bulk_perform_async)
+
+      described_class.new.up
+    end
+  end
+end

spec/workers/namespaceless_project_destroy_worker_spec.rb

@@ -75,5 +75,19 @@ describe NamespacelessProjectDestroyWorker do
         end
       end
     end
+
+    context 'project has non-existing namespace' do
+      let!(:project) do
+        project = build(:project, namespace_id: Namespace.maximum(:id).to_i.succ)
+        project.save(validate: false)
+        project
+      end
+
+      it 'deletes the project' do
+        subject.perform(project.id)
+
+        expect(Project.unscoped.all).not_to include(project)
+      end
+    end
   end
 end