From 7b22381603d4f2cddc87b1b70e5be076df63cf01 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Wed, 7 Mar 2018 13:52:29 +0100 Subject: [PATCH] Reschedule commits_count background migration We still have >100K unmigrated MergeRequestDiffs which don't have commits_count set yet (see https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/17567#note_61904891) This migration re-schedules the original background migration. To assure that records are not processed twice, records with commits_count set are skipped. Related to #41698 and !17567 --- ...le_commits_count_for_merge_request_diff.rb | 30 +++++++++++++++ db/schema.rb | 2 +- .../add_merge_request_diff_commits_count.rb | 2 +- ...d_merge_request_diff_commits_count_spec.rb | 12 ++++++ ...mmits_count_for_merge_request_diff_spec.rb | 37 +++++++++++++++++++ 5 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20180309121820_reschedule_commits_count_for_merge_request_diff.rb create mode 100644 spec/migrations/reschedule_commits_count_for_merge_request_diff_spec.rb diff --git a/db/migrate/20180309121820_reschedule_commits_count_for_merge_request_diff.rb b/db/migrate/20180309121820_reschedule_commits_count_for_merge_request_diff.rb new file mode 100644 index 00000000000..990759104b0 --- /dev/null +++ b/db/migrate/20180309121820_reschedule_commits_count_for_merge_request_diff.rb @@ -0,0 +1,30 @@ +class RescheduleCommitsCountForMergeRequestDiff < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + MIGRATION = 'AddMergeRequestDiffCommitsCount'.freeze + BATCH_SIZE = 5000 + DELAY_INTERVAL = 5.minutes.to_i + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + include ::EachBatch + end + + disable_ddl_transaction! + + def up + say 'Populating the MergeRequestDiff `commits_count` (reschedule)' + + execute("SET statement_timeout TO '60s'") if Gitlab::Database.postgresql? + + MergeRequestDiff.where(commits_count: nil).each_batch(of: BATCH_SIZE) do |relation, index| + start_id, end_id = relation.pluck('MIN(id), MAX(id)').first + delay = index * DELAY_INTERVAL + + BackgroundMigrationWorker.perform_in(delay, MIGRATION, [start_id, end_id]) + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 75a094bbbb6..970b1ad9948 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180308052825) do +ActiveRecord::Schema.define(version: 20180309121820) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb index 7bffffec94d..d5cf9e0d53a 100644 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -19,7 +19,7 @@ module Gitlab WHERE merge_request_diffs.id = merge_request_diff_commits.merge_request_diff_id )'.squish - MergeRequestDiff.where(id: start_id..stop_id).update_all(update) + MergeRequestDiff.where(id: start_id..stop_id).where(commits_count: nil).update_all(update) end end end diff --git a/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb b/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb index 21a791f5695..c43ed72038e 100644 --- a/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb +++ b/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb @@ -37,6 +37,18 @@ describe Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount, :migratio expect(diff.reload.commits_count).to eq(0) end + it 'skips diffs that have commits_count already set' do + timestamp = 2.days.ago + diff = merge_request_diffs_table.create!( + merge_request_id: merge_request.id, + commits_count: 0, + updated_at: timestamp) + + subject.perform(diff.id, diff.id) + + expect(diff.reload.updated_at).to be_within(1.second).of(timestamp) + end + it 'migrates multiple diffs to the correct values' do diffs = Array.new(3).map.with_index { |_, i| create_diff!(i, commits: 3) } diff --git a/spec/migrations/reschedule_commits_count_for_merge_request_diff_spec.rb b/spec/migrations/reschedule_commits_count_for_merge_request_diff_spec.rb new file mode 100644 index 00000000000..26489ef58bd --- /dev/null +++ b/spec/migrations/reschedule_commits_count_for_merge_request_diff_spec.rb @@ -0,0 +1,37 @@ +require 'spec_helper' +require Rails.root.join('db', 'migrate', '20180309121820_reschedule_commits_count_for_merge_request_diff') + +describe RescheduleCommitsCountForMergeRequestDiff, :migration, :sidekiq do + let(:merge_request_diffs) { table(:merge_request_diffs) } + let(:merge_requests) { table(:merge_requests) } + let(:projects) { table(:projects) } + let(:namespaces) { table(:namespaces) } + + before do + stub_const("#{described_class.name}::BATCH_SIZE", 1) + + namespaces.create!(id: 1, name: 'gitlab', path: 'gitlab') + + projects.create!(id: 1, namespace_id: 1) + + merge_requests.create!(id: 1, target_project_id: 1, source_project_id: 1, target_branch: 'feature', source_branch: 'master') + + merge_request_diffs.create!(id: 1, merge_request_id: 1) + merge_request_diffs.create!(id: 2, merge_request_id: 1) + merge_request_diffs.create!(id: 3, merge_request_id: 1, commits_count: 0) + merge_request_diffs.create!(id: 4, merge_request_id: 1) + end + + it 'correctly schedules background migrations' do + Sidekiq::Testing.fake! do + Timecop.freeze do + migrate! + + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(5.minutes, 1, 1) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(10.minutes, 2, 2) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(15.minutes, 4, 4) + expect(BackgroundMigrationWorker.jobs.size).to eq 3 + end + end + end +end -- GitLab