From 918ababba6fce694c61d54bb2ff6983a886f696a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 26 May 2017 14:46:45 +0200 Subject: [PATCH] Add pipeline stages post deployment migration --- ... 20170525132202_create_pipeline_stages.rb} | 2 +- .../20170526101042_migrate_pipeline_stages.rb | 43 ++++++++++++++++++ db/schema.rb | 5 ++- .../migrate_pipeline_stages_spec.rb | 44 ++++++++++++++++--- spec/spec_helper.rb | 4 +- spec/support/db_cleaner.rb | 4 ++ spec/support/migrations_helpers.rb | 4 ++ 7 files changed, 95 insertions(+), 11 deletions(-) rename db/migrate/{20170525132202_migrate_pipeline_stages.rb => 20170525132202_create_pipeline_stages.rb} (81%) create mode 100644 db/post_migrate/20170526101042_migrate_pipeline_stages.rb diff --git a/db/migrate/20170525132202_migrate_pipeline_stages.rb b/db/migrate/20170525132202_create_pipeline_stages.rb similarity index 81% rename from db/migrate/20170525132202_migrate_pipeline_stages.rb rename to db/migrate/20170525132202_create_pipeline_stages.rb index a9fe00ebf72..c562df27357 100644 --- a/db/migrate/20170525132202_migrate_pipeline_stages.rb +++ b/db/migrate/20170525132202_create_pipeline_stages.rb @@ -1,4 +1,4 @@ -class MigratePipelineStages < ActiveRecord::Migration +class CreatePipelineStages < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false diff --git a/db/post_migrate/20170526101042_migrate_pipeline_stages.rb b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb new file mode 100644 index 00000000000..12ad0db3a60 --- /dev/null +++ b/db/post_migrate/20170526101042_migrate_pipeline_stages.rb @@ -0,0 +1,43 @@ +class MigratePipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + disable_statement_timeout + + execute <<-SQL.strip_heredoc + INSERT INTO ci_stages (project_id, pipeline_id, name) + SELECT project_id, commit_id, stage FROM ci_builds + WHERE stage IS NOT NULL + GROUP BY project_id, commit_id, stage, stage_idx + ORDER BY stage_idx + SQL + + add_concurrent_index(:ci_stages, [:pipeline_id, :name]) + + add_column(:ci_builds, :stage_id, :integer) + + stage_id = Arel.sql('(SELECT id FROM ci_stages ' \ + 'WHERE ci_stages.pipeline_id = ci_builds.commit_id ' \ + 'AND ci_stages.name = ci_builds.stage)') + update_column_in_batches(:ci_builds, :stage_id, stage_id) + + # add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade + # add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade + end + + def down + execute('TRUNCATE TABLE ci_stages') + + if column_exists?(:ci_builds, :stage_id) + remove_column(:ci_builds, :stage_id) + end + + if index_exists?(:ci_stages, [:pipeline_id, :name]) + remove_index(:ci_stages, [:pipeline_id, :name]) + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 581060f62fc..f1df8d6bd8c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170525132202) do +ActiveRecord::Schema.define(version: 20170526101042) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -233,6 +233,7 @@ ActiveRecord::Schema.define(version: 20170525132202) do t.string "coverage_regex" t.integer "auto_canceled_by_id" t.boolean "retried" + t.integer "stage_id" end add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree @@ -333,6 +334,8 @@ ActiveRecord::Schema.define(version: 20170525132202) do t.datetime "updated_at" end + add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree + create_table "ci_trigger_requests", force: :cascade do |t| t.integer "trigger_id", null: false t.text "variables" diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb index 4a469174b9e..9cf13a1dabc 100644 --- a/spec/migrations/migrate_pipeline_stages_spec.rb +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -1,19 +1,49 @@ require 'spec_helper' -require Rails.root.join('db', 'migrate', '20170525132202_migrate_pipeline_stages.rb') +require Rails.root.join('db', 'post_migrate', '20170526101042_migrate_pipeline_stages.rb') -describe MigratePipelineStages, :migration, schema: 20170523091700 do +describe MigratePipelineStages, :migration, schema: 20170525132202 do ## - # Create test data + # Create test data - pipeline and CI/CD jobs. # + + let(:jobs) { table(:ci_builds) } + let(:stages) { table(:ci_stages) } + let(:pipelines) { table(:ci_pipelines) } + before do - table(:ci_pipelines).create!(ref: 'master', sha: 'adf43c3a') + # Create CI/CD pipelines + # + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb') + + # Create CI/CD jobs + # + jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') + jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') + jobs.create!(id: 5, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') + jobs.create!(id: 8, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 9, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 10, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') + jobs.create!(id: 11, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy') end - it 'correctly migrates pipeline stages' do |migration, meta| - expect(ActiveRecord::Base.connection.table_exists?('ci_stages')).to eq false + it 'correctly migrates pipeline stages' do + expect(stages.count).to be_zero migrate! - expect(ActiveRecord::Base.connection.table_exists?('ci_stages')).to eq true + expect(stages.count).to eq 7 + expect(stages.all.pluck(:name)) + .to match_array %w[test build deploy test:1 test:2 deploy deploy] + expect(stages.where(pipeline_id: 1).order(:id).pluck(:name)) + .to eq %w[test build deploy] + expect(stages.where(pipeline_id: 2).order(:id).pluck(:name)) + .to eq %w[test:1 test:2 deploy] + + expect(jobs.where(stage_id: nil)).to be_empty end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 2c1c252be37..b4ece5c60c1 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -95,8 +95,8 @@ RSpec.configure do |config| end config.around(:example, migration: true) do |example| - ActiveRecord::Migrator - .migrate(migrations_paths, example.metadata.fetch(:schema)) + schema_version = example.metadata.fetch(:schema) + ActiveRecord::Migrator.migrate(migrations_paths, schema_version) example.run diff --git a/spec/support/db_cleaner.rb b/spec/support/db_cleaner.rb index 6f31828b825..7f5769209bb 100644 --- a/spec/support/db_cleaner.rb +++ b/spec/support/db_cleaner.rb @@ -19,6 +19,10 @@ RSpec.configure do |config| DatabaseCleaner.strategy = :truncation end + config.before(:each, :migration) do + DatabaseCleaner.strategy = :truncation + end + config.before(:each) do DatabaseCleaner.start end diff --git a/spec/support/migrations_helpers.rb b/spec/support/migrations_helpers.rb index 965b00b5731..ee17d1a40b7 100644 --- a/spec/support/migrations_helpers.rb +++ b/spec/support/migrations_helpers.rb @@ -7,6 +7,10 @@ module MigrationsHelpers ActiveRecord::Migrator.migrations_paths end + def table_exists?(name) + ActiveRecord::Base.connection.table_exists?(name) + end + def migrate! ActiveRecord::Migrator.up(migrations_paths) do |migration| migration.name == described_class.name -- GitLab