From 6ed490401f49a8941dc7a9e3757ec4012f14ef0b Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Thu, 24 Aug 2017 13:01:33 +0200 Subject: [PATCH] Implement the implied CI/CD config for AutoDevOps Behind an application setting, which defaults to false, this commit implements the implied CI/CD config. Which means that in the case we can't find the `.gitlab-ci.yml` on the commit we want to start a pipeline for, we fall back to an implied configuration. For now the Bash template has been copied to `Auto-Devops.gitlab-ci.yml` so the tests actually work. Fixes #34777 --- app/helpers/application_settings_helper.rb | 1 + app/models/ci/pipeline.rb | 22 +++++++--- app/models/project.rb | 6 +++ app/models/project_auto_devops.rb | 3 ++ .../application_settings/_form.html.haml | 8 +++- .../unreleased/zj-auto-devops-table.yml | 5 +++ config/initializers/0_inflections.rb | 7 ++- ..._devops_enabled_to_application_settings.rb | 15 +++++++ ...70828093725_create_project_auto_dev_ops.rb | 24 ++++++++++ db/schema.rb | 23 +++++++--- spec/factories/project_auto_devops.rb | 6 +++ spec/models/application_setting_spec.rb | 1 + spec/models/ci/pipeline_spec.rb | 44 ++++++++++++++++--- spec/models/project_auto_devops_spec.rb | 12 +++++ .../gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml | 35 +++++++++++++++ 15 files changed, 194 insertions(+), 18 deletions(-) create mode 100644 app/models/project_auto_devops.rb create mode 100644 changelogs/unreleased/zj-auto-devops-table.yml create mode 100644 db/migrate/20170824101926_add_auto_devops_enabled_to_application_settings.rb create mode 100644 db/migrate/20170828093725_create_project_auto_dev_ops.rb create mode 100644 spec/factories/project_auto_devops.rb create mode 100644 spec/models/project_auto_devops_spec.rb create mode 100644 vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 04955ed625e..f97f7199648 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -103,6 +103,7 @@ module ApplicationSettingsHelper :after_sign_up_text, :akismet_api_key, :akismet_enabled, + :auto_devops_enabled, :clientside_sentry_dsn, :clientside_sentry_enabled, :container_registry_token_expire_delay, diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 2d40f8012a3..53ff42c04f4 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -338,12 +338,10 @@ module Ci def ci_yaml_file return @ci_yaml_file if defined?(@ci_yaml_file) - @ci_yaml_file = begin - project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path) - rescue Rugged::ReferenceError, GRPC::NotFound, GRPC::Internal - self.yaml_errors = - "Failed to load CI/CD config file at #{ci_yaml_file_path}" - nil + @ci_yaml_file = (ci_yaml_from_repo || implied_ci_yaml_file).tap do |config| + unless config + self.yaml_errors = "Failed to load CI/CD config file for #{sha}" + end end end @@ -430,6 +428,18 @@ module Ci private + def implied_ci_yaml_file + if project.auto_devops_enabled? + Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content + end + end + + def ci_yaml_from_repo + project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path) + rescue GRPC::NotFound, Rugged::ReferenceError, GRPC::Internal + nil + end + def pipeline_data Gitlab::DataBuilder::Pipeline.build(self) end diff --git a/app/models/project.rb b/app/models/project.rb index 5b4904a5c51..56ba2620687 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -186,6 +186,8 @@ class Project < ActiveRecord::Base has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' + has_one :auto_devops, class_name: 'ProjectAutoDevops' + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature accepts_nested_attributes_for :import_data @@ -464,6 +466,10 @@ class Project < ActiveRecord::Base self[:lfs_enabled] && Gitlab.config.lfs.enabled end + def auto_devops_enabled? + auto_devops&.enabled? || current_application_settings.auto_devops_enabled? + end + def repository_storage_path Gitlab.config.repositories.storages[repository_storage]['path'] end diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb new file mode 100644 index 00000000000..fe73c6f0a85 --- /dev/null +++ b/app/models/project_auto_devops.rb @@ -0,0 +1,3 @@ +class ProjectAutoDevops < ActiveRecord::Base + belongs_to :project +end diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index 734a08c61fa..a487f2b6995 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -217,7 +217,13 @@ .help-block 0 for unlimited %fieldset - %legend Continuous Integration + %legend Continuous Integration and Continuous Deployment + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :auto_devops_enabled do + = f.check_box :auto_devops_enabled + Allow projects to get automaticly configured to use GitLab CI/CD. .form-group .col-sm-offset-2.col-sm-10 .checkbox diff --git a/changelogs/unreleased/zj-auto-devops-table.yml b/changelogs/unreleased/zj-auto-devops-table.yml new file mode 100644 index 00000000000..f1a004ebd19 --- /dev/null +++ b/changelogs/unreleased/zj-auto-devops-table.yml @@ -0,0 +1,5 @@ +--- +title: Allow users and administrator to configure Auto-DevOps +merge_request: 13923 +author: +type: added diff --git a/config/initializers/0_inflections.rb b/config/initializers/0_inflections.rb index f977104ff9d..1ad9ddca877 100644 --- a/config/initializers/0_inflections.rb +++ b/config/initializers/0_inflections.rb @@ -10,5 +10,10 @@ # end # ActiveSupport::Inflector.inflections do |inflect| - inflect.uncountable %w(award_emoji project_statistics system_note_metadata) + inflect.uncountable %w( + award_emoji + project_statistics + system_note_metadata + project_auto_devops + ) end diff --git a/db/migrate/20170824101926_add_auto_devops_enabled_to_application_settings.rb b/db/migrate/20170824101926_add_auto_devops_enabled_to_application_settings.rb new file mode 100644 index 00000000000..da518d8215c --- /dev/null +++ b/db/migrate/20170824101926_add_auto_devops_enabled_to_application_settings.rb @@ -0,0 +1,15 @@ +class AddAutoDevopsEnabledToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default(:application_settings, :auto_devops_enabled, :boolean, default: false) + end + + def down + remove_column(:application_settings, :auto_devops_enabled, :boolean) + end +end diff --git a/db/migrate/20170828093725_create_project_auto_dev_ops.rb b/db/migrate/20170828093725_create_project_auto_dev_ops.rb new file mode 100644 index 00000000000..818fc8ed970 --- /dev/null +++ b/db/migrate/20170828093725_create_project_auto_dev_ops.rb @@ -0,0 +1,24 @@ +class CreateProjectAutoDevOps < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + create_table :project_auto_devops do |t| + t.belongs_to :project, index: true + t.boolean :enabled, default: true + t.string :domain + end + + add_timestamps_with_timezone(:project_auto_devops, null: false) + + # No need to check for violations as its a new table + add_concurrent_foreign_key(:project_auto_devops, :projects, column: :project_id) + end + + def down + drop_table(:project_auto_devops) + end +end diff --git a/db/schema.rb b/db/schema.rb index 0f4b0c0c3b3..66ab6988649 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170824162758) do +ActiveRecord::Schema.define(version: 20170828093725) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -125,10 +125,11 @@ ActiveRecord::Schema.define(version: 20170824162758) do t.boolean "prometheus_metrics_enabled", default: false, null: false t.boolean "help_page_hide_commercial_content", default: false t.string "help_page_support_url" - t.integer "performance_bar_allowed_group_id" t.boolean "password_authentication_enabled" - t.boolean "project_export_enabled", default: true, null: false + t.integer "performance_bar_allowed_group_id" t.boolean "hashed_storage_enabled", default: false, null: false + t.boolean "project_export_enabled", default: true, null: false + t.boolean "auto_devops_enabled", default: false, null: false end create_table "audit_events", force: :cascade do |t| @@ -249,6 +250,7 @@ ActiveRecord::Schema.define(version: 20170824162758) do add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree + add_index "ci_builds", ["id"], name: "index_for_ci_builds_retried_migration", where: "(retried IS NULL)", using: :btree, opclasses: {"id)"=>"WHERE"} add_index "ci_builds", ["project_id"], name: "index_ci_builds_on_project_id", using: :btree add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree add_index "ci_builds", ["stage_id"], name: "index_ci_builds_on_stage_id", using: :btree @@ -1116,6 +1118,16 @@ ActiveRecord::Schema.define(version: 20170824162758) do add_index "project_authorizations", ["project_id"], name: "index_project_authorizations_on_project_id", using: :btree add_index "project_authorizations", ["user_id", "project_id", "access_level"], name: "index_project_authorizations_on_user_id_project_id_access_level", unique: true, using: :btree + create_table "project_auto_devops", force: :cascade do |t| + t.integer "project_id" + t.boolean "enabled", default: true + t.string "domain" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + add_index "project_auto_devops", ["project_id"], name: "index_project_auto_devops_on_project_id", using: :btree + create_table "project_features", force: :cascade do |t| t.integer "project_id" t.integer "merge_requests_access_level" @@ -1199,6 +1211,7 @@ ActiveRecord::Schema.define(version: 20170824162758) do t.string "repository_storage", default: "default", null: false t.boolean "request_access_enabled", default: false, null: false t.boolean "has_external_wiki" + t.string "ci_config_path" t.boolean "lfs_enabled" t.text "description_html" t.boolean "only_allow_merge_if_all_discussions_are_resolved" @@ -1206,9 +1219,8 @@ ActiveRecord::Schema.define(version: 20170824162758) do t.integer "auto_cancel_pending_pipelines", default: 1, null: false t.string "import_jid" t.integer "cached_markdown_version" - t.datetime "last_repository_updated_at" - t.string "ci_config_path" t.text "delete_error" + t.datetime "last_repository_updated_at" t.integer "storage_version", limit: 2 end @@ -1722,6 +1734,7 @@ ActiveRecord::Schema.define(version: 20170824162758) do add_foreign_key "personal_access_tokens", "users" add_foreign_key "project_authorizations", "projects", on_delete: :cascade add_foreign_key "project_authorizations", "users", on_delete: :cascade + add_foreign_key "project_auto_devops", "projects", name: "fk_45436b12b2", on_delete: :cascade add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade diff --git a/spec/factories/project_auto_devops.rb b/spec/factories/project_auto_devops.rb new file mode 100644 index 00000000000..2e5a7c805c9 --- /dev/null +++ b/spec/factories/project_auto_devops.rb @@ -0,0 +1,6 @@ +FactoryGirl.define do + factory :project_auto_devops do + project + enabled true + end +end diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 359753b600e..95ebd016064 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -5,6 +5,7 @@ describe ApplicationSetting do it { expect(setting).to be_valid } it { expect(setting.uuid).to be_present } + it { expect(setting).to have_db_column(:auto_devops_enabled) } describe 'validations' do let(:http) { 'http://example.com' } diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index b84e3ff18e8..a7e0da04f55 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -784,13 +784,47 @@ describe Ci::Pipeline, :mailer do end describe '#ci_yaml_file' do - it 'reports error if the file is not found' do - allow(pipeline.project).to receive(:ci_config_path) { 'custom' } + let(:implied_yml) { Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content } - pipeline.ci_yaml_file + context 'when AutoDevops is enabled' do + it 'returns the configuration if found' do + allow(pipeline.project.repository).to receive(:gitlab_ci_yml_for) + .and_return('config') - expect(pipeline.yaml_errors) - .to eq('Failed to load CI/CD config file at custom') + expect(pipeline.ci_yaml_file).to be_a(String) + expect(pipeline.ci_yaml_file).not_to eq(implied_yml) + expect(pipeline.yaml_errors).to be_nil + end + + it 'returns the implied configuration when its not found' do + allow_any_instance_of(ApplicationSetting) + .to receive(:auto_devops_enabled?) { true } + allow(pipeline.project).to receive(:ci_config_path) { 'custom' } + + expect(pipeline.ci_yaml_file).to eq(implied_yml) + end + end + + context 'when AudoDevOps is disabled' do + context 'when an invalid path is given' do + it 'sets the yaml errors' do + allow(pipeline.project).to receive(:ci_config_path) { 'custom' } + + expect(pipeline.ci_yaml_file).to be_nil + expect(pipeline.yaml_errors) + .to start_with('Failed to load CI/CD config file') + end + end + + context 'when the config file can be found' do + it 'has no yaml_errors' do + allow(pipeline.project.repository).to receive(:gitlab_ci_yml_for) + .and_return('config') + + expect(pipeline.ci_yaml_file).to eq('config') + expect(pipeline.yaml_errors).to be_nil + end + end end end diff --git a/spec/models/project_auto_devops_spec.rb b/spec/models/project_auto_devops_spec.rb new file mode 100644 index 00000000000..d534dacf079 --- /dev/null +++ b/spec/models/project_auto_devops_spec.rb @@ -0,0 +1,12 @@ +require 'spec_helper' + +describe ProjectAutoDevops, type: :model do + subject { build_stubbed(:project_auto_devops) } + + it { is_expected.to belong_to(:project) } + + it { is_expected.to respond_to(:created_at) } + it { is_expected.to respond_to(:updated_at) } + + it { is_expected.to be_enabled } +end diff --git a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml new file mode 100644 index 00000000000..27537689b80 --- /dev/null +++ b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml @@ -0,0 +1,35 @@ +# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options + +# you can delete this line if you're not using Docker +image: busybox:latest + +before_script: + - echo "Before script section" + - echo "For example you might run an update here or install a build dependency" + - echo "Or perhaps you might print out some debugging details" + +after_script: + - echo "After script section" + - echo "For example you might do some cleanup here" + +build1: + stage: build + script: + - echo "Do your build here" + +test1: + stage: test + script: + - echo "Do a test here" + - echo "For example run a test suite" + +test2: + stage: test + script: + - echo "Do another parallel test here" + - echo "For example run a lint test" + +deploy1: + stage: deploy + script: + - echo "Do your deploy here" \ No newline at end of file -- GitLab