diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index d2402b55184ce10b47223aacfba476cfd2ec7958..85960f1b6bb2a1dd698c48ed0dcc965878f5907b 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -6,6 +6,8 @@ module Ci include Presentable include Importable + MissingDependenciesError = Class.new(StandardError) + belongs_to :runner belongs_to :trigger_request belongs_to :erased_by, class_name: 'User' @@ -139,6 +141,10 @@ module Ci Ci::Build.retry(build, build.user) end end + + before_transition any => [:running] do |build| + build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies') + end end def detailed_status(current_user) @@ -478,6 +484,20 @@ module Ci options[:dependencies]&.empty? end + def validates_dependencies! + dependencies.each do |dependency| + raise MissingDependenciesError unless dependency.valid_dependency? + end + end + + def valid_dependency? + return false unless complete? + return false if artifacts_expired? + return false if erased? + + true + end + def hide_secrets(trace) return unless trace diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index ee21ed8e42095502e8654b471c69ea0b66a7c076..c0263c0b4e2a7447a13ad1361637e7eb73f1fc44 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -43,7 +43,8 @@ class CommitStatus < ActiveRecord::Base script_failure: 1, api_failure: 2, stuck_or_timeout_failure: 3, - runner_system_failure: 4 + runner_system_failure: 4, + missing_dependency_failure: 5 } ## diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index 2ef76e030310e0bd5832a5c5ed9ed9804ca2e488..c8b6450c9b59432a0e3fa5ac45900935883a0b97 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -54,6 +54,9 @@ module Ci # we still have to return 409 in the end, # to make sure that this is properly handled by runner. valid = false + rescue Ci::Build::MissingDependenciesError + build.drop!(:missing_dependency_failure) + valid = false end end diff --git a/changelogs/unreleased/feature-sm-34834-missing-dependency-should-fail-job-2.yml b/changelogs/unreleased/feature-sm-34834-missing-dependency-should-fail-job-2.yml new file mode 100644 index 0000000000000000000000000000000000000000..ab85b8ee51580aa7af3327bcd3d3d308a6e893e0 --- /dev/null +++ b/changelogs/unreleased/feature-sm-34834-missing-dependency-should-fail-job-2.yml @@ -0,0 +1,5 @@ +--- +title: Fail jobs if its dependency is missing +merge_request: 14009 +author: +type: fixed diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 86b436d89ddae02caf2f165d3765c07991bb40e6..33f8a69c249cf0b3546c1f0430d958b3b540c0a0 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -128,6 +128,45 @@ steps below. 1. Save the file and [restart GitLab][] for the changes to take effect. +## Validation for dependencies + +> Introduced in GitLab 10.3. + +To disable [the dependencies validation](../ci/yaml/README.md#when-a-dependent-job-will-fail), +you can flip the feature flag from a Rails console. + +--- + +**In Omnibus installations:** + +1. Enter the Rails console: + + ```sh + sudo gitlab-rails console + ``` + +1. Flip the switch and disable it: + + ```ruby + Feature.enable('ci_disable_validates_dependencies') + ``` +--- + +**In installations from source:** + +1. Enter the Rails console: + + ```sh + cd /home/git/gitlab + RAILS_ENV=production sudo -u git -H bundle exec rails console + ``` + +1. Flip the switch and disable it: + + ```ruby + Feature.enable('ci_disable_validates_dependencies') + ``` + ## Set the maximum file size of the artifacts Provided the artifacts are enabled, you can change the maximum file size of the diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index f40d2c5e34770da3c4e767a4158329495bf16929..32464cbb259e24c938d297ed2f877d7ac582c386 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1153,6 +1153,20 @@ deploy: script: make deploy ``` +#### When a dependent job will fail + +> Introduced in GitLab 10.3. + +If the artifacts of the job that is set as a dependency have been +[expired](#artifacts-expire_in) or +[erased](../../user/project/pipelines/job_artifacts.md#erasing-artifacts), then +the dependent job will fail. + +NOTE: **Note:** +You can ask your administrator to +[flip this switch](../../administration/job_artifacts.md#validation-for-dependencies) +and bring back the old behavior. + ### before_script and after_script It's possible to overwrite the globally defined `before_script` and `after_script`: diff --git a/doc/user/project/pipelines/job_artifacts.md b/doc/user/project/pipelines/job_artifacts.md index f9a268fb789df3935f432d2dc28de27b804f577e..402989f4508e70f8d1268a22e678371bf2ce5ad6 100644 --- a/doc/user/project/pipelines/job_artifacts.md +++ b/doc/user/project/pipelines/job_artifacts.md @@ -44,7 +44,7 @@ the artifacts will be kept forever. For more examples on artifacts, follow the [artifacts reference in `.gitlab-ci.yml`](../../../ci/yaml/README.md#artifacts). -## Browsing job artifacts +## Browsing artifacts >**Note:** With GitLab 9.2, PDFs, images, videos and other formats can be previewed @@ -77,7 +77,7 @@ one HTML file that you can view directly online when --- -## Downloading job artifacts +## Downloading artifacts If you need to download the whole archive, there are buttons in various places inside GitLab that make that possible. @@ -102,7 +102,7 @@ inside GitLab that make that possible. ![Job artifacts browser](img/job_artifacts_browser.png) -## Downloading the latest job artifacts +## Downloading the latest artifacts It is possible to download the latest artifacts of a job via a well known URL so you can use it for scripting purposes. @@ -163,6 +163,18 @@ information in the UI. ![Latest artifacts button](img/job_latest_artifacts_browser.png) +## Erasing artifacts + +DANGER: **Warning:** +This is a destructive action that leads to data loss. Use with caution. + +If you have at least Developer [permissions](../../permissions.md#gitlab-ci-cd-permissions) +on the project, you can erase a single job via the UI which will also remove the +artifacts and the job's trace. + +1. Navigate to a job's page. +1. Click the trash icon at the top right of the job's trace. +1. Confirm the deletion. [expiry date]: ../../../ci/yaml/README.md#artifacts-expire_in [ce-14399]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14399 diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 26d33663dad365b1548308e17ec4701893b79d70..a6258676767d967f53131ea9707c56e076648385 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1868,6 +1868,94 @@ describe Ci::Build do end end + describe 'state transition: any => [:running]' do + shared_examples 'validation is active' do + context 'when depended job has not been completed yet' do + let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it { expect { job.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + end + + context 'when artifacts of depended job has been expired' do + let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it { expect { job.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + end + + context 'when artifacts of depended job has been erased' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) } + + before do + pre_stage_job.erase + end + + it { expect { job.run! }.to raise_error(Ci::Build::MissingDependenciesError) } + end + end + + shared_examples 'validation is not active' do + context 'when depended job has not been completed yet' do + let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it { expect { job.run! }.not_to raise_error } + end + + context 'when artifacts of depended job has been expired' do + let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it { expect { job.run! }.not_to raise_error } + end + + context 'when artifacts of depended job has been erased' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) } + + before do + pre_stage_job.erase + end + + it { expect { job.run! }.not_to raise_error } + end + end + + let!(:job) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: options) } + + context 'when validates for dependencies is enabled' do + before do + stub_feature_flags(ci_disable_validates_dependencies: false) + end + + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) } + + context 'when "dependencies" keyword is not defined' do + let(:options) { {} } + + it { expect { job.run! }.not_to raise_error } + end + + context 'when "dependencies" keyword is empty' do + let(:options) { { dependencies: [] } } + + it { expect { job.run! }.not_to raise_error } + end + + context 'when "dependencies" keyword is specified' do + let(:options) { { dependencies: ['test'] } } + + it_behaves_like 'validation is active' + end + end + + context 'when validates for dependencies is disabled' do + let(:options) { { dependencies: ['test'] } } + + before do + stub_feature_flags(ci_disable_validates_dependencies: true) + end + + it_behaves_like 'validation is not active' + end + end + describe 'state transition when build fails' do let(:service) { MergeRequests::AddTodoWhenBuildFailsService.new(project, user) } diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index d4b1e7c8dd4fc343f260860901da743d3858e3f8..bb89e09389008c16eb9bb3b2d111963b57ad6c08 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -1244,7 +1244,7 @@ describe Ci::Pipeline, :mailer do describe '#execute_hooks' do let!(:build_a) { create_build('a', 0) } - let!(:build_b) { create_build('b', 1) } + let!(:build_b) { create_build('b', 0) } let!(:hook) do create(:project_hook, project: project, pipeline_events: enabled) @@ -1300,6 +1300,8 @@ describe Ci::Pipeline, :mailer do end context 'when stage one failed' do + let!(:build_b) { create_build('b', 1) } + before do build_a.drop end diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index decdd577226c1c350e52f65647d7d925c9bad5c2..3ee59014b5b1bfd434af51d54adae636392f8642 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -276,6 +276,89 @@ module Ci end end + context 'when "dependencies" keyword is specified' do + shared_examples 'not pick' do + it 'does not pick the build and drops the build' do + expect(subject).to be_nil + expect(pending_job.reload).to be_failed + expect(pending_job).to be_missing_dependency_failure + end + end + + shared_examples 'validation is active' do + context 'when depended job has not been completed yet' do + let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it_behaves_like 'not pick' + end + + context 'when artifacts of depended job has been expired' do + let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it_behaves_like 'not pick' + end + + context 'when artifacts of depended job has been erased' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) } + + before do + pre_stage_job.erase + end + + it_behaves_like 'not pick' + end + end + + shared_examples 'validation is not active' do + context 'when depended job has not been completed yet' do + let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it { expect(subject).to eq(pending_job) } + end + + context 'when artifacts of depended job has been expired' do + let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) } + + it { expect(subject).to eq(pending_job) } + end + + context 'when artifacts of depended job has been erased' do + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) } + + before do + pre_stage_job.erase + end + + it { expect(subject).to eq(pending_job) } + end + end + + before do + stub_feature_flags(ci_disable_validates_dependencies: false) + end + + let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) } + let!(:pending_job) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['test'] } ) } + + subject { execute(specific_runner) } + + context 'when validates for dependencies is enabled' do + before do + stub_feature_flags(ci_disable_validates_dependencies: false) + end + + it_behaves_like 'validation is active' + end + + context 'when validates for dependencies is disabled' do + before do + stub_feature_flags(ci_disable_validates_dependencies: true) + end + + it_behaves_like 'validation is not active' + end + end + def execute(runner) described_class.new(runner).execute.build end