diff --git a/app/services/git/base_hooks_service.rb b/app/services/git/base_hooks_service.rb index fce4040e39037edefa408d4c47f1b46e3770167b..a8478e3a904d50891f7c322dabddd64be0c2f30e 100644 --- a/app/services/git/base_hooks_service.rb +++ b/app/services/git/base_hooks_service.rb @@ -51,6 +51,8 @@ module Git end def create_pipelines + return unless params.fetch(:create_pipelines, true) + Ci::CreatePipelineService .new(project, current_user, push_data) .execute(:push, pipeline_options) diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index a5554f07699bebfaed0073e6d4e51785b264ce1a..337efa7919b722e441a52a39fea70c9864f454a6 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -3,6 +3,8 @@ class PostReceive include ApplicationWorker + PIPELINE_PROCESS_LIMIT = 4 + def perform(gl_repository, identifier, changes, push_options = {}) project, repo_type = Gitlab::GlRepository.parse(gl_repository) @@ -36,23 +38,24 @@ class PostReceive return false end - post_received.changes_refs do |oldrev, newrev, ref| - if Gitlab::Git.tag_ref?(ref) - Git::TagPushService.new( - post_received.project, - @user, - oldrev: oldrev, - newrev: newrev, - ref: ref, - push_options: post_received.push_options).execute - elsif Gitlab::Git.branch_ref?(ref) - Git::BranchPushService.new( + post_received.enum_for(:changes_refs).with_index do |(oldrev, newrev, ref), index| + service_klass = + if Gitlab::Git.tag_ref?(ref) + Git::TagPushService + elsif Gitlab::Git.branch_ref?(ref) + Git::BranchPushService + end + + if service_klass + service_klass.new( post_received.project, @user, oldrev: oldrev, newrev: newrev, ref: ref, - push_options: post_received.push_options).execute + push_options: post_received.push_options, + create_pipelines: index < PIPELINE_PROCESS_LIMIT || Feature.enabled?(:git_push_create_all_pipelines, post_received.project) + ).execute end changes << Gitlab::DataBuilder::Repository.single_change(oldrev, newrev, ref) diff --git a/changelogs/unreleased/limit-amount-of-created-pipelines.yml b/changelogs/unreleased/limit-amount-of-created-pipelines.yml new file mode 100644 index 0000000000000000000000000000000000000000..51fdbb4d7ffbdfeea35bda186c59d4a9fcbd459e --- /dev/null +++ b/changelogs/unreleased/limit-amount-of-created-pipelines.yml @@ -0,0 +1,5 @@ +--- +title: Process at most 4 pipelines during push +merge_request: 27205 +author: +type: performance diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 5e44de13b516e1766f2bed53b11b60953fd0a717..79ab6ee6d62a65d004af7f3a9354c2afdc403857 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -2697,6 +2697,15 @@ Not to be confused with [`trigger`](#trigger-premium). [Read more in the triggers documentation.](../triggers/README.md) +## Processing Git pushes + +GitLab will create at most 4 branch and tags pipelines when +doing pushing multiple changes in single `git push` invocation. + +This limitation does not affect any of the updated Merge Request pipelines, +all updated Merge Requests will have a pipeline created when using +[pipelines for merge requests](../merge_request_pipelines/index.md). + ## Skipping jobs If your commit message contains `[ci skip]` or `[skip ci]`, using any diff --git a/spec/workers/post_receive_spec.rb b/spec/workers/post_receive_spec.rb index a3fe8fa4501e955335a46e73aa2f1431e8e04ee7..39f1beb4efaccdfcfef4eb479706224d1c4cae21 100644 --- a/spec/workers/post_receive_spec.rb +++ b/spec/workers/post_receive_spec.rb @@ -99,11 +99,21 @@ describe PostReceive do end context "gitlab-ci.yml" do - let(:changes) { "123456 789012 refs/heads/feature\n654321 210987 refs/tags/tag" } + let(:changes) do + <<-EOF.strip_heredoc + 123456 789012 refs/heads/feature + 654321 210987 refs/tags/tag + 123456 789012 refs/heads/feature2 + 123458 789013 refs/heads/feature3 + 123459 789015 refs/heads/feature4 + EOF + end + + let(:changes_count) { changes.lines.count } subject { described_class.new.perform(gl_repository, key_id, base64_changes) } - context "creates a Ci::Pipeline for every change" do + context "with valid .gitlab-ci.yml" do before do stub_ci_pipeline_to_return_yaml_file @@ -116,7 +126,33 @@ describe PostReceive do .and_return(true) end - it { expect { subject }.to change { Ci::Pipeline.count }.by(2) } + context 'when git_push_create_all_pipelines is disabled' do + before do + stub_feature_flags(git_push_create_all_pipelines: false) + end + + it "creates pipeline for branches and tags" do + subject + + expect(Ci::Pipeline.pluck(:ref)).to contain_exactly("feature", "tag", "feature2", "feature3") + end + + it "creates exactly #{described_class::PIPELINE_PROCESS_LIMIT} pipelines" do + expect(changes_count).to be > described_class::PIPELINE_PROCESS_LIMIT + + expect { subject }.to change { Ci::Pipeline.count }.by(described_class::PIPELINE_PROCESS_LIMIT) + end + end + + context 'when git_push_create_all_pipelines is enabled' do + before do + stub_feature_flags(git_push_create_all_pipelines: true) + end + + it "creates all pipelines" do + expect { subject }.to change { Ci::Pipeline.count }.by(changes_count) + end + end end context "does not create a Ci::Pipeline" do