diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 7940f7c2ee702e54b1056e2308b56b5dec939465..80a634342838a88d0bd43b518d70c50dbe5d02bb 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -52,13 +52,13 @@ module Ci end # Find stale live traces and return their build ids - def find_stale_in_batches(finished_before: 1.hour.ago) + def find_builds_from_stale_live_trace include(EachBatch) .select(:build_id) .group(:build_id) .joins(:build) .merge(Ci::Build.finished) - .where('ci_builds.finished_at < ?', finished_before) + .where('ci_builds.finished_at < ?', 1.hour.ago) .each_batch(column: :build_id) do |chunks| build_ids = chunks.map { |chunk| chunk.build_id } diff --git a/app/workers/ci/rescue_stale_live_trace_worker.rb b/app/workers/ci/rescue_stale_live_trace_worker.rb index be0194b63aea0b9e6556f4ec8b7ea7eee108d6d2..c4a462446d4bbf8237f87352bef17bcce150f64c 100644 --- a/app/workers/ci/rescue_stale_live_trace_worker.rb +++ b/app/workers/ci/rescue_stale_live_trace_worker.rb @@ -4,12 +4,10 @@ module Ci include CronjobQueue def perform - # Reschedule to archive live traces - # - # The targets are jobs with the following conditions - # - Jobs had been finished 1 hour ago, but they don't have an archived trace yet - # This could happen when their sidekiq-jobs are lost by SIGKILL - Ci::BuildTraceChunk.find_stale_in_batches(finished_before: 1.hour.ago) do |build_ids| + # Archive live traces which still resides in redis or database + # This could happen when sidekiq-jobs for archivements are lost by SIGKILL + # Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/36791 + Ci::BuildTraceChunk.find_builds_from_stale_live_trace do |build_ids| Ci::Build.where(id: build_ids).find_each do |build| begin build.trace.archive! diff --git a/spec/models/ci/build_trace_chunk_spec.rb b/spec/models/ci/build_trace_chunk_spec.rb index 246152d1c8fa7c5f853c254cea439d86cb2a3266..5d524d7cf4969ea7ad4d659c6838be4ee26b4f1f 100644 --- a/spec/models/ci/build_trace_chunk_spec.rb +++ b/spec/models/ci/build_trace_chunk_spec.rb @@ -35,8 +35,8 @@ describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do end end - describe '.find_stale_in_batches' do - subject { described_class.find_stale_in_batches } + describe '.find_builds_from_stale_live_trace' do + subject { described_class.find_builds_from_stale_live_trace } context 'when build status is finished' do context 'when build finished 2 days ago' do @@ -44,7 +44,7 @@ describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do let!(:build) { create(:ci_build, :success, :trace_artifact, finished_at: 2.days.ago) } it 'does not yield build id' do - expect { |b| described_class.find_stale_in_batches(&b) }.not_to yield_control + expect { |b| described_class.find_builds_from_stale_live_trace(&b) }.not_to yield_control end end @@ -52,7 +52,7 @@ describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do let!(:build) { create(:ci_build, :success, :trace_live, finished_at: 2.days.ago) } it 'yields build id' do - expect { |b| described_class.find_stale_in_batches(&b) }.to yield_with_args([build.id]) + expect { |b| described_class.find_builds_from_stale_live_trace(&b) }.to yield_with_args([build.id]) end end end @@ -61,7 +61,7 @@ describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do let!(:build) { create(:ci_build, :success, :trace_live, finished_at: 10.minutes.ago) } it 'does not yield build id' do - expect { |b| described_class.find_stale_in_batches(&b) }.not_to yield_control + expect { |b| described_class.find_builds_from_stale_live_trace(&b) }.not_to yield_control end end end @@ -70,7 +70,7 @@ describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do let!(:build) { create(:ci_build, :running, :trace_live) } it 'does not yield build id' do - expect { |b| described_class.find_stale_in_batches(&b) }.not_to yield_control + expect { |b| described_class.find_builds_from_stale_live_trace(&b) }.not_to yield_control end end end