diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index ca74c91b0627b5861e8052b0c94c31a081c22ee1..c6a7984870fcc80c4c4bfc07d8fe1eec4048db11 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -19,6 +19,10 @@ module Ci
       name
     end
 
+    def index
+      statuses.first.stage_idx
+    end
+
     def statuses_count
       @statuses_count ||= statuses.count
     end
@@ -45,6 +49,14 @@ module Ci
       status.to_s == 'success'
     end
 
+    def failed?
+      status.to_s == 'failed'
+    end
+
+    def canceled?
+      status.to_s == 'canceled'
+    end
+
     def has_warnings?
       if @warnings.nil?
         statuses.latest.failed_but_allowed.any?
diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb
index de410be7f14162a0319612a19e1a4d561eb7f354..b7a9a55dd9e1dcbaa72e6ee9636e45d8d0e9d5ae 100644
--- a/app/services/ci/retry_build_service.rb
+++ b/app/services/ci/retry_build_service.rb
@@ -9,11 +9,7 @@ module Ci
     end
 
     def retry!
-      unless can?(@user, :update_build, @build)
-        raise Gitlab::Access::AccessDeniedError
-      end
-
-      clone_build.tap do |new_build|
+      reprocess!.tap do |new_build|
         new_build.enqueue!
 
         MergeRequests::AddTodoWhenBuildFailsService
@@ -24,9 +20,11 @@ module Ci
       end
     end
 
-    private
+    def reprocess!
+      unless can?(@user, :update_build, @build)
+        raise Gitlab::Access::AccessDeniedError
+      end
 
-    def clone_build
       Ci::Build.create(
         ref: @build.ref,
         tag: @build.tag,
diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb
index f873acc9964c364ba9afebe2b8ce465f322e1bf6..9d8e4d361f449aa2c25fd8d94f9c7e5517ff2725 100644
--- a/app/services/ci/retry_pipeline_service.rb
+++ b/app/services/ci/retry_pipeline_service.rb
@@ -12,10 +12,31 @@ module Ci
         raise Gitlab::Access::AccessDeniedError
       end
 
-      @pipeline.stages.each do |stage|
-        stage.builds.failed_or_canceled.find_each do |build|
-          Ci::Build.retry(build, @user)
+      ##
+      # Reprocess builds in subsequent stages if any
+      #
+      # TODO, refactor.
+      #
+      @pipeline.builds
+        .where('stage_idx > ?', resume_stage.index)
+        .failed_or_canceled.find_each do |build|
+          Ci::RetryBuildService.new(build, @user).reprocess!
         end
+
+      ##
+      # Retry builds in the first unsuccessful stage
+      #
+      resume_stage.builds.failed_or_canceled.find_each do |build|
+        Ci::Build.retry(build, @user)
+      end
+
+    end
+
+    private
+
+    def resume_stage
+      @resume_stage ||= @pipeline.stages.find do |stage|
+        stage.failed? || stage.canceled?
       end
     end
   end
diff --git a/spec/services/ci/retry_pipeline_service_spec.rb b/spec/services/ci/retry_pipeline_service_spec.rb
index 6fc1d8849202cfa3ece1de58ce97a7da6b1bed32..aeb3ee228adbf14bf019b8dda88ac0c7a2121610 100644
--- a/spec/services/ci/retry_pipeline_service_spec.rb
+++ b/spec/services/ci/retry_pipeline_service_spec.rb
@@ -21,6 +21,51 @@ describe Ci::RetryPipelineService, '#execute', :services do
 
         expect(build('rspec 2')).to be_pending
         expect(build('rspec 3')).to be_pending
+        expect(pipeline.reload).to be_running
+      end
+    end
+
+    context 'when there are failed or canceled builds in the first stage' do
+      before do
+        create_build(name: 'rspec 1', status: :failed, stage_num: 0)
+        create_build(name: 'rspec 2', status: :canceled, stage_num: 0)
+        create_build(name: 'rspec 3', status: :skipped, stage_num: 1)
+        create_build(name: 'deploy 1', status: :skipped, stage_num: 2)
+      end
+
+      it 'retries builds failed builds and marks subsequent for processing' do
+        service.execute
+
+        expect(build('rspec 1')).to be_pending
+        expect(build('rspec 2')).to be_pending
+        expect(build('rspec 3')).to be_created
+        expect(build('deploy 1')).to be_created
+        expect(pipeline.reload).to be_running
+      end
+    end
+
+    context 'when there is failed build present which was run on failure' do
+      before do
+        create_build(name: 'rspec 1', status: :failed, stage_num: 0)
+        create_build(name: 'rspec 2', status: :canceled, stage_num: 0)
+        create_build(name: 'rspec 3', status: :skipped, stage_num: 1)
+        create_build(name: 'report 1', status: :failed, stage_num: 2)
+      end
+
+      it 'retries builds failed builds and marks subsequent for processing' do
+        service.execute
+
+        expect(build('rspec 1')).to be_pending
+        expect(build('rspec 2')).to be_pending
+        expect(build('rspec 3')).to be_created
+        expect(build('report 1')).to be_created
+        expect(pipeline.reload).to be_running
+      end
+
+      it 'creates a new job for report job in this case' do
+        service.execute
+
+        expect(statuses.where(name: 'report 1').count).to eq 2
       end
     end
   end
@@ -32,8 +77,12 @@ describe Ci::RetryPipelineService, '#execute', :services do
     end
   end
 
+  def statuses
+    pipeline.reload.statuses
+  end
+
   def build(name)
-    pipeline.statuses.find_by(name: name)
+    statuses.latest.find_by(name: name)
   end
 
   def create_build(name:, status:, stage_num:)