Allow to expire build artifacts

Gemfile

@@ -210,6 +210,9 @@ gem 'mousetrap-rails', '~> 1.4.6'
 # Detect and convert string character encoding
 gem 'charlock_holmes', '~> 0.7.3'
 
+# Parse duration
+gem 'chronic_duration', '~> 0.10.6'
+
 gem "sass-rails", '~> 5.0.0'
 gem "coffee-rails", '~> 4.1.0'
 gem "uglifier", '~> 2.7.2'

Gemfile.lock

@@ -133,6 +133,8 @@ GEM
       mime-types (>= 1.16)
     cause (0.1)
     charlock_holmes (0.7.3)
+    chronic_duration (0.10.6)
+      numerizer (~> 0.1.1)
     chunky_png (1.3.5)
     cliver (0.3.2)
     coderay (1.1.0)
@@ -424,6 +426,7 @@ GEM
     nokogiri (1.6.8)
       mini_portile2 (~> 2.1.0)
       pkg-config (~> 1.1.7)
+    numerizer (0.1.1)
     oauth (0.4.7)
     oauth2 (1.0.0)
       faraday (>= 0.8, < 0.10)
@@ -857,6 +860,7 @@ DEPENDENCIES
   capybara-screenshot (~> 1.0.0)
   carrierwave (~> 0.10.0)
   charlock_holmes (~> 0.7.3)
+  chronic_duration (~> 0.10.6)
   coffee-rails (~> 4.1.0)
   connection_pool (~> 2.0)
   coveralls (~> 0.8.2)

app/controllers/projects/builds_controller.rb

@@ -78,6 +78,12 @@ class Projects::BuildsController < Projects::ApplicationController
     end
   end
 
+  def keep_artifacts
+    @build.keep_artifacts
+    redirect_to namespace_project_build_path(project.namespace, project, @build),
+                notice: "Artifacts will not be removed!"
+  end
+
   private
 
   def build

app/models/ci/build.rb

@@ -11,6 +11,8 @@ module Ci
 
     scope :unstarted, ->() { where(runner_id: nil) }
     scope :ignore_failures, ->() { where(allow_failure: false) }
+    scope :with_artifacts, ->() { where.not(artifacts_file: nil) }
+    scope :with_artifacts_expired, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) }
 
     mount_uploader :artifacts_file, ArtifactUploader
     mount_uploader :artifacts_metadata, ArtifactUploader
@@ -328,11 +330,15 @@ module Ci
       Gitlab::Ci::Build::Artifacts::Metadata.new(artifacts_metadata.path, path, **options).to_entry
     end
 
+    def erase_artifacts!
+      remove_artifacts_file!
+      remove_artifacts_metadata!
+    end
+
     def erase(opts = {})
       return false unless erasable?
 
-      remove_artifacts_file!
-      remove_artifacts_metadata!
+      erase_artifacts!
       erase_trace!
       update_erased!(opts[:erased_by])
     end
@@ -345,6 +351,14 @@ module Ci
       !self.erased_at.nil?
     end
 
+    def artifacts_expired?
+      self.artifacts_expire_at < Time.now && !artifacts?
+    end
+
+    def keep_artifacts
+      self.update(artifacts_expire_at: nil)
+    end
+
     private
 
     def erase_trace!

app/views/projects/builds/_sidebar.html.haml

@@ -44,6 +44,15 @@
       %p.build-detail-row
         %span.build-light-text Erased:
         #{time_ago_with_tooltip(@build.erased_at)}
+    - elsif @build.artifacts_expired?
+      %p.build-detail-row.artifacts-expired.alert.alert-warning
+        The artifacts were removed #{time_ago_with_tooltip(@build.artifacts_expire_at)}
+    - elsif @build.artifacts_expire_at
+      %p.build-detail-row.artifacts-expired.alert.alert-info
+        The artifacts will be removed at #{time_ago_with_tooltip(@build.artifacts_expire_at)}
+        .pull-right
+          = link_to keep_artifacts_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-primary' do
+            Keep
     %p.build-detail-row
       %span.build-light-text Runner:
       - if @build.runner && current_user && current_user.admin

app/workers/expire_build_artifacts.rb

+class ExpireBuildArtifacts
+  include Sidekiq::Worker
+
+  def perform
+    Rails.logger.info 'Cleaning old build artifacts'
+
+    builds = Ci::Build.with_artifacts_expired
+    builds.find_each(batch_size: 50).each do |build|
+      build.erase_artifacts!
+    end
+  end
+end

config/gitlab.yml.example

@@ -164,6 +164,9 @@ production: &base
     # Flag stuck CI builds as failed
     stuck_ci_builds_worker:
       cron: "0 0 * * *"
+    # Remove old artifacts
+    expire_build_artifacts:
+      cron: "50 * * * *"
     # Periodically run 'git fsck' on all repositories. If started more than
     # once per hour you will have concurrent 'git fsck' jobs.
     repository_check_worker:

config/initializers/1_settings.rb

@@ -279,6 +279,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *'
 Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker'
+Settings.cron_jobs['expire_build_artifacts'] ||= Settingslogic.new({})
+Settings.cron_jobs['expire_build_artifacts']['cron'] ||= '0 0 * * *'
+Settings.cron_jobs['expire_build_artifacts']['job_class'] = 'ExpireBuildArtifacts'
 Settings.cron_jobs['repository_check_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['repository_check_worker']['cron'] ||= '20 * * * *'
 Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheck::BatchWorker'

config/routes.rb

@@ -714,6 +714,7 @@ Rails.application.routes.draw do
             post :cancel
             post :retry
             post :erase
+            post :keep_artifacts
             get :trace
             get :raw
           end

db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb

+class AddArtifactsExpireDateToCiBuilds < ActiveRecord::Migration
+  def change
+    add_column :ci_builds, :artifacts_expire_at, :timestamp
+  end
+end

lib/ci/api/builds.rb

@@ -114,6 +114,7 @@ module Ci
         #   id (required) - The ID of a build
         #   token (required) - The build authorization token
         #   file (required) - Artifacts file
+        #   expire_in (optional) - Specify when artifacts should expire (ex. 7d)
         # Parameters (accelerated by GitLab Workhorse):
         #   file.path - path to locally stored body (generated by Workhorse)
         #   file.name - real filename as send in Content-Disposition
@@ -145,6 +146,7 @@ module Ci
 
           build.artifacts_file = artifacts
           build.artifacts_metadata = metadata
+          build.artifacts_expire_at = Time.now + ChronicDuration.parse(params['expire_in'])
 
           if build.save
             present(build, with: Entities::BuildDetails)

lib/ci/gitlab_ci_yaml_processor.rb

@@ -9,7 +9,7 @@ module Ci
                         :allow_failure, :type, :stage, :when, :artifacts, :cache,
                         :dependencies, :before_script, :after_script, :variables]
     ALLOWED_CACHE_KEYS = [:key, :untracked, :paths]
-    ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when]
+    ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in]
 
     attr_reader :before_script, :after_script, :image, :services, :path, :cache