Merge branch 'feature/gb/variables-expressions-in-only-except' into 'master'

Pipeline variables expression in only/except configuration

Closes #37397

See merge request gitlab-org/gitlab-ce!17316

app/models/ci/build.rb

@@ -6,6 +6,7 @@ module Ci
     include ObjectStorage::BackgroundMove
     include Presentable
     include Importable
+    include Gitlab::Utils::StrongMemoize
 
     MissingDependenciesError = Class.new(StandardError)
 
@@ -25,15 +26,17 @@ module Ci
     has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
 
     has_one :metadata, class_name: 'Ci::BuildMetadata'
-
     delegate :timeout, to: :metadata, prefix: true, allow_nil: true
 
-    # The "environment" field for builds is a String, and is the unexpanded name
+    ##
+    # The "environment" field for builds is a String, and is the unexpanded name!
+    #
     def persisted_environment
-      @persisted_environment ||= Environment.find_by(
-        name: expanded_environment_name,
-        project: project
-      )
+      return unless has_environment?
+
+      strong_memoize(:persisted_environment) do
+        Environment.find_by(name: expanded_environment_name, project: project)
+      end
     end
 
     serialize :options # rubocop:disable Cop/ActiveRecordSerialize
@@ -212,7 +215,11 @@ module Ci
     end
 
     def expanded_environment_name
-      ExpandVariables.expand(environment, simple_variables) if environment
+      return unless has_environment?
+
+      strong_memoize(:expanded_environment_name) do
+        ExpandVariables.expand(environment, simple_variables)
+      end
     end
 
     def has_environment?
@@ -258,31 +265,52 @@ module Ci
       Gitlab::Utils.slugify(ref.to_s)
     end
 
-    # Variables whose value does not depend on environment
-    def simple_variables
-      variables(environment: nil)
-    end
-
-    # All variables, including those dependent on environment, which could
-    # contain unexpanded variables.
-    def variables(environment: persisted_environment)
-      collection = Gitlab::Ci::Variables::Collection.new.tap do |variables|
+    ##
+    # Variables in the environment name scope.
+    #
+    def scoped_variables(environment: expanded_environment_name)
+      Gitlab::Ci::Variables::Collection.new.tap do |variables|
         variables.concat(predefined_variables)
         variables.concat(project.predefined_variables)
         variables.concat(pipeline.predefined_variables)
         variables.concat(runner.predefined_variables) if runner
-        variables.concat(project.deployment_variables(environment: environment)) if has_environment?
+        variables.concat(project.deployment_variables(environment: environment)) if environment
         variables.concat(yaml_variables)
         variables.concat(user_variables)
-        variables.concat(project.group.secret_variables_for(ref, project)) if project.group
-        variables.concat(secret_variables(environment: environment))
+        variables.concat(secret_group_variables)
+        variables.concat(secret_project_variables(environment: environment))
         variables.concat(trigger_request.user_variables) if trigger_request
         variables.concat(pipeline.variables)
         variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule
-        variables.concat(persisted_environment_variables) if environment
       end
+    end
+
+    ##
+    # Variables that do not depend on the environment name.
+    #
+    def simple_variables
+      strong_memoize(:simple_variables) do
+        scoped_variables(environment: nil).to_runner_variables
+      end
+    end
 
-      collection.to_runner_variables
+    ##
+    # All variables, including persisted environment variables.
+    #
+    def variables
+      Gitlab::Ci::Variables::Collection.new
+        .concat(persisted_variables)
+        .concat(scoped_variables)
+        .concat(persisted_environment_variables)
+        .to_runner_variables
+    end
+
+    ##
+    # Regular Ruby hash of scoped variables, without duplicates that are
+    # possible to be present in an array of hashes returned from `variables`.
+    #
+    def scoped_variables_hash
+      scoped_variables.to_hash
     end
 
     def features
@@ -459,9 +487,14 @@ module Ci
       end
     end
 
-    def secret_variables(environment: persisted_environment)
+    def secret_group_variables
+      return [] unless project.group
+
+      project.group.secret_variables_for(ref, project)
+    end
+
+    def secret_project_variables(environment: persisted_environment)
       project.secret_variables_for(ref: ref, environment: environment)
-        .map(&:to_runner_variable)
     end
 
     def steps
@@ -558,6 +591,21 @@ module Ci
 
     CI_REGISTRY_USER = 'gitlab-ci-token'.freeze
 
+    def persisted_variables
+      Gitlab::Ci::Variables::Collection.new.tap do |variables|
+        return variables unless persisted?
+
+        variables
+          .append(key: 'CI_JOB_ID', value: id.to_s)
+          .append(key: 'CI_JOB_TOKEN', value: token, public: false)
+          .append(key: 'CI_BUILD_ID', value: id.to_s)
+          .append(key: 'CI_BUILD_TOKEN', value: token, public: false)
+          .append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER)
+          .append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false)
+          .append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false)
+      end
+    end
+
     def predefined_variables
       Gitlab::Ci::Variables::Collection.new.tap do |variables|
         variables.append(key: 'CI', value: 'true')
@@ -566,16 +614,11 @@ module Ci
         variables.append(key: 'CI_SERVER_NAME', value: 'GitLab')
         variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION)
         variables.append(key: 'CI_SERVER_REVISION', value: Gitlab::REVISION)
-        variables.append(key: 'CI_JOB_ID', value: id.to_s)
         variables.append(key: 'CI_JOB_NAME', value: name)
         variables.append(key: 'CI_JOB_STAGE', value: stage)
-        variables.append(key: 'CI_JOB_TOKEN', value: token, public: false)
         variables.append(key: 'CI_COMMIT_SHA', value: sha)
         variables.append(key: 'CI_COMMIT_REF_NAME', value: ref)
         variables.append(key: 'CI_COMMIT_REF_SLUG', value: ref_slug)
-        variables.append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER)
-        variables.append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false)
-        variables.append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false)
         variables.append(key: "CI_COMMIT_TAG", value: ref) if tag?
         variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request
         variables.append(key: "CI_JOB_MANUAL", value: 'true') if action?
@@ -583,23 +626,8 @@ module Ci
       end
     end
 
-    def persisted_environment_variables
-      Gitlab::Ci::Variables::Collection.new.tap do |variables|
-        return variables unless persisted_environment
-
-        variables.concat(persisted_environment.predefined_variables)
-
-        # Here we're passing unexpanded environment_url for runner to expand,
-        # and we need to make sure that CI_ENVIRONMENT_NAME and
-        # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded.
-        variables.append(key: 'CI_ENVIRONMENT_URL', value: environment_url) if environment_url
-      end
-    end
-
     def legacy_variables
       Gitlab::Ci::Variables::Collection.new.tap do |variables|
-        variables.append(key: 'CI_BUILD_ID', value: id.to_s)
-        variables.append(key: 'CI_BUILD_TOKEN', value: token, public: false)
         variables.append(key: 'CI_BUILD_REF', value: sha)
         variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha)
         variables.append(key: 'CI_BUILD_REF_NAME', value: ref)
@@ -612,6 +640,19 @@ module Ci
       end
     end
 
+    def persisted_environment_variables
+      Gitlab::Ci::Variables::Collection.new.tap do |variables|
+        return variables unless persisted? && persisted_environment.present?
+
+        variables.concat(persisted_environment.predefined_variables)
+
+        # Here we're passing unexpanded environment_url for runner to expand,
+        # and we need to make sure that CI_ENVIRONMENT_NAME and
+        # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded.
+        variables.append(key: 'CI_ENVIRONMENT_URL', value: environment_url) if environment_url
+      end
+    end
+
     def environment_url
       options&.dig(:environment, :url) || persisted_environment&.external_url
     end

changelogs/unreleased/feature-gb-variables-expressions-in-only-except.yml

+---
+title: Add support for pipeline variables expressions in only/except
+merge_request: 17316
+author:
+type: added

doc/ci/variables/README.md

@@ -449,6 +449,72 @@ export CI_REGISTRY_USER="gitlab-ci-token"
 export CI_REGISTRY_PASSWORD="longalfanumstring"
 ```
 
+## Variables expressions
+
+> Variables expressions were added in GitLab 10.7.
+
+It is possible to use variables expressions with only / except policies in
+`.gitlab-ci.yml`. By using this approach you can limit what builds are going to
+be created within a pipeline after pushing code to GitLab.
+
+This is particularly useful in combination with secret variables and triggered
+pipeline variables.
+
+```yaml
+deploy:
+  script: cap staging deploy
+  environment: staging
+  only:
+    variables:
+      - $RELEASE == "staging"
+      - $STAGING
+```
+
+Each provided variables expression is going to be evaluated before creating
+a pipeline.
+
+If any of the conditions in `variables` evaluates to truth when using `only`,
+a new job is going to be created. If any of the expressions evaluates to truth
+when `except` is being used, a job is not going to be created.
+
+This follows usual rules for `only` / `except` policies.
+
+### Supported syntax
+
+Below you can find currently supported syntax reference:
+
+1. Equality matching using a string
+
+    Example: `$VARIABLE == "some value"`
+
+    You can use equality operator `==` to compare a variable content to a
+    string. We support both, double quotes and single quotes to define a string
+    value, so both `$VARIABLE == "some value"` and `$VARIABLE == 'some value'`
+    are supported. `"some value" == $VARIABLE` is correct too.
+
+1. Checking for an undefined value
+
+    It sometimes happens that you want to check whether variable is defined or
+    not. To do that, you can compare variable to `null` value, like
+    `$VARIABLE == null`. This expression is going to evaluate to truth if
+    variable is not set.
+
+1. Checking for an empty variable
+
+    If you want to check whether a variable is defined, but is empty, you can
+    simply compare it against an empty string, like `$VAR == ''`.
+
+1. Comparing two variables
+
+    It is possible to compare two variables. `$VARIABLE_1 == $VARIABLE_2`.
+
+1. Variable presence check
+
+    If you only want to create a job when there is some variable present,
+    which means that it is defined and non-empty, you can simply use
+    variable name as an expression, like `$STAGING`. If `$STAGING` variable
+    is defined, and is non empty, expression will evaluate to truth.
+
 [ce-13784]: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 "Simple protection of CI secret variables"
 [eep]: https://about.gitlab.com/products/ "Available only in GitLab Premium"
 [envs]: ../environments.md

doc/ci/yaml/README.md

@@ -315,9 +315,14 @@ policy configuration.
 GitLab now supports both, simple and complex strategies, so it is possible to
 use an array and a hash configuration scheme.
 
-Two keys are now available: `refs` and `kubernetes`. Refs strategy equals to
-simplified only/except configuration, whereas kubernetes strategy accepts only
-`active` keyword.
+Three keys are now available: `refs`, `kubernetes` and `variables`.
+Refs strategy equals to simplified only/except configuration, whereas
+kubernetes strategy accepts only `active` keyword.
+
+`variables` keyword is used to define variables expressions. In other words
+you can use predefined variables / secret variables / project / group or
+environment-scoped variables to define an expression GitLab is going to
+evaluate in order to decide whether a job should be created or not.
 
 See the example below. Job is going to be created only when pipeline has been
 scheduled or runs for a `master` branch, and only if kubernetes service is
@@ -332,6 +337,20 @@ job:
     kubernetes: active
 ```
 
+Example of using variables expressions:
+
+```yaml
+deploy:
+  only:
+    refs:
+      - branches
+    variables:
+      - $RELEASE == "staging"
+      - $STAGING
+```
+
+Learn more about variables expressions on a separate page.
+
 ## `tags`
 
 `tags` is used to select specific Runners from the list of all Runners that are

lib/gitlab/ci/build/policy/kubernetes.rb

@@ -9,7 +9,7 @@ module Gitlab
             end
           end
 
-          def satisfied_by?(pipeline)
+          def satisfied_by?(pipeline, seed = nil)
             pipeline.has_kubernetes_active?
           end
         end

lib/gitlab/ci/build/policy/refs.rb

@@ -7,7 +7,7 @@ module Gitlab
             @patterns = Array(refs)
           end
 
-          def satisfied_by?(pipeline)
+          def satisfied_by?(pipeline, seed = nil)
             @patterns.any? do |pattern|
               pattern, path = pattern.split('@', 2)
 

lib/gitlab/ci/build/policy/specification.rb

@@ -15,7 +15,7 @@ module Gitlab
             @spec = spec
           end
 
-          def satisfied_by?(pipeline)
+          def satisfied_by?(pipeline, seed = nil)
             raise NotImplementedError
           end
         end

lib/gitlab/ci/build/policy/variables.rb

+module Gitlab
+  module Ci
+    module Build
+      module Policy
+        class Variables < Policy::Specification
+          def initialize(expressions)
+            @expressions = Array(expressions)
+          end
+
+          def satisfied_by?(pipeline, seed)
+            variables = seed.to_resource.scoped_variables_hash
+
+            statements = @expressions.map do |statement|
+              ::Gitlab::Ci::Pipeline::Expression::Statement
+                .new(statement, variables)
+            end
+
+            statements.any?(&:truthful?)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/policy.rb

@@ -25,15 +25,31 @@ module Gitlab
             include Entry::Validatable
             include Entry::Attributable
 
-            attributes :refs, :kubernetes
+            attributes :refs, :kubernetes, :variables
 
             validations do
               validates :config, presence: true
-              validates :config, allowed_keys: %i[refs kubernetes]
+              validates :config, allowed_keys: %i[refs kubernetes variables]
+              validate :variables_expressions_syntax
 
               with_options allow_nil: true do
                 validates :refs, array_of_strings_or_regexps: true
                 validates :kubernetes, allowed_values: %w[active]
+                validates :variables, array_of_strings: true
+              end
+
+              def variables_expressions_syntax
+                return unless variables.is_a?(Array)
+
+                statements = variables.map do |statement|
+                  ::Gitlab::Ci::Pipeline::Expression::Statement.new(statement)
+                end
+
+                statements.each do |statement|
+                  unless statement.valid?
+                    errors.add(:variables, "Invalid expression syntax")
+                  end
+                end
               end
             end
           end

lib/gitlab/ci/pipeline/chain/populate.rb

@@ -17,8 +17,6 @@ module Gitlab
             # Populate pipeline with all stages and builds from pipeline seeds.
             #
             pipeline.stage_seeds.each do |stage|
-              stage.user = current_user
-
               pipeline.stages << stage.to_resource
 
               stage.seeds.each do |build|

lib/gitlab/ci/pipeline/expression/lexeme/string.rb

@@ -4,7 +4,7 @@ module Gitlab
       module Expression
         module Lexeme
           class String < Lexeme::Value
-            PATTERN = /("(?<string>.+?)")|('(?<string>.+?)')/.freeze
+            PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze
 
             def initialize(value)
               @value = value

lib/gitlab/ci/pipeline/expression/lexeme/variable.rb

@@ -11,7 +11,7 @@ module Gitlab
             end
 
             def evaluate(variables = {})
-              HashWithIndifferentAccess.new(variables).fetch(@name, nil)
+              variables.with_indifferent_access.fetch(@name, nil)
             end
 
             def self.build(string)

lib/gitlab/ci/pipeline/expression/statement.rb

@@ -14,12 +14,9 @@ module Gitlab
             %w[variable]
           ].freeze
 
-          def initialize(statement, pipeline)
+          def initialize(statement, variables = {})
             @lexer = Expression::Lexer.new(statement)
-
-            @variables = pipeline.variables.map do |variable|
-              [variable.key, variable.value]
-            end
+            @variables = variables.with_indifferent_access
           end
 
           def parse_tree
@@ -35,6 +32,16 @@ module Gitlab
           def evaluate
             parse_tree.evaluate(@variables.to_h)
           end
+
+          def truthful?
+            evaluate.present?
+          end
+
+          def valid?
+            parse_tree.is_a?(Lexeme::Base)
+          rescue StatementError
+            false
+          end
         end
       end
     end

lib/gitlab/ci/pipeline/seed/build.rb

@@ -11,21 +11,16 @@ module Gitlab
             @pipeline = pipeline
             @attributes = attributes
 
-            @only = attributes.delete(:only)
-            @except = attributes.delete(:except)
-          end
-
-          def user=(current_user)
-            @attributes.merge!(user: current_user)
+            @only = Gitlab::Ci::Build::Policy
+              .fabricate(attributes.delete(:only))
+            @except = Gitlab::Ci::Build::Policy
+              .fabricate(attributes.delete(:except))
           end
 
           def included?
             strong_memoize(:inclusion) do
-              only_specs = Gitlab::Ci::Build::Policy.fabricate(@only)
-              except_specs = Gitlab::Ci::Build::Policy.fabricate(@except)
-
-              only_specs.all? { |spec| spec.satisfied_by?(@pipeline) } &&
-                except_specs.none? { |spec| spec.satisfied_by?(@pipeline) }
+              @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } &&
+                @except.none? { |spec| spec.satisfied_by?(@pipeline, self) }
             end
           end
 
@@ -33,6 +28,7 @@ module Gitlab
             @attributes.merge(
               pipeline: @pipeline,
               project: @pipeline.project,
+              user: @pipeline.user,
               ref: @pipeline.ref,
               tag: @pipeline.tag,
               trigger_request: @pipeline.legacy_trigger,

lib/gitlab/ci/pipeline/seed/stage.rb

@@ -17,10 +17,6 @@ module Gitlab
             end
           end
 
-          def user=(current_user)
-            @builds.each { |seed| seed.user = current_user }
-          end
-
           def attributes
             { name: @attributes.fetch(:name),
               pipeline: @pipeline,

lib/gitlab/ci/variables/collection.rb

@@ -30,7 +30,13 @@ module Gitlab
         end
 
         def to_runner_variables
-          self.map(&:to_hash)
+          self.map(&:to_runner_variable)
+        end
+
+        def to_hash
+          self.to_runner_variables
+            .map { |env| [env.fetch(:key), env.fetch(:value)] }
+            .to_h.with_indifferent_access
         end
       end
     end

lib/gitlab/ci/variables/collection/item.rb

@@ -17,7 +17,7 @@ module Gitlab
           end
 
           def ==(other)
-            to_hash == self.class.fabricate(other).to_hash
+            to_runner_variable == self.class.fabricate(other).to_runner_variable
           end
 
           ##
@@ -25,7 +25,7 @@ module Gitlab
           # don't expose `file` attribute at all (stems from what the runner
           # expects).
           #
-          def to_hash
+          def to_runner_variable
             @variable.reject do |hash_key, hash_value|
               hash_key == :file && hash_value == false
             end

spec/lib/gitlab/ci/build/policy/variables_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Policy::Variables do
+  set(:project) { create(:project) }
+
+  let(:pipeline) do
+    build(:ci_empty_pipeline, project: project, ref: 'master', source: :push)
+  end
+
+  let(:ci_build) do
+    build(:ci_build, pipeline: pipeline, project: project, ref: 'master')
+  end
+
+  let(:seed) { double('build seed', to_resource: ci_build) }
+
+  before do
+    pipeline.variables.build(key: 'CI_PROJECT_NAME', value: '')
+  end
+
+  describe '#satisfied_by?' do
+    it 'is satisfied by at least one matching statement' do
+      policy = described_class.new(['$CI_PROJECT_ID', '$UNDEFINED'])
+
+      expect(policy).to be_satisfied_by(pipeline, seed)
+    end
+
+    it 'is not satisfied by an overriden empty variable' do
+      policy = described_class.new(['$CI_PROJECT_NAME'])
+
+      expect(policy).not_to be_satisfied_by(pipeline, seed)
+    end
+
+    it 'is satisfied by a truthy pipeline expression' do
+      policy = described_class.new([%($CI_PIPELINE_SOURCE == "push")])
+
+      expect(policy).to be_satisfied_by(pipeline, seed)
+    end
+
+    it 'is not satisfied by a falsy pipeline expression' do
+      policy = described_class.new([%($CI_PIPELINE_SOURCE == "invalid source")])
+
+      expect(policy).not_to be_satisfied_by(pipeline, seed)
+    end
+
+    it 'is satisfied by a truthy expression using undefined variable' do
+      policy = described_class.new(['$UNDEFINED == null'])
+
+      expect(policy).to be_satisfied_by(pipeline, seed)
+    end
+
+    it 'is not satisfied by a falsy expression using undefined variable' do
+      policy = described_class.new(['$UNDEFINED'])
+
+      expect(policy).not_to be_satisfied_by(pipeline, seed)
+    end
+
+    it 'allows to evaluate regular secret variables' do
+      create(:ci_variable, project: project, key: 'SECRET', value: 'my secret')
+
+      policy = described_class.new(["$SECRET == 'my secret'"])
+
+      expect(policy).to be_satisfied_by(pipeline, seed)
+    end
+
+    it 'does not persist neither pipeline nor build' do
+      described_class.new('$VAR').satisfied_by?(pipeline, seed)
+
+      expect(pipeline).not_to be_persisted
+      expect(seed.to_resource).not_to be_persisted
+    end
+  end
+end

spec/lib/gitlab/ci/config/entry/policy_spec.rb

@@ -83,6 +83,39 @@ describe Gitlab::Ci::Config::Entry::Policy do
       end
     end
 
+    context 'when specifying valid variables expressions policy' do
+      let(:config) { { variables: ['$VAR == null'] } }
+
+      it 'is a correct configuraton' do
+        expect(entry).to be_valid
+        expect(entry.value).to eq(config)
+      end
+    end
+
+    context 'when specifying variables expressions in invalid format' do
+      let(:config) { { variables: '$MY_VAR' } }
+
+      it 'reports an error about invalid format' do
+        expect(entry.errors).to include /should be an array of strings/
+      end
+    end
+
+    context 'when specifying invalid variables expressions statement' do
+      let(:config) { { variables: ['$MY_VAR =='] } }
+
+      it 'reports an error about invalid statement' do
+        expect(entry.errors).to include /invalid expression syntax/
+      end
+    end
+
+    context 'when specifying invalid variables expressions token' do
+      let(:config) { { variables: ['$MY_VAR == 123'] } }
+
+      it 'reports an error about invalid statement' do
+        expect(entry.errors).to include /invalid expression syntax/
+      end
+    end
+
     context 'when specifying unknown policy' do
       let(:config) { { refs: ['master'], invalid: :something } }
 

spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb

@@ -6,7 +6,8 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
 
   let(:pipeline) do
     build(:ci_pipeline_with_one_job, project: project,
-                                     ref: 'master')
+                                     ref: 'master',
+                                     user: user)
   end
 
   let(:command) do
@@ -42,6 +43,10 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
       expect(pipeline.stages.first.builds).to be_one
       expect(pipeline.stages.first.builds.first).not_to be_persisted
     end
+
+    it 'correctly assigns user' do
+      expect(pipeline.builds).to all(have_attributes(user: user))
+    end
   end
 
   context 'when pipeline is empty' do

spec/lib/gitlab/ci/pipeline/expression/lexeme/string_spec.rb

@@ -73,6 +73,22 @@ describe Gitlab::Ci::Pipeline::Expression::Lexeme::String do
         expect(token).not_to be_nil
         expect(token.build.evaluate).to eq 'some " string'
       end
+
+      it 'allows to use an empty string inside single quotes' do
+        scanner = StringScanner.new(%(''))
+
+        token = described_class.scan(scanner)
+
+        expect(token.build.evaluate).to eq ''
+      end
+
+      it 'allow to use an empty string inside double quotes' do
+        scanner = StringScanner.new(%(""))
+
+        token = described_class.scan(scanner)
+
+        expect(token.build.evaluate).to eq ''
+      end
     end
   end
 

spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb

 require 'spec_helper'
 
 describe Gitlab::Ci::Pipeline::Expression::Statement do
-  let(:pipeline) { build(:ci_pipeline) }
-
   subject do
-    described_class.new(text, pipeline)
+    described_class.new(text, variables)
+  end
+
+  let(:variables) do
+    { 'PRESENT_VARIABLE' => 'my variable',
+      EMPTY_VARIABLE: '' }
   end
 
-  before do
-    pipeline.variables.build([key: 'VARIABLE', value: 'my variable'])
+  describe '.new' do
+    context 'when variables are not provided' do
+      it 'allows to properly initializes the statement' do
+        statement = described_class.new('$PRESENT_VARIABLE')
+
+        expect(statement.evaluate).to be_nil
+      end
+    end
   end
 
   describe '#parse_tree' do
@@ -23,18 +32,26 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do
 
     context 'when expression grammar is incorrect' do
       table = [
-        '$VAR "text"',      # missing operator
-        '== "123"',         # invalid right side
-        "'single quotes'",  # single quotes string
-        '$VAR ==',          # invalid right side
-        '12345',            # unknown syntax
-        ''                  # empty statement
+        '$VAR "text"',   # missing operator
+        '== "123"',      # invalid left side
+        '"some string"', # only string provided
+        '$VAR ==',       # invalid right side
+        '12345',         # unknown syntax
+        ''               # empty statement
       ]
 
       table.each do |syntax|
-        it "raises an error when syntax is `#{syntax}`" do
-          expect { described_class.new(syntax, pipeline).parse_tree }
-            .to raise_error described_class::StatementError
+        context "when expression grammar is #{syntax.inspect}" do
+          let(:text) { syntax }
+
+          it 'aises a statement error exception' do
+            expect { subject.parse_tree }
+              .to raise_error described_class::StatementError
+          end
+
+          it 'is an invalid statement' do
+            expect(subject).not_to be_valid
+          end
         end
       end
     end
@@ -47,10 +64,14 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do
           expect(subject.parse_tree)
             .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Equals
         end
+
+        it 'is a valid statement' do
+          expect(subject).to be_valid
+        end
       end
 
       context 'when using a single token' do
-        let(:text) { '$VARIABLE' }
+        let(:text) { '$PRESENT_VARIABLE' }
 
         it 'returns a single token instance' do
           expect(subject.parse_tree)
@@ -62,14 +83,17 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do
 
   describe '#evaluate' do
     statements = [
-      ['$VARIABLE == "my variable"', true],
-      ["$VARIABLE == 'my variable'", true],
-      ['"my variable" == $VARIABLE', true],
-      ['$VARIABLE == null', false],
-      ['$VAR == null', true],
-      ['null == $VAR', true],
-      ['$VARIABLE', 'my variable'],
-      ['$VAR', nil]
+      ['$PRESENT_VARIABLE == "my variable"', true],
+      ["$PRESENT_VARIABLE == 'my variable'", true],
+      ['"my variable" == $PRESENT_VARIABLE', true],
+      ['$PRESENT_VARIABLE == null', false],
+      ['$EMPTY_VARIABLE == null', false],
+      ['"" == $EMPTY_VARIABLE', true],
+      ['$EMPTY_VARIABLE', ''],
+      ['$UNDEFINED_VARIABLE == null', true],
+      ['null == $UNDEFINED_VARIABLE', true],
+      ['$PRESENT_VARIABLE', 'my variable'],
+      ['$UNDEFINED_VARIABLE', nil]
     ]
 
     statements.each do |expression, value|
@@ -82,4 +106,25 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do
       end
     end
   end
+
+  describe '#truthful?' do
+    statements = [
+      ['$PRESENT_VARIABLE == "my variable"', true],
+      ["$PRESENT_VARIABLE == 'no match'", false],
+      ['$UNDEFINED_VARIABLE == null', true],
+      ['$PRESENT_VARIABLE', true],
+      ['$UNDEFINED_VARIABLE', false],
+      ['$EMPTY_VARIABLE', false]
+    ]
+
+    statements.each do |expression, value|
+      context "when using expression `#{expression}`" do
+        let(:text) { expression }
+
+        it "returns `#{value.inspect}`" do
+          expect(subject.truthful?).to eq value
+        end
+      end
+    end
+  end
 end

spec/lib/gitlab/ci/pipeline/seed/build_spec.rb

@@ -21,16 +21,6 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
     end
   end
 
-  describe '#user=' do
-    let(:user) { build(:user) }
-
-    it 'assignes user to a build' do
-      subject.user = user
-
-      expect(subject.attributes).to include(user: user)
-    end
-  end
-
   describe '#to_resource' do
     it 'returns a valid build resource' do
       expect(subject.to_resource).to be_a(::Ci::Build)

spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb

@@ -95,16 +95,6 @@ describe Gitlab::Ci::Pipeline::Seed::Stage do
     end
   end
 
-  describe '#user=' do
-    let(:user) { build(:user) }
-
-    it 'assignes relevant pipeline attributes' do
-      subject.user = user
-
-      expect(subject.seeds.map(&:attributes)).to all(include(user: user))
-    end
-  end
-
   describe '#to_resource' do
     it 'builds a valid stage object with all builds' do
       subject.to_resource.save!

spec/lib/gitlab/ci/variables/collection/item_spec.rb

@@ -46,9 +46,13 @@ describe Gitlab::Ci::Variables::Collection::Item do
     end
   end
 
-  describe '#to_hash' do
-    it 'returns a hash representation of a collection item' do
-      expect(described_class.new(**variable).to_hash).to eq variable
+  describe '#to_runner_variable' do
+    it 'returns a runner-compatible hash representation' do
+      runner_variable = described_class
+        .new(**variable)
+        .to_runner_variable
+
+      expect(runner_variable).to eq variable
     end
   end
 end

spec/lib/gitlab/ci/variables/collection_spec.rb

@@ -7,7 +7,7 @@ describe Gitlab::Ci::Variables::Collection do
 
       collection = described_class.new([variable])
 
-      expect(collection.first.to_hash).to eq variable
+      expect(collection.first.to_runner_variable).to eq variable
     end
 
     it 'can be initialized without an argument' do
@@ -96,4 +96,19 @@ describe Gitlab::Ci::Variables::Collection do
         .to eq [{ key: 'TEST', value: 1, public: true }]
     end
   end
+
+  describe '#to_hash' do
+    it 'returns regular hash in valid order without duplicates' do
+      collection = described_class.new
+        .append(key: 'TEST1', value: 'test-1')
+        .append(key: 'TEST2', value: 'test-2')
+        .append(key: 'TEST1', value: 'test-3')
+
+      expect(collection.to_hash).to eq('TEST1' => 'test-3',
+                                       'TEST2' => 'test-2')
+
+      expect(collection.to_hash).to include(TEST1: 'test-3')
+      expect(collection.to_hash).not_to include(TEST1: 'test-1')
+    end
+  end
 end

spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -1311,6 +1311,14 @@ module Gitlab
             Gitlab::Ci::YamlProcessor.new(config)
           end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec dependencies should be an array of strings")
         end
+
+        it 'returns errors if pipeline variables expression is invalid' do
+          config = YAML.dump({ rspec: { script: 'test', only: { variables: ['== null'] } } })
+
+          expect { Gitlab::Ci::YamlProcessor.new(config) }
+            .to raise_error(Gitlab::Ci::YamlProcessor::ValidationError,
+                            'jobs:rspec:only variables invalid expression syntax')
+        end
       end
 
       describe "Validate configuration templates" do

spec/models/ci/build_spec.rb

@@ -1463,24 +1463,24 @@ describe Ci::Build do
     let(:container_registry_enabled) { false }
     let(:predefined_variables) do
       [
+        { key: 'CI_JOB_ID', value: build.id.to_s, public: true },
+        { key: 'CI_JOB_TOKEN', value: build.token, public: false },
+        { key: 'CI_BUILD_ID', value: build.id.to_s, public: true },
+        { key: 'CI_BUILD_TOKEN', value: build.token, public: false },
+        { key: 'CI_REGISTRY_USER', value: 'gitlab-ci-token', public: true },
+        { key: 'CI_REGISTRY_PASSWORD', value: build.token, public: false },
+        { key: 'CI_REPOSITORY_URL', value: build.repo_url, public: false },
         { key: 'CI', value: 'true', public: true },
         { key: 'GITLAB_CI', value: 'true', public: true },
         { key: 'GITLAB_FEATURES', value: project.namespace.features.join(','), public: true },
         { key: 'CI_SERVER_NAME', value: 'GitLab', public: true },
         { key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true },
         { key: 'CI_SERVER_REVISION', value: Gitlab::REVISION, public: true },
-        { key: 'CI_JOB_ID', value: build.id.to_s, public: true },
         { key: 'CI_JOB_NAME', value: 'test', public: true },
         { key: 'CI_JOB_STAGE', value: 'test', public: true },
-        { key: 'CI_JOB_TOKEN', value: build.token, public: false },
         { key: 'CI_COMMIT_SHA', value: build.sha, public: true },
         { key: 'CI_COMMIT_REF_NAME', value: build.ref, public: true },
         { key: 'CI_COMMIT_REF_SLUG', value: build.ref_slug, public: true },
-        { key: 'CI_REGISTRY_USER', value: 'gitlab-ci-token', public: true },
-        { key: 'CI_REGISTRY_PASSWORD', value: build.token, public: false },
-        { key: 'CI_REPOSITORY_URL', value: build.repo_url, public: false },
-        { key: 'CI_BUILD_ID', value: build.id.to_s, public: true },
-        { key: 'CI_BUILD_TOKEN', value: build.token, public: false },
         { key: 'CI_BUILD_REF', value: build.sha, public: true },
         { key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true },
         { key: 'CI_BUILD_REF_NAME', value: build.ref, public: true },
@@ -1945,6 +1945,7 @@ describe Ci::Build do
         before do
           allow(build).to receive(:predefined_variables) { [build_pre_var] }
           allow(build).to receive(:yaml_variables) { [build_yaml_var] }
+          allow(build).to receive(:persisted_variables) { [] }
 
           allow_any_instance_of(Project)
             .to receive(:predefined_variables) { [project_pre_var] }
@@ -1993,6 +1994,106 @@ describe Ci::Build do
         end
       end
     end
+
+    context 'when build has not been persisted yet' do
+      let(:build) do
+        described_class.new(
+          name: 'rspec',
+          stage: 'test',
+          ref: 'feature',
+          project: project,
+          pipeline: pipeline
+        )
+      end
+
+      it 'returns static predefined variables' do
+        expect(build.variables.size).to be >= 28
+        expect(build.variables)
+          .to include(key: 'CI_COMMIT_REF_NAME', value: 'feature', public: true)
+        expect(build).not_to be_persisted
+      end
+    end
+  end
+
+  describe '#scoped_variables' do
+    context 'when build has not been persisted yet' do
+      let(:build) do
+        described_class.new(
+          name: 'rspec',
+          stage: 'test',
+          ref: 'feature',
+          project: project,
+          pipeline: pipeline
+        )
+      end
+
+      it 'does not persist the build' do
+        expect(build).to be_valid
+        expect(build).not_to be_persisted
+
+        build.scoped_variables
+
+        expect(build).not_to be_persisted
+      end
+
+      it 'returns static predefined variables' do
+        keys = %w[CI_JOB_NAME
+                  CI_COMMIT_SHA
+                  CI_COMMIT_REF_NAME
+                  CI_COMMIT_REF_SLUG
+                  CI_JOB_STAGE]
+
+        variables = build.scoped_variables
+
+        variables.map { |env| env[:key] }.tap do |names|
+          expect(names).to include(*keys)
+        end
+
+        expect(variables)
+          .to include(key: 'CI_COMMIT_REF_NAME', value: 'feature', public: true)
+      end
+
+      it 'does not return prohibited variables' do
+        keys = %w[CI_JOB_ID
+                  CI_JOB_TOKEN
+                  CI_BUILD_ID
+                  CI_BUILD_TOKEN
+                  CI_REGISTRY_USER
+                  CI_REGISTRY_PASSWORD
+                  CI_REPOSITORY_URL
+                  CI_ENVIRONMENT_URL]
+
+        build.scoped_variables.map { |env| env[:key] }.tap do |names|
+          expect(names).not_to include(*keys)
+        end
+      end
+    end
+  end
+
+  describe '#scoped_variables_hash' do
+    context 'when overriding secret variables' do
+      before do
+        project.variables.create!(key: 'MY_VAR', value: 'my value 1')
+        pipeline.variables.create!(key: 'MY_VAR', value: 'my value 2')
+      end
+
+      it 'returns a regular hash created using valid ordering' do
+        expect(build.scoped_variables_hash).to include('MY_VAR': 'my value 2')
+        expect(build.scoped_variables_hash).not_to include('MY_VAR': 'my value 1')
+      end
+    end
+
+    context 'when overriding user-provided variables' do
+      before do
+        pipeline.variables.build(key: 'MY_VAR', value: 'pipeline value')
+        build.yaml_variables = [{ key: 'MY_VAR', value: 'myvar', public: true }]
+      end
+
+      it 'returns a hash including variable with higher precedence' do
+        expect(build.scoped_variables_hash).to include('MY_VAR': 'pipeline value')
+        expect(build.scoped_variables_hash).not_to include('MY_VAR': 'myvar')
+      end
+    end
   end
 
   describe 'state transition: any => [:pending]' do

spec/models/ci/pipeline_spec.rb

@@ -346,6 +346,20 @@ describe Ci::Pipeline, :mailer do
           end
         end
       end
+
+      context 'when variables policy is specified' do
+        let(:config) do
+          { unit: { script: 'minitest', only: { variables: ['$CI_PIPELINE_SOURCE'] } },
+            feature: { script: 'spinach', only: { variables: ['$UNDEFINED'] } } }
+        end
+
+        it 'returns stage seeds only when variables expression is truthy' do
+          seeds = pipeline.stage_seeds
+
+          expect(seeds.size).to eq 1
+          expect(seeds.dig(0, 0, :name)).to eq 'unit'
+        end
+      end
     end
 
     describe '#seeds_size' do