diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index ed02af05e3d56aeb1cb0ca15d30cf2f7d0d6daf9..18e963891996b4b5b6c4d8dd45409c142b5214eb 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -6,6 +6,7 @@ module Ci include ObjectStorage::BackgroundMove include Presentable include Importable + include Gitlab::Utils::StrongMemoize MissingDependenciesError = Class.new(StandardError) @@ -25,15 +26,17 @@ module Ci has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :metadata, class_name: 'Ci::BuildMetadata' - delegate :timeout, to: :metadata, prefix: true, allow_nil: true - # The "environment" field for builds is a String, and is the unexpanded name + ## + # The "environment" field for builds is a String, and is the unexpanded name! + # def persisted_environment - @persisted_environment ||= Environment.find_by( - name: expanded_environment_name, - project: project - ) + return unless has_environment? + + strong_memoize(:persisted_environment) do + Environment.find_by(name: expanded_environment_name, project: project) + end end serialize :options # rubocop:disable Cop/ActiveRecordSerialize @@ -212,7 +215,11 @@ module Ci end def expanded_environment_name - ExpandVariables.expand(environment, simple_variables) if environment + return unless has_environment? + + strong_memoize(:expanded_environment_name) do + ExpandVariables.expand(environment, simple_variables) + end end def has_environment? @@ -258,31 +265,52 @@ module Ci Gitlab::Utils.slugify(ref.to_s) end - # Variables whose value does not depend on environment - def simple_variables - variables(environment: nil) - end - - # All variables, including those dependent on environment, which could - # contain unexpanded variables. - def variables(environment: persisted_environment) - collection = Gitlab::Ci::Variables::Collection.new.tap do |variables| + ## + # Variables in the environment name scope. + # + def scoped_variables(environment: expanded_environment_name) + Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.concat(predefined_variables) variables.concat(project.predefined_variables) variables.concat(pipeline.predefined_variables) variables.concat(runner.predefined_variables) if runner - variables.concat(project.deployment_variables(environment: environment)) if has_environment? + variables.concat(project.deployment_variables(environment: environment)) if environment variables.concat(yaml_variables) variables.concat(user_variables) - variables.concat(project.group.secret_variables_for(ref, project)) if project.group - variables.concat(secret_variables(environment: environment)) + variables.concat(secret_group_variables) + variables.concat(secret_project_variables(environment: environment)) variables.concat(trigger_request.user_variables) if trigger_request variables.concat(pipeline.variables) variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule - variables.concat(persisted_environment_variables) if environment end + end + + ## + # Variables that do not depend on the environment name. + # + def simple_variables + strong_memoize(:simple_variables) do + scoped_variables(environment: nil).to_runner_variables + end + end - collection.to_runner_variables + ## + # All variables, including persisted environment variables. + # + def variables + Gitlab::Ci::Variables::Collection.new + .concat(persisted_variables) + .concat(scoped_variables) + .concat(persisted_environment_variables) + .to_runner_variables + end + + ## + # Regular Ruby hash of scoped variables, without duplicates that are + # possible to be present in an array of hashes returned from `variables`. + # + def scoped_variables_hash + scoped_variables.to_hash end def features @@ -459,9 +487,14 @@ module Ci end end - def secret_variables(environment: persisted_environment) + def secret_group_variables + return [] unless project.group + + project.group.secret_variables_for(ref, project) + end + + def secret_project_variables(environment: persisted_environment) project.secret_variables_for(ref: ref, environment: environment) - .map(&:to_runner_variable) end def steps @@ -558,6 +591,21 @@ module Ci CI_REGISTRY_USER = 'gitlab-ci-token'.freeze + def persisted_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + return variables unless persisted? + + variables + .append(key: 'CI_JOB_ID', value: id.to_s) + .append(key: 'CI_JOB_TOKEN', value: token, public: false) + .append(key: 'CI_BUILD_ID', value: id.to_s) + .append(key: 'CI_BUILD_TOKEN', value: token, public: false) + .append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) + .append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) + .append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) + end + end + def predefined_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI', value: 'true') @@ -566,16 +614,11 @@ module Ci variables.append(key: 'CI_SERVER_NAME', value: 'GitLab') variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) variables.append(key: 'CI_SERVER_REVISION', value: Gitlab::REVISION) - variables.append(key: 'CI_JOB_ID', value: id.to_s) variables.append(key: 'CI_JOB_NAME', value: name) variables.append(key: 'CI_JOB_STAGE', value: stage) - variables.append(key: 'CI_JOB_TOKEN', value: token, public: false) variables.append(key: 'CI_COMMIT_SHA', value: sha) variables.append(key: 'CI_COMMIT_REF_NAME', value: ref) variables.append(key: 'CI_COMMIT_REF_SLUG', value: ref_slug) - variables.append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) - variables.append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) - variables.append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) variables.append(key: "CI_COMMIT_TAG", value: ref) if tag? variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request variables.append(key: "CI_JOB_MANUAL", value: 'true') if action? @@ -583,23 +626,8 @@ module Ci end end - def persisted_environment_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables unless persisted_environment - - variables.concat(persisted_environment.predefined_variables) - - # Here we're passing unexpanded environment_url for runner to expand, - # and we need to make sure that CI_ENVIRONMENT_NAME and - # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded. - variables.append(key: 'CI_ENVIRONMENT_URL', value: environment_url) if environment_url - end - end - def legacy_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_BUILD_ID', value: id.to_s) - variables.append(key: 'CI_BUILD_TOKEN', value: token, public: false) variables.append(key: 'CI_BUILD_REF', value: sha) variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) variables.append(key: 'CI_BUILD_REF_NAME', value: ref) @@ -612,6 +640,19 @@ module Ci end end + def persisted_environment_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + return variables unless persisted? && persisted_environment.present? + + variables.concat(persisted_environment.predefined_variables) + + # Here we're passing unexpanded environment_url for runner to expand, + # and we need to make sure that CI_ENVIRONMENT_NAME and + # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded. + variables.append(key: 'CI_ENVIRONMENT_URL', value: environment_url) if environment_url + end + end + def environment_url options&.dig(:environment, :url) || persisted_environment&.external_url end diff --git a/changelogs/unreleased/feature-gb-variables-expressions-in-only-except.yml b/changelogs/unreleased/feature-gb-variables-expressions-in-only-except.yml new file mode 100644 index 0000000000000000000000000000000000000000..84977ce11c8c00f5e018e3b43a63936fe04d17cc --- /dev/null +++ b/changelogs/unreleased/feature-gb-variables-expressions-in-only-except.yml @@ -0,0 +1,5 @@ +--- +title: Add support for pipeline variables expressions in only/except +merge_request: 17316 +author: +type: added diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index bd4aeb006bdb2e4a6eb6887859f578a15efa77d0..9f268f47e6fe421532a88d40387e7b156df2e8d8 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -449,6 +449,72 @@ export CI_REGISTRY_USER="gitlab-ci-token" export CI_REGISTRY_PASSWORD="longalfanumstring" ``` +## Variables expressions + +> Variables expressions were added in GitLab 10.7. + +It is possible to use variables expressions with only / except policies in +`.gitlab-ci.yml`. By using this approach you can limit what builds are going to +be created within a pipeline after pushing code to GitLab. + +This is particularly useful in combination with secret variables and triggered +pipeline variables. + +```yaml +deploy: + script: cap staging deploy + environment: staging + only: + variables: + - $RELEASE == "staging" + - $STAGING +``` + +Each provided variables expression is going to be evaluated before creating +a pipeline. + +If any of the conditions in `variables` evaluates to truth when using `only`, +a new job is going to be created. If any of the expressions evaluates to truth +when `except` is being used, a job is not going to be created. + +This follows usual rules for `only` / `except` policies. + +### Supported syntax + +Below you can find currently supported syntax reference: + +1. Equality matching using a string + + Example: `$VARIABLE == "some value"` + + You can use equality operator `==` to compare a variable content to a + string. We support both, double quotes and single quotes to define a string + value, so both `$VARIABLE == "some value"` and `$VARIABLE == 'some value'` + are supported. `"some value" == $VARIABLE` is correct too. + +1. Checking for an undefined value + + It sometimes happens that you want to check whether variable is defined or + not. To do that, you can compare variable to `null` value, like + `$VARIABLE == null`. This expression is going to evaluate to truth if + variable is not set. + +1. Checking for an empty variable + + If you want to check whether a variable is defined, but is empty, you can + simply compare it against an empty string, like `$VAR == ''`. + +1. Comparing two variables + + It is possible to compare two variables. `$VARIABLE_1 == $VARIABLE_2`. + +1. Variable presence check + + If you only want to create a job when there is some variable present, + which means that it is defined and non-empty, you can simply use + variable name as an expression, like `$STAGING`. If `$STAGING` variable + is defined, and is non empty, expression will evaluate to truth. + [ce-13784]: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 "Simple protection of CI secret variables" [eep]: https://about.gitlab.com/products/ "Available only in GitLab Premium" [envs]: ../environments.md diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index c2b06e53c2ffa71d22441457fe2754f4ead4b5f9..9aa443fa69d0b9b8d99bd50e090b10b388b62646 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -315,9 +315,14 @@ policy configuration. GitLab now supports both, simple and complex strategies, so it is possible to use an array and a hash configuration scheme. -Two keys are now available: `refs` and `kubernetes`. Refs strategy equals to -simplified only/except configuration, whereas kubernetes strategy accepts only -`active` keyword. +Three keys are now available: `refs`, `kubernetes` and `variables`. +Refs strategy equals to simplified only/except configuration, whereas +kubernetes strategy accepts only `active` keyword. + +`variables` keyword is used to define variables expressions. In other words +you can use predefined variables / secret variables / project / group or +environment-scoped variables to define an expression GitLab is going to +evaluate in order to decide whether a job should be created or not. See the example below. Job is going to be created only when pipeline has been scheduled or runs for a `master` branch, and only if kubernetes service is @@ -332,6 +337,20 @@ job: kubernetes: active ``` +Example of using variables expressions: + +```yaml +deploy: + only: + refs: + - branches + variables: + - $RELEASE == "staging" + - $STAGING +``` + +Learn more about variables expressions on a separate page. + ## `tags` `tags` is used to select specific Runners from the list of all Runners that are diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index b20d374288fa7a3651defeaba3e29d4c4c9c85b9..782f6c4c0af28741b43fc7e7eef73fb0602c2638 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -9,7 +9,7 @@ module Gitlab end end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) pipeline.has_kubernetes_active? end end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index eadc0948d2f7b2f83190983839a111acb835535f..4aa5dc89f47aaf012a2d89b11bb22b2c5145f76b 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -7,7 +7,7 @@ module Gitlab @patterns = Array(refs) end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) @patterns.any? do |pattern| pattern, path = pattern.split('@', 2) diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index c317291f29df332b45188caaf599fdcdaf573067..f09ba42c074276f9e8ac8e12c5ab3ae5294613b2 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -15,7 +15,7 @@ module Gitlab @spec = spec end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) raise NotImplementedError end end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb new file mode 100644 index 0000000000000000000000000000000000000000..9d2a362b7d4b63cf84078b1929bfca6b1cf46395 --- /dev/null +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Build + module Policy + class Variables < Policy::Specification + def initialize(expressions) + @expressions = Array(expressions) + end + + def satisfied_by?(pipeline, seed) + variables = seed.to_resource.scoped_variables_hash + + statements = @expressions.map do |statement| + ::Gitlab::Ci::Pipeline::Expression::Statement + .new(statement, variables) + end + + statements.any?(&:truthful?) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 0027e9ec8c5d89a2e62c776406a84a26bc141450..09e8e52b60fb4a355323716348779b90c6d96464 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -25,15 +25,31 @@ module Gitlab include Entry::Validatable include Entry::Attributable - attributes :refs, :kubernetes + attributes :refs, :kubernetes, :variables validations do validates :config, presence: true - validates :config, allowed_keys: %i[refs kubernetes] + validates :config, allowed_keys: %i[refs kubernetes variables] + validate :variables_expressions_syntax with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] + validates :variables, array_of_strings: true + end + + def variables_expressions_syntax + return unless variables.is_a?(Array) + + statements = variables.map do |statement| + ::Gitlab::Ci::Pipeline::Expression::Statement.new(statement) + end + + statements.each do |statement| + unless statement.valid? + errors.add(:variables, "Invalid expression syntax") + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index b2b00c8cb4b8ab521db85f7d907cba9042775c68..d299a5677deaf221b71eaa2d3a84270a76643696 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -17,8 +17,6 @@ module Gitlab # Populate pipeline with all stages and builds from pipeline seeds. # pipeline.stage_seeds.each do |stage| - stage.user = current_user - pipeline.stages << stage.to_resource stage.seeds.each do |build| diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 48bde213d44dc42dccd9c374cfc5b78f6bb5ed40..346c92dc51ea3f8bbdb5900dedd7cb0ce305ccdc 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -4,7 +4,7 @@ module Gitlab module Expression module Lexeme class String < Lexeme::Value - PATTERN = /("(?.+?)")|('(?.+?)')/.freeze + PATTERN = /("(?.*?)")|('(?.*?)')/.freeze def initialize(value) @value = value diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index b781c15fd67b73b79ca54465a2ea211b03ae2804..37643c8ef53c03e32a151c1acdf12f82c6a22b93 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -11,7 +11,7 @@ module Gitlab end def evaluate(variables = {}) - HashWithIndifferentAccess.new(variables).fetch(@name, nil) + variables.with_indifferent_access.fetch(@name, nil) end def self.build(string) diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 4f0e101b7301fe9eec4f66d1c68a0561991928ab..09a7c98464b52d49f80588d895b0000054015200 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -14,12 +14,9 @@ module Gitlab %w[variable] ].freeze - def initialize(statement, pipeline) + def initialize(statement, variables = {}) @lexer = Expression::Lexer.new(statement) - - @variables = pipeline.variables.map do |variable| - [variable.key, variable.value] - end + @variables = variables.with_indifferent_access end def parse_tree @@ -35,6 +32,16 @@ module Gitlab def evaluate parse_tree.evaluate(@variables.to_h) end + + def truthful? + evaluate.present? + end + + def valid? + parse_tree.is_a?(Lexeme::Base) + rescue StatementError + false + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 7cd7c864448dcad57b185590144d33f1c589fb63..6980b0b7aff29ae56e510b9917d19443cc52c2d7 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,21 +11,16 @@ module Gitlab @pipeline = pipeline @attributes = attributes - @only = attributes.delete(:only) - @except = attributes.delete(:except) - end - - def user=(current_user) - @attributes.merge!(user: current_user) + @only = Gitlab::Ci::Build::Policy + .fabricate(attributes.delete(:only)) + @except = Gitlab::Ci::Build::Policy + .fabricate(attributes.delete(:except)) end def included? strong_memoize(:inclusion) do - only_specs = Gitlab::Ci::Build::Policy.fabricate(@only) - except_specs = Gitlab::Ci::Build::Policy.fabricate(@except) - - only_specs.all? { |spec| spec.satisfied_by?(@pipeline) } && - except_specs.none? { |spec| spec.satisfied_by?(@pipeline) } + @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } && + @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } end end @@ -33,6 +28,7 @@ module Gitlab @attributes.merge( pipeline: @pipeline, project: @pipeline.project, + user: @pipeline.user, ref: @pipeline.ref, tag: @pipeline.tag, trigger_request: @pipeline.legacy_trigger, diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 1fcbdc1b15ace6479356655523e384c0b0f19856..c101f30d6e81ce252501b8c593fcdd6ae78b32c7 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -17,10 +17,6 @@ module Gitlab end end - def user=(current_user) - @builds.each { |seed| seed.user = current_user } - end - def attributes { name: @attributes.fetch(:name), pipeline: @pipeline, diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index 0deca55fe8fd62cce1524ed07578c852616eedb1..ad30b3f427c21e858d0d119aa24c735d909bb3db 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -30,7 +30,13 @@ module Gitlab end def to_runner_variables - self.map(&:to_hash) + self.map(&:to_runner_variable) + end + + def to_hash + self.to_runner_variables + .map { |env| [env.fetch(:key), env.fetch(:value)] } + .to_h.with_indifferent_access end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 939912981e658c51c39800ab36b709d08216f8c9..23ed71db8b0e1bdc15b9c699466b957f6148af5f 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -17,7 +17,7 @@ module Gitlab end def ==(other) - to_hash == self.class.fabricate(other).to_hash + to_runner_variable == self.class.fabricate(other).to_runner_variable end ## @@ -25,7 +25,7 @@ module Gitlab # don't expose `file` attribute at all (stems from what the runner # expects). # - def to_hash + def to_runner_variable @variable.reject do |hash_key, hash_value| hash_key == :file && hash_value == false end diff --git a/spec/lib/gitlab/ci/build/policy/variables_spec.rb b/spec/lib/gitlab/ci/build/policy/variables_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..2ce858836e3975d1f641f8ccc23f97c6da39511d --- /dev/null +++ b/spec/lib/gitlab/ci/build/policy/variables_spec.rb @@ -0,0 +1,72 @@ +require 'spec_helper' + +describe Gitlab::Ci::Build::Policy::Variables do + set(:project) { create(:project) } + + let(:pipeline) do + build(:ci_empty_pipeline, project: project, ref: 'master', source: :push) + end + + let(:ci_build) do + build(:ci_build, pipeline: pipeline, project: project, ref: 'master') + end + + let(:seed) { double('build seed', to_resource: ci_build) } + + before do + pipeline.variables.build(key: 'CI_PROJECT_NAME', value: '') + end + + describe '#satisfied_by?' do + it 'is satisfied by at least one matching statement' do + policy = described_class.new(['$CI_PROJECT_ID', '$UNDEFINED']) + + expect(policy).to be_satisfied_by(pipeline, seed) + end + + it 'is not satisfied by an overriden empty variable' do + policy = described_class.new(['$CI_PROJECT_NAME']) + + expect(policy).not_to be_satisfied_by(pipeline, seed) + end + + it 'is satisfied by a truthy pipeline expression' do + policy = described_class.new([%($CI_PIPELINE_SOURCE == "push")]) + + expect(policy).to be_satisfied_by(pipeline, seed) + end + + it 'is not satisfied by a falsy pipeline expression' do + policy = described_class.new([%($CI_PIPELINE_SOURCE == "invalid source")]) + + expect(policy).not_to be_satisfied_by(pipeline, seed) + end + + it 'is satisfied by a truthy expression using undefined variable' do + policy = described_class.new(['$UNDEFINED == null']) + + expect(policy).to be_satisfied_by(pipeline, seed) + end + + it 'is not satisfied by a falsy expression using undefined variable' do + policy = described_class.new(['$UNDEFINED']) + + expect(policy).not_to be_satisfied_by(pipeline, seed) + end + + it 'allows to evaluate regular secret variables' do + create(:ci_variable, project: project, key: 'SECRET', value: 'my secret') + + policy = described_class.new(["$SECRET == 'my secret'"]) + + expect(policy).to be_satisfied_by(pipeline, seed) + end + + it 'does not persist neither pipeline nor build' do + described_class.new('$VAR').satisfied_by?(pipeline, seed) + + expect(pipeline).not_to be_persisted + expect(seed.to_resource).not_to be_persisted + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/policy_spec.rb b/spec/lib/gitlab/ci/config/entry/policy_spec.rb index 5e83abf645b87f1532aa0fdc5009876030ae4037..08718c382b9675054163f09bf0d95ffb2258ea7a 100644 --- a/spec/lib/gitlab/ci/config/entry/policy_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/policy_spec.rb @@ -83,6 +83,39 @@ describe Gitlab::Ci::Config::Entry::Policy do end end + context 'when specifying valid variables expressions policy' do + let(:config) { { variables: ['$VAR == null'] } } + + it 'is a correct configuraton' do + expect(entry).to be_valid + expect(entry.value).to eq(config) + end + end + + context 'when specifying variables expressions in invalid format' do + let(:config) { { variables: '$MY_VAR' } } + + it 'reports an error about invalid format' do + expect(entry.errors).to include /should be an array of strings/ + end + end + + context 'when specifying invalid variables expressions statement' do + let(:config) { { variables: ['$MY_VAR =='] } } + + it 'reports an error about invalid statement' do + expect(entry.errors).to include /invalid expression syntax/ + end + end + + context 'when specifying invalid variables expressions token' do + let(:config) { { variables: ['$MY_VAR == 123'] } } + + it 'reports an error about invalid statement' do + expect(entry.errors).to include /invalid expression syntax/ + end + end + context 'when specifying unknown policy' do let(:config) { { refs: ['master'], invalid: :something } } diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index 2258ae83f382925374c43f1d1b4b7d809c57a41c..8312fa47cfa5a23c9b5f4fc4837e60911d3d9ab0 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -6,7 +6,8 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do let(:pipeline) do build(:ci_pipeline_with_one_job, project: project, - ref: 'master') + ref: 'master', + user: user) end let(:command) do @@ -42,6 +43,10 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do expect(pipeline.stages.first.builds).to be_one expect(pipeline.stages.first.builds.first).not_to be_persisted end + + it 'correctly assigns user' do + expect(pipeline.builds).to all(have_attributes(user: user)) + end end context 'when pipeline is empty' do diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/string_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/string_spec.rb index 86234dfb9e53c8261263611af54ac5cfa2814474..1ccb792d1dade3e4b769b1ef99b16b11796eb344 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/lexeme/string_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/string_spec.rb @@ -73,6 +73,22 @@ describe Gitlab::Ci::Pipeline::Expression::Lexeme::String do expect(token).not_to be_nil expect(token.build.evaluate).to eq 'some " string' end + + it 'allows to use an empty string inside single quotes' do + scanner = StringScanner.new(%('')) + + token = described_class.scan(scanner) + + expect(token.build.evaluate).to eq '' + end + + it 'allow to use an empty string inside double quotes' do + scanner = StringScanner.new(%("")) + + token = described_class.scan(scanner) + + expect(token.build.evaluate).to eq '' + end end end diff --git a/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb index 472a58599d8eb7de0858dfe7143897c9de971541..6685bf5385b272c1812497b4393697d91d83c9db 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb @@ -1,14 +1,23 @@ require 'spec_helper' describe Gitlab::Ci::Pipeline::Expression::Statement do - let(:pipeline) { build(:ci_pipeline) } - subject do - described_class.new(text, pipeline) + described_class.new(text, variables) + end + + let(:variables) do + { 'PRESENT_VARIABLE' => 'my variable', + EMPTY_VARIABLE: '' } end - before do - pipeline.variables.build([key: 'VARIABLE', value: 'my variable']) + describe '.new' do + context 'when variables are not provided' do + it 'allows to properly initializes the statement' do + statement = described_class.new('$PRESENT_VARIABLE') + + expect(statement.evaluate).to be_nil + end + end end describe '#parse_tree' do @@ -23,18 +32,26 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do context 'when expression grammar is incorrect' do table = [ - '$VAR "text"', # missing operator - '== "123"', # invalid right side - "'single quotes'", # single quotes string - '$VAR ==', # invalid right side - '12345', # unknown syntax - '' # empty statement + '$VAR "text"', # missing operator + '== "123"', # invalid left side + '"some string"', # only string provided + '$VAR ==', # invalid right side + '12345', # unknown syntax + '' # empty statement ] table.each do |syntax| - it "raises an error when syntax is `#{syntax}`" do - expect { described_class.new(syntax, pipeline).parse_tree } - .to raise_error described_class::StatementError + context "when expression grammar is #{syntax.inspect}" do + let(:text) { syntax } + + it 'aises a statement error exception' do + expect { subject.parse_tree } + .to raise_error described_class::StatementError + end + + it 'is an invalid statement' do + expect(subject).not_to be_valid + end end end end @@ -47,10 +64,14 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do expect(subject.parse_tree) .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Equals end + + it 'is a valid statement' do + expect(subject).to be_valid + end end context 'when using a single token' do - let(:text) { '$VARIABLE' } + let(:text) { '$PRESENT_VARIABLE' } it 'returns a single token instance' do expect(subject.parse_tree) @@ -62,14 +83,17 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do describe '#evaluate' do statements = [ - ['$VARIABLE == "my variable"', true], - ["$VARIABLE == 'my variable'", true], - ['"my variable" == $VARIABLE', true], - ['$VARIABLE == null', false], - ['$VAR == null', true], - ['null == $VAR', true], - ['$VARIABLE', 'my variable'], - ['$VAR', nil] + ['$PRESENT_VARIABLE == "my variable"', true], + ["$PRESENT_VARIABLE == 'my variable'", true], + ['"my variable" == $PRESENT_VARIABLE', true], + ['$PRESENT_VARIABLE == null', false], + ['$EMPTY_VARIABLE == null', false], + ['"" == $EMPTY_VARIABLE', true], + ['$EMPTY_VARIABLE', ''], + ['$UNDEFINED_VARIABLE == null', true], + ['null == $UNDEFINED_VARIABLE', true], + ['$PRESENT_VARIABLE', 'my variable'], + ['$UNDEFINED_VARIABLE', nil] ] statements.each do |expression, value| @@ -82,4 +106,25 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do end end end + + describe '#truthful?' do + statements = [ + ['$PRESENT_VARIABLE == "my variable"', true], + ["$PRESENT_VARIABLE == 'no match'", false], + ['$UNDEFINED_VARIABLE == null', true], + ['$PRESENT_VARIABLE', true], + ['$UNDEFINED_VARIABLE', false], + ['$EMPTY_VARIABLE', false] + ] + + statements.each do |expression, value| + context "when using expression `#{expression}`" do + let(:text) { expression } + + it "returns `#{value.inspect}`" do + expect(subject.truthful?).to eq value + end + end + end + end end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 116573379e0c2b54535547389c9308dc262b2e22..fffa727c2ed80ee1269ff4a0d9c32e0548b12cee 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -21,16 +21,6 @@ describe Gitlab::Ci::Pipeline::Seed::Build do end end - describe '#user=' do - let(:user) { build(:user) } - - it 'assignes user to a build' do - subject.user = user - - expect(subject.attributes).to include(user: user) - end - end - describe '#to_resource' do it 'returns a valid build resource' do expect(subject.to_resource).to be_a(::Ci::Build) diff --git a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb index 8f0bf40d624eb28347a6a18d3734ad06bde8505c..eb1b285c7bd32f44b3e400690f5393329336e1e4 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb @@ -95,16 +95,6 @@ describe Gitlab::Ci::Pipeline::Seed::Stage do end end - describe '#user=' do - let(:user) { build(:user) } - - it 'assignes relevant pipeline attributes' do - subject.user = user - - expect(subject.seeds.map(&:attributes)).to all(include(user: user)) - end - end - describe '#to_resource' do it 'builds a valid stage object with all builds' do subject.to_resource.save! diff --git a/spec/lib/gitlab/ci/variables/collection/item_spec.rb b/spec/lib/gitlab/ci/variables/collection/item_spec.rb index cc1257484d263ef6d7bfc5872b820f541c74fa22..bf9208f1ff4e5d55235198770aabb876c1524fcc 100644 --- a/spec/lib/gitlab/ci/variables/collection/item_spec.rb +++ b/spec/lib/gitlab/ci/variables/collection/item_spec.rb @@ -46,9 +46,13 @@ describe Gitlab::Ci::Variables::Collection::Item do end end - describe '#to_hash' do - it 'returns a hash representation of a collection item' do - expect(described_class.new(**variable).to_hash).to eq variable + describe '#to_runner_variable' do + it 'returns a runner-compatible hash representation' do + runner_variable = described_class + .new(**variable) + .to_runner_variable + + expect(runner_variable).to eq variable end end end diff --git a/spec/lib/gitlab/ci/variables/collection_spec.rb b/spec/lib/gitlab/ci/variables/collection_spec.rb index 90b6e1782420ed4213f26d01f252f574891aa29c..cb2f7718c9cf67770da45a5625d341ddd2497df2 100644 --- a/spec/lib/gitlab/ci/variables/collection_spec.rb +++ b/spec/lib/gitlab/ci/variables/collection_spec.rb @@ -7,7 +7,7 @@ describe Gitlab::Ci::Variables::Collection do collection = described_class.new([variable]) - expect(collection.first.to_hash).to eq variable + expect(collection.first.to_runner_variable).to eq variable end it 'can be initialized without an argument' do @@ -96,4 +96,19 @@ describe Gitlab::Ci::Variables::Collection do .to eq [{ key: 'TEST', value: 1, public: true }] end end + + describe '#to_hash' do + it 'returns regular hash in valid order without duplicates' do + collection = described_class.new + .append(key: 'TEST1', value: 'test-1') + .append(key: 'TEST2', value: 'test-2') + .append(key: 'TEST1', value: 'test-3') + + expect(collection.to_hash).to eq('TEST1' => 'test-3', + 'TEST2' => 'test-2') + + expect(collection.to_hash).to include(TEST1: 'test-3') + expect(collection.to_hash).not_to include(TEST1: 'test-1') + end + end end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index fbc2af29b98150e22070c9586f0b6d6861a23d19..ecb16daec960a19e119ab7d9020c38efad3a41bf 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -1311,6 +1311,14 @@ module Gitlab Gitlab::Ci::YamlProcessor.new(config) end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec dependencies should be an array of strings") end + + it 'returns errors if pipeline variables expression is invalid' do + config = YAML.dump({ rspec: { script: 'test', only: { variables: ['== null'] } } }) + + expect { Gitlab::Ci::YamlProcessor.new(config) } + .to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, + 'jobs:rspec:only variables invalid expression syntax') + end end describe "Validate configuration templates" do diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index f5534d22a5432e7b9057b2218c47920f63a7a49b..a12717835b0d2489f7a1ac8b7cb039f7e10af865 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1463,24 +1463,24 @@ describe Ci::Build do let(:container_registry_enabled) { false } let(:predefined_variables) do [ + { key: 'CI_JOB_ID', value: build.id.to_s, public: true }, + { key: 'CI_JOB_TOKEN', value: build.token, public: false }, + { key: 'CI_BUILD_ID', value: build.id.to_s, public: true }, + { key: 'CI_BUILD_TOKEN', value: build.token, public: false }, + { key: 'CI_REGISTRY_USER', value: 'gitlab-ci-token', public: true }, + { key: 'CI_REGISTRY_PASSWORD', value: build.token, public: false }, + { key: 'CI_REPOSITORY_URL', value: build.repo_url, public: false }, { key: 'CI', value: 'true', public: true }, { key: 'GITLAB_CI', value: 'true', public: true }, { key: 'GITLAB_FEATURES', value: project.namespace.features.join(','), public: true }, { key: 'CI_SERVER_NAME', value: 'GitLab', public: true }, { key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true }, { key: 'CI_SERVER_REVISION', value: Gitlab::REVISION, public: true }, - { key: 'CI_JOB_ID', value: build.id.to_s, public: true }, { key: 'CI_JOB_NAME', value: 'test', public: true }, { key: 'CI_JOB_STAGE', value: 'test', public: true }, - { key: 'CI_JOB_TOKEN', value: build.token, public: false }, { key: 'CI_COMMIT_SHA', value: build.sha, public: true }, { key: 'CI_COMMIT_REF_NAME', value: build.ref, public: true }, { key: 'CI_COMMIT_REF_SLUG', value: build.ref_slug, public: true }, - { key: 'CI_REGISTRY_USER', value: 'gitlab-ci-token', public: true }, - { key: 'CI_REGISTRY_PASSWORD', value: build.token, public: false }, - { key: 'CI_REPOSITORY_URL', value: build.repo_url, public: false }, - { key: 'CI_BUILD_ID', value: build.id.to_s, public: true }, - { key: 'CI_BUILD_TOKEN', value: build.token, public: false }, { key: 'CI_BUILD_REF', value: build.sha, public: true }, { key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true }, { key: 'CI_BUILD_REF_NAME', value: build.ref, public: true }, @@ -1945,6 +1945,7 @@ describe Ci::Build do before do allow(build).to receive(:predefined_variables) { [build_pre_var] } allow(build).to receive(:yaml_variables) { [build_yaml_var] } + allow(build).to receive(:persisted_variables) { [] } allow_any_instance_of(Project) .to receive(:predefined_variables) { [project_pre_var] } @@ -1993,6 +1994,106 @@ describe Ci::Build do end end end + + context 'when build has not been persisted yet' do + let(:build) do + described_class.new( + name: 'rspec', + stage: 'test', + ref: 'feature', + project: project, + pipeline: pipeline + ) + end + + it 'returns static predefined variables' do + expect(build.variables.size).to be >= 28 + expect(build.variables) + .to include(key: 'CI_COMMIT_REF_NAME', value: 'feature', public: true) + expect(build).not_to be_persisted + end + end + end + + describe '#scoped_variables' do + context 'when build has not been persisted yet' do + let(:build) do + described_class.new( + name: 'rspec', + stage: 'test', + ref: 'feature', + project: project, + pipeline: pipeline + ) + end + + it 'does not persist the build' do + expect(build).to be_valid + expect(build).not_to be_persisted + + build.scoped_variables + + expect(build).not_to be_persisted + end + + it 'returns static predefined variables' do + keys = %w[CI_JOB_NAME + CI_COMMIT_SHA + CI_COMMIT_REF_NAME + CI_COMMIT_REF_SLUG + CI_JOB_STAGE] + + variables = build.scoped_variables + + variables.map { |env| env[:key] }.tap do |names| + expect(names).to include(*keys) + end + + expect(variables) + .to include(key: 'CI_COMMIT_REF_NAME', value: 'feature', public: true) + end + + it 'does not return prohibited variables' do + keys = %w[CI_JOB_ID + CI_JOB_TOKEN + CI_BUILD_ID + CI_BUILD_TOKEN + CI_REGISTRY_USER + CI_REGISTRY_PASSWORD + CI_REPOSITORY_URL + CI_ENVIRONMENT_URL] + + build.scoped_variables.map { |env| env[:key] }.tap do |names| + expect(names).not_to include(*keys) + end + end + end + end + + describe '#scoped_variables_hash' do + context 'when overriding secret variables' do + before do + project.variables.create!(key: 'MY_VAR', value: 'my value 1') + pipeline.variables.create!(key: 'MY_VAR', value: 'my value 2') + end + + it 'returns a regular hash created using valid ordering' do + expect(build.scoped_variables_hash).to include('MY_VAR': 'my value 2') + expect(build.scoped_variables_hash).not_to include('MY_VAR': 'my value 1') + end + end + + context 'when overriding user-provided variables' do + before do + pipeline.variables.build(key: 'MY_VAR', value: 'pipeline value') + build.yaml_variables = [{ key: 'MY_VAR', value: 'myvar', public: true }] + end + + it 'returns a hash including variable with higher precedence' do + expect(build.scoped_variables_hash).to include('MY_VAR': 'pipeline value') + expect(build.scoped_variables_hash).not_to include('MY_VAR': 'myvar') + end + end end describe 'state transition: any => [:pending]' do diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 92f00cfbc1916943226fdf3490536cd38b8ce63f..dd94515b0a454d38e81074ec65ecdaa715ebf5bd 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -346,6 +346,20 @@ describe Ci::Pipeline, :mailer do end end end + + context 'when variables policy is specified' do + let(:config) do + { unit: { script: 'minitest', only: { variables: ['$CI_PIPELINE_SOURCE'] } }, + feature: { script: 'spinach', only: { variables: ['$UNDEFINED'] } } } + end + + it 'returns stage seeds only when variables expression is truthy' do + seeds = pipeline.stage_seeds + + expect(seeds.size).to eq 1 + expect(seeds.dig(0, 0, :name)).to eq 'unit' + end + end end describe '#seeds_size' do