Add latest changes from gitlab-org/gitlab@master

app/models/issue.rb

@@ -68,6 +68,12 @@ class Issue < ApplicationRecord
   accepts_nested_attributes_for :sentry_issue
 
   validates :project, presence: true
+  validates :issue_type, presence: true
+
+  enum issue_type: {
+    issue: 0,
+    incident: 1
+  }
 
   alias_attribute :parent_ids, :project_id
   alias_method :issuing_parent, :project

app/models/packages/package_file.rb

@@ -37,11 +37,12 @@ class Packages::PackageFile < ApplicationRecord
 
   update_project_statistics project_statistics_name: :packages_size
 
+  before_save :update_size_from_file
+
   def update_file_metadata
     # The file.object_store is set during `uploader.store!`
     # which happens after object is inserted/updated
     self.update_column(:file_store, file.object_store)
-    self.update_column(:size, file.size) unless file.size == self.size
   end
 
   def download_path
@@ -51,6 +52,12 @@ class Packages::PackageFile < ApplicationRecord
   def local?
     file_store == ::Packages::PackageFileUploader::Store::LOCAL
   end
+
+  private
+
+  def update_size_from_file
+    self.size ||= file.size
+  end
 end
 
 Packages::PackageFile.prepend_if_ee('EE::Packages::PackageFileGeo')

app/services/projects/update_remote_mirror_service.rb

@@ -25,14 +25,8 @@ module Projects
 
     def update_mirror(remote_mirror)
       remote_mirror.update_start!
-
       remote_mirror.ensure_remote!
 
-      # https://gitlab.com/gitlab-org/gitaly/-/issues/2670
-      if Feature.disabled?(:gitaly_ruby_remote_branches_ls_remote, default_enabled: true)
-        repository.fetch_remote(remote_mirror.remote_name, ssh_auth: remote_mirror, no_tags: true)
-      end
-
       response = remote_mirror.update_repository
 
       if response.divergent_refs.any?

changelogs/unreleased/222954-add-issue-type-to-issues.yml

+---
+title: Add issue_type column to issues table
+merge_request: 37402
+author:
+type: added

changelogs/unreleased/225424-fix-package-file-storage-counter.yml

+---
+title: Fix Pypi and Nuget Storage Statistics
+merge_request: 37386
+author:
+type: fixed

db/migrate/20200721034536_add_issue_type_to_issues.rb

+# frozen_string_literal: true
+
+class AddIssueTypeToIssues < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    with_lock_retries do
+      # Set default to issue type
+      add_column :issues, :issue_type, :integer, limit: 2, default: 0, null: false
+    end
+  end
+
+  def down
+    with_lock_retries do
+      remove_column :issues, :issue_type
+    end
+  end
+end

db/migrate/20200721043502_add_issue_type_index_to_issues.rb

+# frozen_string_literal: true
+
+class AddIssueTypeIndexToIssues < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  disable_ddl_transaction!
+
+  INCIDENT_TYPE = 1
+  INDEX_NAME    = 'index_issues_on_incident_issue_type'
+
+  def up
+    add_concurrent_index :issues,
+                         :issue_type, where: "issue_type = #{INCIDENT_TYPE}",
+                         name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index :issues, :issue_type
+  end
+end

db/schema_migrations/20200721034536

+9d30ae6ea32db6cbc5871e214a9c2bf8f1a37fbb586f5e39f6bc2ab58768607b
\ No newline at end of file

db/schema_migrations/20200721043502

+13731676720dd93887dc55374c9052f1087a2e817eb347fd63a19d9398899d75
\ No newline at end of file

db/structure.sql

@@ -12527,6 +12527,7 @@ CREATE TABLE public.issues (
     health_status smallint,
     external_key character varying(255),
     sprint_id bigint,
+    issue_type smallint DEFAULT 0 NOT NULL,
     CONSTRAINT check_fba63f706d CHECK ((lock_version IS NOT NULL))
 );
 
@@ -19696,6 +19697,8 @@ CREATE INDEX index_issues_on_description_trigram ON public.issues USING gin (des
 
 CREATE INDEX index_issues_on_duplicated_to_id ON public.issues USING btree (duplicated_to_id) WHERE (duplicated_to_id IS NOT NULL);
 
+CREATE INDEX index_issues_on_incident_issue_type ON public.issues USING btree (issue_type) WHERE (issue_type = 1);
+
 CREATE INDEX index_issues_on_last_edited_by_id ON public.issues USING btree (last_edited_by_id);
 
 CREATE INDEX index_issues_on_milestone_id ON public.issues USING btree (milestone_id);

lib/gitlab/database/migration_helpers.rb

@@ -815,7 +815,7 @@ module Gitlab
         BEFORE INSERT OR UPDATE
         ON #{table}
         FOR EACH ROW
-        EXECUTE PROCEDURE #{trigger}()
+        EXECUTE FUNCTION #{trigger}()
         EOF
       end
 

lib/gitlab/database/schema_helpers.rb

@@ -25,7 +25,7 @@ module Gitlab
           CREATE TRIGGER #{name}
           #{fires} ON #{table_name}
           FOR EACH ROW
-          EXECUTE PROCEDURE #{function_name}()
+          EXECUTE FUNCTION #{function_name}()
         SQL
       end
 

spec/factories/issues.rb

@@ -7,6 +7,7 @@ FactoryBot.define do
     author { project.creator }
     updated_by { author }
     relative_position { RelativePositioning::START_POSITION }
+    issue_type { :issue }
 
     trait :confidential do
       confidential { true }
@@ -41,5 +42,9 @@ FactoryBot.define do
         issue.update!(labels: evaluator.labels)
       end
     end
+
+    factory :incident do
+      issue_type { :incident }
+    end
   end
 end

spec/lib/gitlab/database/migration_helpers_spec.rb

@@ -712,7 +712,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
             expect(model).to receive(:add_not_null_constraint).with(:users, :new)
             expect(model).to receive(:execute).with("UPDATE \"users\" SET \"new\" = cast_to_jsonb_with_default(\"users\".\"id\") WHERE \"users\".\"id\" >= #{user.id}")
             expect(model).to receive(:execute).with("DROP TRIGGER IF EXISTS #{trigger_name}\nON \"users\"\n")
-            expect(model).to receive(:execute).with("CREATE TRIGGER #{trigger_name}\nBEFORE INSERT OR UPDATE\nON \"users\"\nFOR EACH ROW\nEXECUTE PROCEDURE #{trigger_name}()\n")
+            expect(model).to receive(:execute).with("CREATE TRIGGER #{trigger_name}\nBEFORE INSERT OR UPDATE\nON \"users\"\nFOR EACH ROW\nEXECUTE FUNCTION #{trigger_name}()\n")
             expect(model).to receive(:execute).with("CREATE OR REPLACE FUNCTION #{trigger_name}()\nRETURNS trigger AS\n$BODY$\nBEGIN\n  NEW.\"new\" := NEW.\"id\";\n  RETURN NEW;\nEND;\n$BODY$\nLANGUAGE 'plpgsql'\nVOLATILE\n")
 
             model.rename_column_concurrently(:users, :id, :new, type_cast_function: 'cast_to_jsonb_with_default')

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -32,6 +32,7 @@ Issue:
 - discussion_locked
 - health_status
 - external_key
+- issue_type
 Event:
 - id
 - target_type

spec/models/issue_spec.rb

@@ -58,6 +58,26 @@ RSpec.describe Issue do
     end
   end
 
+  describe 'validations' do
+    subject { issue.valid? }
+
+    describe 'issue_type' do
+      let(:issue) { build(:issue, issue_type: issue_type) }
+
+      context 'when a valid type' do
+        let(:issue_type) { :issue }
+
+        it { is_expected.to eq(true) }
+      end
+
+      context 'empty type' do
+        let(:issue_type) { nil }
+
+        it { is_expected.to eq(false) }
+      end
+    end
+  end
+
   subject { create(:issue) }
 
   describe 'callbacks' do

spec/models/packages/package_file_spec.rb

@@ -32,11 +32,17 @@ RSpec.describe Packages::PackageFile, type: :model do
     end
   end
 
-  it_behaves_like 'UpdateProjectStatistics' do
-    subject { build(:package_file, :jar, size: 42) }
+  context 'updating project statistics' do
+    context 'when the package file has an explicit size' do
+      it_behaves_like 'UpdateProjectStatistics' do
+        subject { build(:package_file, :jar, size: 42) }
+      end
+    end
 
-    before do
-      allow_any_instance_of(Packages::PackageFileUploader).to receive(:size).and_return(42)
+    context 'when the package file does not have a size' do
+      it_behaves_like 'UpdateProjectStatistics' do
+        subject { build(:package_file, size: nil) }
+      end
     end
   end
 

spec/services/projects/update_remote_mirror_service_spec.rb

@@ -10,10 +10,6 @@ RSpec.describe Projects::UpdateRemoteMirrorService do
 
   subject(:service) { described_class.new(project, project.creator) }
 
-  before do
-    stub_feature_flags(gitaly_ruby_remote_branches_ls_remote: false)
-  end
-
   describe '#execute' do
     subject(:execute!) { service.execute(remote_mirror, 0) }
 
@@ -26,17 +22,14 @@ RSpec.describe Projects::UpdateRemoteMirrorService do
     end
 
     it 'ensures the remote exists' do
-      stub_fetch_remote(project, remote_name: remote_name, ssh_auth: remote_mirror)
-
       expect(remote_mirror).to receive(:ensure_remote!)
 
       execute!
     end
 
-    it 'fetches the remote repository' do
-      expect(project.repository)
-        .to receive(:fetch_remote)
-              .with(remote_mirror.remote_name, no_tags: true, ssh_auth: remote_mirror)
+    it 'does not fetch the remote repository' do
+      # See https://gitlab.com/gitlab-org/gitaly/-/issues/2670
+      expect(project.repository).not_to receive(:fetch_remote)
 
       execute!
     end
@@ -48,8 +41,6 @@ RSpec.describe Projects::UpdateRemoteMirrorService do
     end
 
     it 'marks the mirror as successfully finished' do
-      stub_fetch_remote(project, remote_name: remote_name, ssh_auth: remote_mirror)
-
       result = execute!
 
       expect(result[:status]).to eq(:success)
@@ -57,7 +48,7 @@ RSpec.describe Projects::UpdateRemoteMirrorService do
     end
 
     it 'marks the mirror as failed and raises the error when an unexpected error occurs' do
-      allow(project.repository).to receive(:fetch_remote).and_raise('Badly broken')
+      allow(remote_mirror).to receive(:update_repository).and_raise('Badly broken')
 
       expect { execute! }.to raise_error(/Badly broken/)
 
@@ -67,33 +58,30 @@ RSpec.describe Projects::UpdateRemoteMirrorService do
 
     context 'when the update fails because of a `Gitlab::Git::CommandError`' do
       before do
-        allow(project.repository).to receive(:fetch_remote).and_raise(Gitlab::Git::CommandError.new('fetch failed'))
+        allow(remote_mirror).to receive(:update_repository)
+          .and_raise(Gitlab::Git::CommandError.new('update failed'))
       end
 
       it 'wraps `Gitlab::Git::CommandError`s in a service error' do
-        expect(execute!).to eq(status: :error, message: 'fetch failed')
+        expect(execute!).to eq(status: :error, message: 'update failed')
       end
 
       it 'marks the mirror as to be retried' do
         execute!
 
         expect(remote_mirror).to be_to_retry
-        expect(remote_mirror.last_error).to include('fetch failed')
+        expect(remote_mirror.last_error).to include('update failed')
       end
 
       it "marks the mirror as failed after #{described_class::MAX_TRIES} tries" do
         service.execute(remote_mirror, described_class::MAX_TRIES)
 
         expect(remote_mirror).to be_failed
-        expect(remote_mirror.last_error).to include('fetch failed')
+        expect(remote_mirror.last_error).to include('update failed')
       end
     end
 
     context 'when there are divergent refs' do
-      before do
-        stub_fetch_remote(project, remote_name: remote_name, ssh_auth: remote_mirror)
-      end
-
       it 'marks the mirror as failed and sets an error message' do
         response = double(divergent_refs: %w[refs/heads/master refs/heads/develop])
         expect(remote_mirror).to receive(:update_repository).and_return(response)
@@ -106,37 +94,5 @@ RSpec.describe Projects::UpdateRemoteMirrorService do
         expect(remote_mirror.last_error).to include("refs/heads/develop")
       end
     end
-
-    # https://gitlab.com/gitlab-org/gitaly/-/issues/2670
-    context 'when `gitaly_ruby_remote_branches_ls_remote` is enabled' do
-      before do
-        stub_feature_flags(gitaly_ruby_remote_branches_ls_remote: true)
-      end
-
-      it 'does not perform a fetch' do
-        expect(project.repository).not_to receive(:fetch_remote)
-
-        execute!
-      end
-    end
-  end
-
-  def stub_fetch_remote(project, remote_name:, ssh_auth:)
-    allow(project.repository)
-      .to receive(:fetch_remote)
-      .with(remote_name, no_tags: true, ssh_auth: ssh_auth) { fetch_remote(project.repository, remote_name) }
-  end
-
-  def fetch_remote(repository, remote_name)
-    local_branch_names(repository).each do |branch|
-      commit = repository.commit(branch)
-      repository.write_ref("refs/remotes/#{remote_name}/#{branch}", commit.id) if commit
-    end
-  end
-
-  def local_branch_names(repository)
-    branch_names = repository.branches.map(&:name)
-    # we want the protected branch to be pushed first
-    branch_names.unshift(branch_names.delete('master'))
   end
 end

spec/support/helpers/trigger_helpers.rb

@@ -28,8 +28,6 @@ module TriggerHelpers
     expect(timing).to eq(expected_timing.to_s)
     expect(events).to match_array(Array.wrap(expected_events))
 
-    # TODO: Update CREATE TRIGGER syntax to use EXECUTE FUNCTION
-    # https://gitlab.com/gitlab-org/gitlab/-/issues/227089
     expect(definition).to match(%r{execute (?:procedure|function) #{fn_name}()})
   end
 

spec/support/shared_examples/models/update_project_statistics_shared_examples.rb

@@ -17,11 +17,14 @@ RSpec.shared_examples 'UpdateProjectStatistics' do
 
   context 'when creating' do
     it 'updates the project statistics' do
-      delta = read_attribute
+      delta0 = reload_stat
 
-      expect { subject.save! }
-        .to change { reload_stat }
-        .by(delta)
+      subject.save!
+
+      delta1 = reload_stat
+
+      expect(delta1).to eq(delta0 + read_attribute)
+      expect(delta1).to be > delta0
     end
 
     it 'schedules a namespace statistics worker' do
@@ -80,15 +83,14 @@ RSpec.shared_examples 'UpdateProjectStatistics' do
     end
 
     it 'updates the project statistics' do
-      delta = -read_attribute
+      delta0 = reload_stat
 
-      expect(ProjectStatistics)
-        .to receive(:increment_statistic)
-        .and_call_original
+      subject.destroy!
 
-      expect { subject.destroy! }
-        .to change { reload_stat }
-        .by(delta)
+      delta1 = reload_stat
+
+      expect(delta1).to eq(delta0 - read_attribute)
+      expect(delta1).to be < delta0
     end
 
     it 'schedules a namespace statistics worker' do

spec/workers/gitlab/jira_import/import_issue_worker_spec.rb

@@ -24,7 +24,8 @@ RSpec.describe Gitlab::JiraImport::ImportIssueWorker do
       build(:issue, project_id: project.id, title: 'jira issue')
         .as_json.merge(
           'label_ids' => [jira_issue_label_1.id, jira_issue_label_2.id], 'assignee_ids' => assignee_ids
-        ).compact
+        ).except('issue_type')
+        .compact
     end
 
     context 'when any exception raised while inserting to DB' do