Add latest changes from gitlab-org/gitlab@master

changelogs/unreleased/jc-add-seed-to-repository-storages-weighted.yml

+---
+title: Fix existing repository_storages_weighted migrations
+merge_request: 35814
+author:
+type: fixed

changelogs/unreleased/sh-reseed-repository-storages.yml

+---
+title: Fix error 500s creating new projects due to empty weights
+merge_request: 35829
+author:
+type: fixed

db/migrate/20200508203901_add_repository_storages_weighted_to_application_settings.rb

@@ -3,11 +3,37 @@
 class AddRepositoryStoragesWeightedToApplicationSettings < ActiveRecord::Migration[6.0]
   DOWNTIME = false
 
+  class ApplicationSetting < ActiveRecord::Base
+    serialize :repository_storages
+    self.table_name = 'application_settings'
+  end
+
   def up
     add_column :application_settings, :repository_storages_weighted, :jsonb, default: {}, null: false
+
+    seed_repository_storages_weighted
   end
 
   def down
     remove_column :application_settings, :repository_storages_weighted
   end
+
+  private
+
+  def seed_repository_storages_weighted
+    # We need to flush the cache to ensure the newly-added column is loaded
+    ApplicationSetting.reset_column_information
+
+    # There should only be one row here due to
+    # 20200420162730_remove_additional_application_settings_rows.rb
+    ApplicationSetting.all.each do |settings|
+      storages = Gitlab.config.repositories.storages.keys.collect do |storage|
+        weight = settings.repository_storages.include?(storage) ? 100 : 0
+        [storage.to_sym, weight]
+      end
+
+      settings.repository_storages_weighted = Hash[storages]
+      settings.save!
+    end
+  end
 end

db/migrate/20200509203901_reseed_repository_storages_weighted.rb

+# frozen_string_literal: true
+
+class ReseedRepositoryStoragesWeighted < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  class ApplicationSetting < ActiveRecord::Base
+    serialize :repository_storages
+    self.table_name = 'application_settings'
+  end
+
+  def up
+    reseed_repository_storages_weighted
+  end
+
+  private
+
+  def reseed_repository_storages_weighted
+    # We need to flush the cache to ensure the newly-added column is loaded
+    ApplicationSetting.reset_column_information
+
+    # There should only be one row here due to
+    # 20200420162730_remove_additional_application_settings_rows.rb
+    ApplicationSetting.all.each do |settings|
+      # Admins may have already tweaked these values, so don't do anything
+      # if there is data already.
+      next if settings.repository_storages_weighted.present?
+
+      storages = Gitlab.config.repositories.storages.keys.collect do |storage|
+        weight = settings.repository_storages.include?(storage) ? 100 : 0
+        [storage.to_sym, weight]
+      end
+
+      settings.repository_storages_weighted = Hash[storages]
+      settings.save!
+    end
+  end
+end

db/migrate/20200702123805_change_project_id_index_to_be_unique_on_vulnerability_statistics_table.rb

+# frozen_string_literal: true
+
+class ChangeProjectIdIndexToBeUniqueOnVulnerabilityStatisticsTable < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  INDEX_NAME = 'index_vulnerability_statistics_on_unique_project_id'
+
+  def up
+    remove_index :vulnerability_statistics, :project_id # rubocop:disable Migration/RemoveIndex (table is empty)
+    add_index :vulnerability_statistics, :project_id, name: INDEX_NAME, unique: true # rubocop:disable Migration/AddIndex (table is empty)
+  end
+
+  def down
+    remove_index :vulnerability_statistics, name: INDEX_NAME # rubocop:disable Migration/RemoveIndex (table is empty)
+    add_index :vulnerability_statistics, :project_id # rubocop:disable Migration/AddIndex (table is empty)
+  end
+end

db/post_migrate/20200526000407_seed_repository_storages_weighted.rb

 # frozen_string_literal: true
 
 class SeedRepositoryStoragesWeighted < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
   class ApplicationSetting < ActiveRecord::Base
     serialize :repository_storages
     self.table_name = 'application_settings'
   end
 
   def up
+    # We need to flush the cache to ensure the newly-added column is loaded
+    ApplicationSetting.reset_column_information
+
+    # There should only be one row here due to
+    # 20200420162730_remove_additional_application_settings_rows.rb
     ApplicationSetting.all.each do |settings|
       storages = Gitlab.config.repositories.storages.keys.collect do |storage|
         weight = settings.repository_storages.include?(storage) ? 100 : 0
-        [storage, weight]
+        [storage.to_sym, weight]
       end
 
       settings.repository_storages_weighted = Hash[storages]

db/structure.sql

@@ -20357,7 +20357,7 @@ CREATE UNIQUE INDEX index_vulnerability_scanners_on_project_id_and_external_id O
 
 CREATE INDEX index_vulnerability_statistics_on_letter_grade ON public.vulnerability_statistics USING btree (letter_grade);
 
-CREATE INDEX index_vulnerability_statistics_on_project_id ON public.vulnerability_statistics USING btree (project_id);
+CREATE UNIQUE INDEX index_vulnerability_statistics_on_unique_project_id ON public.vulnerability_statistics USING btree (project_id);
 
 CREATE UNIQUE INDEX index_vulnerability_user_mentions_on_note_id ON public.vulnerability_user_mentions USING btree (note_id) WHERE (note_id IS NOT NULL);
 
@@ -23396,6 +23396,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200508091106
 20200508140959
 20200508203901
+20200509203901
 20200511080113
 20200511083541
 20200511092246
@@ -23537,5 +23538,6 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200625190458
 20200626060151
 20200626130220
+20200702123805
 \.
 

doc/administration/gitaly/index.md

@@ -388,9 +388,10 @@ Gitaly makes the following assumptions:
   clients, and that Gitaly server can read and write to `/mnt/gitlab/storage2`.
 - Your `gitaly1.internal` and `gitaly2.internal` Gitaly servers can reach each other.
 
-You can't define Gitaly servers with some as a local directory (with `path`) and some as remote
-server (with `gitaly_address`). However, local and remote Gitaly services can be used together. See
-[mixed configuration](#mixed-configuration) for more information.
+You can't define Gitaly servers with some as a local Gitaly server
+(without `gitaly_address`) and some as remote
+server (with `gitaly_address`) unless you setup with special
+[mixed configuration](#mixed-configuration).
 
 **For Omnibus GitLab**
 
@@ -459,13 +460,14 @@ to all Gitaly servers.
 GitLab can reside on the same server as one of many Gitaly servers, but doesn't support
 configuration that mixes local and remote configuration. The following setup is incorrect, because:
 
-- `path` is invalid for some of the Gitaly servers.
 - All addresses must be reachable from the other Gitaly servers.
+- `storage1` will be assigned a Unix socket for `gitaly_address` which is
+  invalid for some of the Gitaly servers.
 
 ```ruby
 git_data_dirs({
   'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' },
-  'storage1' => { 'path' => '/var/opt/gitlab/git-data' },
+  'storage1' => { 'path' => '/mnt/gitlab/git-data' },
   'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' },
 })
 ```
@@ -477,11 +479,14 @@ example:
 git_data_dirs({
   'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' },
   # Address of the GitLab server that has Gitaly running on it
-  'storage1' => { 'gitaly_address' => 'tcp://gitlab.internal:8075' },
+  'storage1' => { 'gitaly_address' => 'tcp://gitlab.internal:8075', 'path' => '/mnt/gitlab/git-data' },
   'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' },
 })
 ```
 
+`path` can only be included for storage shards on the local Gitaly server.
+If it's excluded, default Git storage directory will be used for that storage shard.
+
 ### Disable Gitaly where not required (optional)
 
 If you are running Gitaly [as a remote service](#run-gitaly-on-its-own-server) you may want to

doc/development/integrations/elasticsearch_for_paid_tiers_on_gitlabcom.md

+# Elasticsearch for paid tiers on GitLab.com
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220246) in GitLab 13.2
+> - It's deployed behind a feature flag, disabled by default.
+> - It's disabled on GitLab.com.
+> - It's not recommended for use in GitLab self-managed instances.
+
+This document describes how to enable Elasticsearch with GitLab for all paid tiers on GitLab.com. Once enabled,
+all paid tiers will have access to the [Advanced Global Search feature](../../integration/elasticsearch.md) on GitLab.com.
+
+## Enable or disable Elasticsearch for all paid tiers on GitLab.com
+
+Since we're still in the process of rolling this out and want to control the timing this is behind a feature flag
+which defaults to off.
+
+To enable it:
+
+```ruby
+# Instance-wide
+Feature.enable(:elasticsearch_index_only_paid_groups)
+```
+
+To disable it:
+
+```ruby
+# Instance-wide
+Feature.disable(:elasticsearch_index_only_paid_groups)
+```

doc/development/ordering_table_columns.md

 # Ordering Table Columns in PostgreSQL
 
+For GitLab we require that columns of new tables are ordered to use the
+least amount of space. An easy way of doing this is to order them based on the
+type size in descending order with variable sizes (`text`, `varchar`, arrays,
+`json`, `jsonb`, and so on) at the end.
+
 Similar to C structures the space of a table is influenced by the order of
 columns. This is because the size of columns is aligned depending on the type of
 the following column. Let's consider an example:
@@ -40,10 +45,9 @@ In these examples, the `id` and `user_id` columns are packed together, which
 means we only need 8 bytes to store _both_ of them. This in turn means each row
 will require 8 bytes less space.
 
-For GitLab we require that columns of new tables are ordered based to use the
-least amount of space. An easy way of doing this is to order them based on the
-type size in descending order with variable sizes (`text`, `varchar`, arrays,
-`json`, `jsonb`, and so on) at the end.
+Note: **NOTE:**
+Since Ruby on Rails 5.1, the default data type for IDs is `bigint`, which uses 8 bytes.
+We are using `integer` in the examples to showcase a more realistic reordering scenario.
 
 ## Type Sizes
 

doc/user/project/repository/reducing_the_repo_size_using_git.md

@@ -152,7 +152,7 @@ However, these refs can be accessed from the Git bundle inside a project export.
    trying to remove internal refs, we will rely on the `commit-map` produced by each run to tell us
    which internal refs to remove.
 
-   NOTE:**Note:**
+   NOTE: **Note:**
    `git filter-repo` creates a new `commit-map` file every run, and overwrite the `commit-map` from
    the previous run. You will need this file from **every** run. Do the next step every time you run
    `git filter-repo`.

spec/migrations/add_repository_storages_weighted_to_application_settings_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20200508203901_add_repository_storages_weighted_to_application_settings.rb')
+
+RSpec.describe AddRepositoryStoragesWeightedToApplicationSettings, :migration do
+  let(:storages) { { "foo" => {}, "baz" => {} } }
+  let(:application_settings) do
+    table(:application_settings).tap do |klass|
+      klass.class_eval do
+        serialize :repository_storages
+      end
+    end
+  end
+
+  before do
+    allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+  end
+
+  let(:application_setting) { application_settings.create! }
+  let(:repository_storages) { ["foo"] }
+
+  it 'populates repository_storages_weighted properly' do
+    application_setting.repository_storages = repository_storages
+    application_setting.save!
+
+    migrate!
+
+    expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq({ "foo" => 100, "baz" => 0 })
+  end
+end

spec/migrations/reseed_repository_storages_weighted_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20200509203901_reseed_repository_storages_weighted.rb')
+
+RSpec.describe ReseedRepositoryStoragesWeighted do
+  let(:storages) { { "foo" => {}, "baz" => {} } }
+  let(:application_settings) do
+    table(:application_settings).tap do |klass|
+      klass.class_eval do
+        serialize :repository_storages
+      end
+    end
+  end
+
+  before do
+    allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
+  end
+
+  let(:repository_storages) { ["foo"] }
+  let!(:application_setting) { application_settings.create!(repository_storages: repository_storages) }
+
+  context 'with empty repository_storages_weighted column' do
+    it 'populates repository_storages_weighted properly' do
+      migrate!
+
+      expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq({ "foo" => 100, "baz" => 0 })
+    end
+  end
+
+  context 'with already-populated repository_storages_weighted column' do
+    let(:existing_weights) { { "foo" => 100, "baz" => 50 } }
+
+    it 'does not change repository_storages_weighted properly' do
+      application_setting.repository_storages_weighted = existing_weights
+      application_setting.save!
+
+      migrate!
+
+      expect(application_settings.find(application_setting.id).repository_storages_weighted).to eq(existing_weights)
+    end
+  end
+end