Move the personal snippet uploads from `system` to `-/system`

Update the markdown unconditionally since the move might have been
done before, but the markdown not updated.

db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class MovePersonalSnippetFilesIntoCorrectFolder < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  NEW_DIRECTORY = File.join('/uploads', '-', 'system', 'personal_snippet')
+  OLD_DIRECTORY = File.join('/uploads', 'system', 'personal_snippet')
+
+  def up
+    return unless file_storage?
+
+    BackgroundMigrationWorker.perform_async('MovePersonalSnippetFiles',
+                                            [OLD_DIRECTORY, NEW_DIRECTORY])
+  end
+
+  def down
+    return unless file_storage?
+
+    BackgroundMigrationWorker.perform_async('MovePersonalSnippetFiles',
+                                            [NEW_DIRECTORY, OLD_DIRECTORY])
+  end
+
+  def file_storage?
+    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
+  end
+end

lib/gitlab/background_migration/move_personal_snippet_files.rb

+module Gitlab
+  module BackgroundMigration
+    class MovePersonalSnippetFiles
+      delegate :select_all, :execute, :quote_string, to: :connection
+
+      def perform(relative_source, relative_destination)
+        @source_relative_location = relative_source
+        @destination_relative_location = relative_destination
+
+        move_personal_snippet_files
+      end
+
+      def move_personal_snippet_files
+        query = "SELECT uploads.path, uploads.model_id FROM uploads "\
+                "INNER JOIN snippets ON snippets.id = uploads.model_id WHERE uploader = 'PersonalFileUploader'"
+        select_all(query).each do |upload|
+          secret = upload['path'].split('/')[0]
+          file_name = upload['path'].split('/')[1]
+
+          move_file(upload['model_id'], secret, file_name)
+          update_markdown(upload['model_id'], secret, file_name)
+        end
+      end
+
+      def move_file(snippet_id, secret, file_name)
+        source_dir = File.join(base_directory, @source_relative_location, snippet_id.to_s, secret)
+        destination_dir = File.join(base_directory, @destination_relative_location, snippet_id.to_s, secret)
+
+        source_file_path = File.join(source_dir, file_name)
+        destination_file_path = File.join(destination_dir, file_name)
+
+        unless File.exist?(source_file_path)
+          say "Source file `#{source_file_path}` doesn't exist. Skipping."
+          return
+        end
+
+        say "Moving file #{source_file_path} -> #{destination_file_path}"
+
+        FileUtils.mkdir_p(destination_dir)
+        FileUtils.move(source_file_path, destination_file_path)
+      end
+
+      def update_markdown(snippet_id, secret, file_name)
+        source_markdown_path = File.join(@source_relative_location, snippet_id.to_s, secret, file_name)
+        destination_markdown_path = File.join(@destination_relative_location, snippet_id.to_s, secret, file_name)
+
+        source_markdown = "](#{source_markdown_path})"
+        destination_markdown = "](#{destination_markdown_path})"
+        quoted_source = quote_string(source_markdown)
+        quoted_destination = quote_string(destination_markdown)
+
+        execute("UPDATE snippets "\
+                "SET description = replace(snippets.description, '#{quoted_source}', '#{quoted_destination}'), description_html = NULL "\
+                "WHERE id = #{snippet_id}")
+
+        query = "SELECT id, note FROM notes WHERE noteable_id = #{snippet_id} "\
+                "AND noteable_type = 'Snippet' AND note IS NOT NULL"
+        select_all(query).each do |note|
+          text = note['note'].gsub(source_markdown, destination_markdown)
+          quoted_text = quote_string(text)
+
+          execute("UPDATE notes SET note = '#{quoted_text}', note_html = NULL WHERE id = #{note['id']}")
+        end
+      end
+
+      def base_directory
+        File.join(Rails.root, 'public')
+      end
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def say(message)
+        Rails.logger.debug(message)
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/move_personal_snippet_files_spec.rb

+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MovePersonalSnippetFiles do
+  let(:test_dir) { File.join(Rails.root, 'tmp', 'tests', 'move_snippet_files_test') }
+  let(:old_uploads_dir) { File.join('uploads', 'system', 'personal_snippet') }
+  let(:new_uploads_dir) { File.join('uploads', '-', 'system', 'personal_snippet') }
+  let(:snippet) do
+    snippet = create(:personal_snippet)
+    create_upload_for_snippet(snippet)
+    snippet.update_attributes!(description: markdown_linking_file(snippet))
+    snippet
+  end
+
+  let(:migration) { described_class.new }
+
+  before do
+    allow(migration).to receive(:base_directory) { test_dir }
+  end
+
+  describe '#perform' do
+    it 'moves the file on the disk'  do
+      expected_path = File.join(test_dir, new_uploads_dir, snippet.id.to_s, "secret#{snippet.id}", 'upload.txt')
+
+      migration.perform(old_uploads_dir, new_uploads_dir)
+
+      expect(File.exist?(expected_path)).to be_truthy
+    end
+
+    it 'updates the markdown of the snippet' do
+      expected_path = File.join(new_uploads_dir, snippet.id.to_s, "secret#{snippet.id}", 'upload.txt')
+      expected_markdown = "[an upload](#{expected_path})"
+
+      migration.perform(old_uploads_dir, new_uploads_dir)
+
+      expect(snippet.reload.description).to eq(expected_markdown)
+    end
+
+    it 'updates the markdown of notes' do
+      expected_path = File.join(new_uploads_dir, snippet.id.to_s, "secret#{snippet.id}", 'upload.txt')
+      expected_markdown = "with [an upload](#{expected_path})"
+
+      note = create(:note_on_personal_snippet, noteable: snippet, note: "with #{markdown_linking_file(snippet)}")
+
+      migration.perform(old_uploads_dir, new_uploads_dir)
+
+      expect(note.reload.note).to eq(expected_markdown)
+    end
+  end
+
+  def create_upload_for_snippet(snippet)
+    snippet_path = path_for_file_in_snippet(snippet)
+    path = File.join(old_uploads_dir, snippet.id.to_s, snippet_path)
+    absolute_path = File.join(test_dir, path)
+
+    FileUtils.mkdir_p(File.dirname(absolute_path))
+    FileUtils.touch(absolute_path)
+
+    create(:upload, model: snippet, path: snippet_path, uploader: PersonalFileUploader)
+  end
+
+  def path_for_file_in_snippet(snippet)
+    secret = "secret#{snippet.id}"
+    filename = 'upload.txt'
+
+    File.join(secret, filename)
+  end
+
+  def markdown_linking_file(snippet)
+    path = File.join(old_uploads_dir, snippet.id.to_s, path_for_file_in_snippet(snippet))
+    "[an upload](#{path})"
+  end
+end

spec/migrations/move_personal_snippets_files_spec.rb

@@ -42,7 +42,7 @@ describe MovePersonalSnippetsFiles do
     describe 'updating the markdown' do
       it 'includes the new path when the file exists' do
         secret = "secret#{snippet.id}"
-        file_location = "/uploads/system/personal_snippet/#{snippet.id}/#{secret}/picture.jpg"
+        file_location = "/uploads/-/system/personal_snippet/#{snippet.id}/#{secret}/picture.jpg"
 
         migration.up
 
@@ -60,7 +60,7 @@ describe MovePersonalSnippetsFiles do
 
       it 'updates the note markdown' do
         secret = "secret#{snippet.id}"
-        file_location = "/uploads/system/personal_snippet/#{snippet.id}/#{secret}/picture.jpg"
+        file_location = "/uploads/-/system/personal_snippet/#{snippet.id}/#{secret}/picture.jpg"
         markdown = markdown_linking_file('picture.jpg', snippet)
         note = create(:note_on_personal_snippet, noteable: snippet, note: "with #{markdown}")
 
@@ -108,7 +108,7 @@ describe MovePersonalSnippetsFiles do
 
       it 'keeps the markdown as is when the file is missing' do
         secret = "secret#{snippet_with_missing_file.id}"
-        file_location = "/uploads/system/personal_snippet/#{snippet_with_missing_file.id}/#{secret}/picture.jpg"
+        file_location = "/uploads/-/system/personal_snippet/#{snippet_with_missing_file.id}/#{secret}/picture.jpg"
 
         migration.down
 
@@ -167,7 +167,7 @@ describe MovePersonalSnippetsFiles do
   def markdown_linking_file(filename, snippet, in_new_path: false)
     markdown =  "![#{filename.split('.')[0]}]"
     markdown += '(/uploads'
-    markdown += '/system' if in_new_path
+    markdown += '/-/system' if in_new_path
     markdown += "/#{model_file_path(filename, snippet)})"
     markdown
   end