Merge branch 'gitaly-get-blob' into 'master'

Add gitaly_git_blob_raw feature

See merge request !12712

GITALY_SERVER_VERSION

-0.15.0
+0.16.0

Gemfile

@@ -386,7 +386,7 @@ gem 'vmstat', '~> 2.3.0'
 gem 'sys-filesystem', '~> 1.1.6'
 
 # Gitaly GRPC client
-gem 'gitaly', '~> 0.13.0'
+gem 'gitaly', '~> 0.14.0'
 
 gem 'toml-rb', '~> 0.3.15', require: false
 

Gemfile.lock

@@ -278,7 +278,7 @@ GEM
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
     gherkin-ruby (0.3.2)
-    gitaly (0.13.0)
+    gitaly (0.14.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.0)
     github-linguist (4.7.6)
@@ -980,7 +980,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.2.0)
-  gitaly (~> 0.13.0)
+  gitaly (~> 0.14.0)
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-markup (~> 1.5.1)

lib/gitlab/git/blob.rb

@@ -41,10 +41,6 @@ module Gitlab
               commit_id: sha
             )
           when :BLOB
-            # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
-            # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
-            # which is what we use below to keep a consistent behavior.
-            detect = CharlockHolmes::EncodingDetector.new(8000).detect(entry.data)
             new(
               id: entry.oid,
               name: name,
@@ -53,7 +49,7 @@ module Gitlab
               mode: entry.mode.to_s(8),
               path: path,
               commit_id: sha,
-              binary: detect && detect[:type] == :binary
+              binary: binary?(entry.data)
             )
           end
         end
@@ -87,14 +83,28 @@ module Gitlab
         end
 
         def raw(repository, sha)
-          blob = repository.lookup(sha)
+          Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled|
+            if is_enabled
+              Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
+            else
+              blob = repository.lookup(sha)
+    
+              new(
+                id: blob.oid,
+                size: blob.size,
+                data: blob.content(MAX_DATA_DISPLAY_SIZE),
+                binary: blob.binary?
+              )
+            end
+          end
+        end
 
-          new(
-            id: blob.oid,
-            size: blob.size,
-            data: blob.content(MAX_DATA_DISPLAY_SIZE),
-            binary: blob.binary?
-          )
+        def binary?(data)
+          # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
+          # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
+          # which is what we use below to keep a consistent behavior.
+          detect = CharlockHolmes::EncodingDetector.new(8000).detect(data)
+          detect && detect[:type] == :binary
         end
 
         # Recursive search of blob id by path
@@ -165,8 +175,17 @@ module Gitlab
         return if @data == '' # don't mess with submodule blobs
         return @data if @loaded_all_data
 
+        Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled|
+          @data = begin
+            if is_enabled
+              Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: id, limit: -1).data
+            else
+              repository.lookup(id).content
+            end
+          end
+        end
+
         @loaded_all_data = true
-        @data = repository.lookup(id).content
         @loaded_size = @data.bytesize
         @binary = nil
       end

lib/gitlab/gitaly_client/blob.rb

+module Gitlab
+  module GitalyClient
+    class Blob
+      def initialize(repository)
+        @gitaly_repo = repository.gitaly_repository
+      end
+      
+      def get_blob(oid:, limit:)
+        request = Gitaly::GetBlobRequest.new(
+          repository: @gitaly_repo,
+          oid: oid,
+          limit: limit
+        )
+        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request)
+
+        blob = response.first
+        return unless blob.oid.present?
+
+        data = response.reduce(blob.data.dup) { |memo, msg| memo << msg.data.dup }
+
+        Gitlab::Git::Blob.new(
+          id: blob.oid,
+          size: blob.size,
+          data: data,
+          binary: Gitlab::Git::Blob.binary?(data)
+        )
+      end
+    end
+  end
+end

spec/lib/gitlab/git/blob_spec.rb

@@ -111,7 +111,7 @@ describe Gitlab::Git::Blob, seed_helper: true do
     end
   end
 
-  describe '.raw' do
+  shared_examples 'finding blobs by ID' do
     let(:raw_blob) { Gitlab::Git::Blob.raw(repository, SeedRepo::RubyBlob::ID) }
     it { expect(raw_blob.id).to eq(SeedRepo::RubyBlob::ID) }
     it { expect(raw_blob.data[0..10]).to eq("require \'fi") }
@@ -136,6 +136,16 @@ describe Gitlab::Git::Blob, seed_helper: true do
     end
   end
 
+  describe '.raw' do
+    context 'when the blob_raw Gitaly feature is enabled' do
+      it_behaves_like 'finding blobs by ID'
+    end
+
+    context 'when the blob_raw Gitaly feature is disabled', skip_gitaly_mock: true do
+      it_behaves_like 'finding blobs by ID'
+    end
+  end
+
   describe 'encoding' do
     context 'file with russian text' do
       let(:blob) { Gitlab::Git::Blob.find(repository, SeedRepo::Commit::ID, "encoding/russian.rb") }