From 49d6ed9ded7d9fad45974eec37c17574d396645b Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Fri, 12 Jan 2018 16:55:48 +0100 Subject: [PATCH] Migrate importing repository to Gitaly Closes gitaly#907 --- GITALY_SERVER_VERSION | 2 +- Gemfile | 2 +- Gemfile.lock | 4 +- lib/gitlab/git/gitlab_projects.rb | 64 ++++++++++++------- lib/gitlab/gitaly_client/commit_service.rb | 2 +- .../gitaly_client/repository_service.rb | 15 +++++ spec/lib/gitlab/git/gitlab_projects_spec.rb | 56 ++++++++++------ 7 files changed, 98 insertions(+), 47 deletions(-) diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 328185caaeb..106d4ac0005 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0.67.0 +0.69.0 diff --git a/Gemfile b/Gemfile index 5c455ab15e3..36da419ba92 100644 --- a/Gemfile +++ b/Gemfile @@ -403,7 +403,7 @@ group :ed25519 do end # Gitaly GRPC client -gem 'gitaly-proto', '~> 0.69.0', require: 'gitaly' +gem 'gitaly-proto', '~> 0.73.0', require: 'gitaly' gem 'toml-rb', '~> 0.3.15', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 8e31ac1f993..b83a3f0f7a4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -284,7 +284,7 @@ GEM po_to_json (>= 1.0.0) rails (>= 3.2.0) gherkin-ruby (0.3.2) - gitaly-proto (0.69.0) + gitaly-proto (0.73.0) google-protobuf (~> 3.1) grpc (~> 1.0) github-linguist (4.7.6) @@ -1054,7 +1054,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.2.0) - gitaly-proto (~> 0.69.0) + gitaly-proto (~> 0.73.0) github-linguist (~> 4.7.0) gitlab-flowdock-git-hook (~> 1.0.1) gitlab-markup (~> 1.6.2) diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index 976fa1ddfe6..e5a747cb987 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -44,29 +44,13 @@ module Gitlab # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false + Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled| + if is_enabled + gitaly_import_repository(source) + else + git_import_repository(source, timeout) + end end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true end def fork_repository(new_shard_path, new_repository_relative_path) @@ -231,6 +215,42 @@ module Gitlab raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") end + def git_import_repository(source, timeout) + # Skip import if repo already exists + return false if File.exist?(repository_absolute_path) + + masked_source = mask_password_in_url(source) + + logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." + cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) + + success = run_with_timeout(cmd, timeout, nil) + + unless success + logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") + FileUtils.rm_rf(repository_absolute_path) + return false + end + + Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) + + # The project was imported successfully. + # Remove the origin URL since it may contain password. + remove_origin_in_repo + + true + end + + def gitaly_import_repository(source) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) + true + rescue GRPC::BadStatus => e + @output << e.message + false + end + def git_fork_repository(new_shard_path, new_repository_relative_path) from_path = repository_absolute_path to_path = File.join(new_shard_path, new_repository_relative_path) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index fed05bb6c64..71b212023d6 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -177,7 +177,7 @@ module Gitlab response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) - rescue GRPC::Unknown # If no repository is found, happens mainly during testing + rescue GRPC::NotFound # If no repository is found, happens mainly during testing [] end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 72ee92e78dc..12016aee2a6 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -100,6 +100,21 @@ module Gitlab ) end + def import_repository(source) + request = Gitaly::CreateRepositoryFromURLRequest.new( + repository: @gitaly_repo, + url: source + ) + + GitalyClient.call( + @storage, + :repository_service, + :create_repository_from_url, + request, + timeout: GitalyClient.default_timeout + ) + end + def rebase_in_progress?(rebase_id) request = Gitaly::IsRebaseInProgressRequest.new( repository: @gitaly_repo, diff --git a/spec/lib/gitlab/git/gitlab_projects_spec.rb b/spec/lib/gitlab/git/gitlab_projects_spec.rb index beef843537d..78e4fbca28e 100644 --- a/spec/lib/gitlab/git/gitlab_projects_spec.rb +++ b/spec/lib/gitlab/git/gitlab_projects_spec.rb @@ -158,39 +158,55 @@ describe Gitlab::Git::GitlabProjects do subject { gl_projects.import_project(import_url, timeout) } - context 'success import' do - it 'imports a repo' do - expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy + shared_examples 'importing repository' do + context 'success import' do + it 'imports a repo' do + expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy - message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>." - expect(logger).to receive(:info).with(message) + is_expected.to be_truthy - is_expected.to be_truthy + expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy + end + end - expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy + context 'already exists' do + it "doesn't import" do + FileUtils.mkdir_p(tmp_repo_path) + + is_expected.to be_falsy + end end end - context 'already exists' do - it "doesn't import" do - FileUtils.mkdir_p(tmp_repo_path) + context 'when Gitaly import_repository feature is enabled' do + it_behaves_like 'importing repository' + end + + context 'when Gitaly import_repository feature is disabled', :disable_gitaly do + describe 'logging' do + it 'imports a repo' do + message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>." + expect(logger).to receive(:info).with(message) - is_expected.to be_falsy + subject + end end - end - context 'timeout' do - it 'does not import a repo' do - stub_spawn_timeout(cmd, timeout, nil) + context 'timeout' do + it 'does not import a repo' do + stub_spawn_timeout(cmd, timeout, nil) - message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed." - expect(logger).to receive(:error).with(message) + message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed." + expect(logger).to receive(:error).with(message) - is_expected.to be_falsy + is_expected.to be_falsy - expect(gl_projects.output).to eq("Timed out\n") - expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy + expect(gl_projects.output).to eq("Timed out\n") + expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy + end end + + it_behaves_like 'importing repository' end end -- GitLab