From dcd4beb8eb7bb7d0c2f720ef85c3da9f97a3dfe6 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 2 Nov 2016 00:33:35 -0200 Subject: [PATCH] Multi-level container image names backend implementation - Adds Registry events API endpoint - Adds container_images_repository and container_images models - Changes JWT authentication to allow multi-level scopes - Adds services for container image maintenance --- app/models/container_image.rb | 58 +++++++++++++++++++ app/models/container_images_repository.rb | 26 +++++++++ app/models/project.rb | 1 + ...ntainer_registry_authentication_service.rb | 9 ++- .../container_images/create_service.rb | 16 +++++ .../container_images/destroy_service.rb | 11 ++++ .../container_images/push_service.rb | 26 +++++++++ .../create_service.rb | 7 +++ ...3736_create_container_images_repository.rb | 31 ++++++++++ .../20161031013926_create_container_image.rb | 32 ++++++++++ lib/api/api.rb | 1 + lib/api/registry_events.rb | 52 +++++++++++++++++ lib/container_registry/client.rb | 4 ++ lib/container_registry/tag.rb | 8 +-- 14 files changed, 275 insertions(+), 7 deletions(-) create mode 100644 app/models/container_image.rb create mode 100644 app/models/container_images_repository.rb create mode 100644 app/services/container_images_repositories/container_images/create_service.rb create mode 100644 app/services/container_images_repositories/container_images/destroy_service.rb create mode 100644 app/services/container_images_repositories/container_images/push_service.rb create mode 100644 app/services/container_images_repositories/create_service.rb create mode 100644 db/migrate/20161029153736_create_container_images_repository.rb create mode 100644 db/migrate/20161031013926_create_container_image.rb create mode 100644 lib/api/registry_events.rb diff --git a/app/models/container_image.rb b/app/models/container_image.rb new file mode 100644 index 00000000000..dcc4a7af629 --- /dev/null +++ b/app/models/container_image.rb @@ -0,0 +1,58 @@ +class ContainerImage < ActiveRecord::Base + belongs_to :container_images_repository + + delegate :registry, :registry_path_with_namespace, :client, to: :container_images_repository + + validates :manifest, presence: true + + before_validation :update_token, on: :create + def update_token + paths = container_images_repository.allowed_paths << name_with_namespace + token = Auth::ContainerRegistryAuthenticationService.full_access_token(paths) + client.update_token(token) + end + + def path + [registry.path, name_with_namespace].compact.join('/') + end + + def name_with_namespace + [registry_path_with_namespace, name].compact.join('/') + end + + def tag(tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + @manifest ||= client.repository_tags(name_with_namespace) + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def blob(config) + ContainerRegistry::Blob.new(self, config) + end + + def delete_tags + return unless tags + + tags.all?(&:delete) + end + + def self.split_namespace(full_path) + image_name = full_path.split('/').last + namespace = full_path.gsub(/(.*)(#{Regexp.escape('/' + image_name)})/, '\1') + if namespace.count('/') < 1 + namespace, image_name = full_path, "" + end + return namespace, image_name + end +end diff --git a/app/models/container_images_repository.rb b/app/models/container_images_repository.rb new file mode 100644 index 00000000000..99e94d2a6d0 --- /dev/null +++ b/app/models/container_images_repository.rb @@ -0,0 +1,26 @@ +class ContainerImagesRepository < ActiveRecord::Base + + belongs_to :project + + has_many :container_images, dependent: :destroy + + delegate :client, to: :registry + + def registry_path_with_namespace + project.path_with_namespace.downcase + end + + def allowed_paths + @allowed_paths ||= [registry_path_with_namespace] + + container_images.map { |i| i.name_with_namespace } + end + + def registry + @registry ||= begin + token = Auth::ContainerRegistryAuthenticationService.full_access_token(allowed_paths) + url = Gitlab.config.registry.api_url + host_port = Gitlab.config.registry.host_port + ContainerRegistry::Registry.new(url, token: token, path: host_port) + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 411299eef63..703e24eb79a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -157,6 +157,7 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete + has_one :container_images_repository, dependent: :destroy has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 5cb7a86a5ee..6b83b38fa4d 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -16,7 +16,7 @@ module Auth { token: authorized_token(scope).encoded } end - def self.full_access_token(*names) + def self.full_access_token(names) registry = Gitlab.config.registry token = JSONWebToken::RSAToken.new(registry.key) token.issuer = registry.issuer @@ -61,7 +61,12 @@ module Auth end def process_repository_access(type, name, actions) - requested_project = Project.find_by_full_path(name) + # Strips image name due to lack of + # per image authentication. + # Removes only last occurence in light + # of future nested groups + namespace, _ = ContainerImage::split_namespace(name) + requested_project = Project.find_by_full_path(namespace) return unless requested_project actions = actions.select do |action| diff --git a/app/services/container_images_repositories/container_images/create_service.rb b/app/services/container_images_repositories/container_images/create_service.rb new file mode 100644 index 00000000000..0c2c69d5183 --- /dev/null +++ b/app/services/container_images_repositories/container_images/create_service.rb @@ -0,0 +1,16 @@ +module ContainerImagesRepositories + module ContainerImages + class CreateService < BaseService + def execute + @container_image = container_images_repository.container_images.create(params) + @container_image if @container_image.valid? + end + + private + + def container_images_repository + @container_images_repository ||= project.container_images_repository + end + end + end +end diff --git a/app/services/container_images_repositories/container_images/destroy_service.rb b/app/services/container_images_repositories/container_images/destroy_service.rb new file mode 100644 index 00000000000..91b8cfeea47 --- /dev/null +++ b/app/services/container_images_repositories/container_images/destroy_service.rb @@ -0,0 +1,11 @@ +module ContainerImagesRepositories + module ContainerImages + class DestroyService < BaseService + def execute(container_image) + return false unless container_image + + container_image.destroy + end + end + end +end diff --git a/app/services/container_images_repositories/container_images/push_service.rb b/app/services/container_images_repositories/container_images/push_service.rb new file mode 100644 index 00000000000..2731cf1d52e --- /dev/null +++ b/app/services/container_images_repositories/container_images/push_service.rb @@ -0,0 +1,26 @@ +module ContainerImagesRepositories + module ContainerImages + class PushService < BaseService + def execute(container_image_name, event) + find_or_create_container_image(container_image_name).valid? + end + + private + + def find_or_create_container_image(container_image_name) + options = {name: container_image_name} + container_images.find_by(options) || + ::ContainerImagesRepositories::ContainerImages::CreateService.new(project, + current_user, options).execute + end + + def container_images_repository + @container_images_repository ||= project.container_images_repository + end + + def container_images + @container_images ||= container_images_repository.container_images + end + end + end +end diff --git a/app/services/container_images_repositories/create_service.rb b/app/services/container_images_repositories/create_service.rb new file mode 100644 index 00000000000..7e9dd3abe5f --- /dev/null +++ b/app/services/container_images_repositories/create_service.rb @@ -0,0 +1,7 @@ +module ContainerImagesRepositories + class CreateService < BaseService + def execute + project.container_images_repository || ::ContainerImagesRepository.create(project: project) + end + end +end diff --git a/db/migrate/20161029153736_create_container_images_repository.rb b/db/migrate/20161029153736_create_container_images_repository.rb new file mode 100644 index 00000000000..d93180b1674 --- /dev/null +++ b/db/migrate/20161029153736_create_container_images_repository.rb @@ -0,0 +1,31 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateContainerImagesRepository < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :container_images_repositories do |t| + t.integer :project_id, null: false + end + end +end diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb new file mode 100644 index 00000000000..94feae280a6 --- /dev/null +++ b/db/migrate/20161031013926_create_container_image.rb @@ -0,0 +1,32 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateContainerImage < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :container_images do |t| + t.integer :container_images_repository_id + t.string :name + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index a0282ff8deb..ed775f898d2 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -84,6 +84,7 @@ module API mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings + mount ::API::RegistryEvents mount ::API::Pipelines mount ::API::ProjectHooks mount ::API::Projects diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb new file mode 100644 index 00000000000..c0473051424 --- /dev/null +++ b/lib/api/registry_events.rb @@ -0,0 +1,52 @@ +module API + # RegistryEvents API + class RegistryEvents < Grape::API + # before { authenticate! } + + content_type :json, 'application/vnd.docker.distribution.events.v1+json' + + params do + requires :events, type: Array, desc: 'The ID of a project' do + requires :id, type: String, desc: 'The ID of the event' + requires :timestamp, type: String, desc: 'Timestamp of the event' + requires :action, type: String, desc: 'Action performed by event' + requires :target, type: Hash, desc: 'Target of the event' do + optional :mediaType, type: String, desc: 'Media type of the target' + optional :size, type: Integer, desc: 'Size in bytes of the target' + requires :digest, type: String, desc: 'Digest of the target' + requires :repository, type: String, desc: 'Repository of target' + optional :url, type: String, desc: 'Url of the target' + optional :tag, type: String, desc: 'Tag of the target' + end + requires :request, type: Hash, desc: 'Request of the event' do + requires :id, type: String, desc: 'The ID of the request' + optional :addr, type: String, desc: 'IP Address of the request client' + optional :host, type: String, desc: 'Hostname of the registry instance' + requires :method, type: String, desc: 'Request method' + requires :useragent, type: String, desc: 'UserAgent header of the request' + end + requires :actor, type: Hash, desc: 'Actor that initiated the event' do + optional :name, type: String, desc: 'Actor name' + end + requires :source, type: Hash, desc: 'Source of the event' do + optional :addr, type: String, desc: 'Hostname of source registry node' + optional :instanceID, type: String, desc: 'Source registry node instanceID' + end + end + end + resource :registry_events do + post do + params['events'].each do |event| + repository = event['target']['repository'] + + if event['action'] == 'push' and !!event['target']['tag'] + namespace, container_image_name = ContainerImage::split_namespace(repository) + ::ContainerImagesRepositories::ContainerImages::PushService.new( + Project::find_with_namespace(namespace), current_user + ).execute(container_image_name, event) + end + end + end + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 2edddb84fc3..2cbb7bfb67d 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,6 +15,10 @@ module ContainerRegistry @options = options end + def update_token(token) + @options[:token] = token + end + def repository_tags(name) response_body faraday.get("/v2/#{name}/tags/list") end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 59040199920..68dd87c979d 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -22,9 +22,7 @@ module ContainerRegistry end def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_manifest(repository.name, name) + @manifest ||= client.repository_manifest(repository.name_with_namespace, name) end def path @@ -40,7 +38,7 @@ module ContainerRegistry def digest return @digest if defined?(@digest) - @digest = client.repository_tag_digest(repository.name, name) + @digest = client.repository_tag_digest(repository.name_with_namespace, name) end def config_blob @@ -82,7 +80,7 @@ module ContainerRegistry def delete return unless digest - client.delete_repository_tag(repository.name, digest) + client.delete_repository_tag(repository.name_with_namespace, digest) end end end -- GitLab