From 3f4a7218a4a4923a0e7ce1b2eb0d2888ce30da58 Mon Sep 17 00:00:00 2001 From: Dino Maric Date: Mon, 31 Jul 2017 17:09:12 +0200 Subject: [PATCH] Azure Storage support (#36) * Microsoft Azure storage support * Add support for Microsoft Azure Storage * Comply with the new headers implementation --- Gemfile | 3 + Gemfile.lock | 18 +++ config/storage_services.yml | 7 ++ lib/active_storage/service/azure_service.rb | 115 ++++++++++++++++++ .../direct_uploads_controller_test.rb | 34 ++++++ test/service/azure_service_test.rb | 14 +++ test/service/configurations-example.yml | 7 ++ 7 files changed, 198 insertions(+) create mode 100644 lib/active_storage/service/azure_service.rb create mode 100644 test/service/azure_service_test.rb diff --git a/Gemfile b/Gemfile index 55b0deec27..b8b6e80176 100644 --- a/Gemfile +++ b/Gemfile @@ -14,6 +14,9 @@ gem "httparty" gem "aws-sdk", "~> 2", require: false gem "google-cloud-storage", "~> 1.3", require: false +# Contains fix to be able to test using StringIO +gem 'azure-core', git: "https://github.com/dixpac/azure-ruby-asm-core.git" +gem 'azure-storage', require: false gem "mini_magick" diff --git a/Gemfile.lock b/Gemfile.lock index 4319b1e22d..7388096778 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,12 @@ +GIT + remote: https://github.com/dixpac/azure-ruby-asm-core.git + revision: 4403389747f44a94b73e7a7522d1ea11f8b1a266 + specs: + azure-core (0.1.8) + faraday (~> 0.9) + faraday_middleware (~> 0.10) + nokogiri (~> 1.7) + GIT remote: https://github.com/rails/rails.git revision: 127b475dc251a06942fe0cd2de2e0545cf5ed69f @@ -79,6 +88,11 @@ GEM aws-sdk-resources (2.10.7) aws-sdk-core (= 2.10.7) aws-sigv4 (1.0.0) + azure-storage (0.11.4.preview) + azure-core (~> 0.1) + faraday (~> 0.9) + faraday_middleware (~> 0.10) + nokogiri (~> 1.6) builder (3.2.3) byebug (9.0.6) concurrent-ruby (1.0.5) @@ -88,6 +102,8 @@ GEM erubi (1.6.1) faraday (0.12.1) multipart-post (>= 1.2, < 3) + faraday_middleware (0.12.0) + faraday (>= 0.7.4, < 1.0) globalid (0.4.0) activesupport (>= 4.2.0) google-api-client (0.13.0) @@ -201,6 +217,8 @@ PLATFORMS DEPENDENCIES activestorage! aws-sdk (~> 2) + azure-core! + azure-storage bundler (~> 1.15) byebug google-cloud-storage (~> 1.3) diff --git a/config/storage_services.yml b/config/storage_services.yml index c80a3e8453..057e15e74d 100644 --- a/config/storage_services.yml +++ b/config/storage_services.yml @@ -21,6 +21,13 @@ google: keyfile: <%= Rails.root.join("path/to/gcs.keyfile") %> bucket: your_own_bucket +microsoft: + service: Azure + path: your_azure_storage_path + storage_account_name: your_account_name + storage_access_key: <%= Rails.application.secrets.azure[:secret_access_key] %> + container: your_container_name + mirror: service: Mirror primary: local diff --git a/lib/active_storage/service/azure_service.rb b/lib/active_storage/service/azure_service.rb new file mode 100644 index 0000000000..a505b9a0ee --- /dev/null +++ b/lib/active_storage/service/azure_service.rb @@ -0,0 +1,115 @@ +require "active_support/core_ext/numeric/bytes" +require "azure/storage" +require "azure/storage/core/auth/shared_access_signature" + +# Wraps the Microsoft Azure Storage Blob Service as a Active Storage service. +# See `ActiveStorage::Service` for the generic API documentation that applies to all services. +class ActiveStorage::Service::AzureService < ActiveStorage::Service + attr_reader :client, :path, :blobs, :container, :signer + + def initialize(path:, storage_account_name:, storage_access_key:, container:) + @client = Azure::Storage::Client.create(storage_account_name: storage_account_name, storage_access_key: storage_access_key) + @signer = Azure::Storage::Core::Auth::SharedAccessSignature.new(storage_account_name, storage_access_key) + @blobs = client.blob_client + @container = container + @path = path + end + + def upload(key, io, checksum: nil) + instrument :upload, key, checksum: checksum do + begin + blobs.create_block_blob(container, key, io, content_md5: checksum) + rescue Azure::Core::Http::HTTPError => e + raise ActiveStorage::IntegrityError + end + end + end + + def download(key) + if block_given? + instrument :streaming_download, key do + stream(key, &block) + end + else + instrument :download, key do + _, io = blobs.get_blob(container, key) + io.force_encoding(Encoding::BINARY) + end + end + end + + def delete(key) + instrument :delete, key do + begin + blobs.delete_blob(container, key) + rescue Azure::Core::Http::HTTPError + false + end + end + end + + def exist?(key) + instrument :exist, key do |payload| + answer = blob_for(key).present? + payload[:exist] = answer + answer + end + end + + def url(key, expires_in:, disposition:, filename:) + instrument :url, key do |payload| + base_url = url_for(key) + generated_url = signer.signed_uri(URI(base_url), false, permissions: "r", + expiry: format_expiry(expires_in), content_disposition: "#{disposition}; filename=\"#{filename}\"").to_s + + payload[:url] = generated_url + + generated_url + end + end + + def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) + instrument :url, key do |payload| + base_url = url_for(key) + generated_url = signer.signed_uri(URI(base_url), false, permissions: "rw", + expiry: format_expiry(expires_in)).to_s + + payload[:url] = generated_url + + generated_url + end + end + + def headers_for_direct_upload(key, content_type:, checksum:, **) + { "Content-Type" => content_type, "Content-MD5" => checksum, "x-ms-blob-type" => "BlockBlob" } + end + + private + def url_for(key) + "#{path}/#{container}/#{key}" + end + + def blob_for(key) + blobs.get_blob_properties(container, key) + rescue Azure::Core::Http::HTTPError + false + end + + def format_expiry(expires_in) + expires_in ? Time.now.utc.advance(seconds: expires_in).iso8601 : nil + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key, options = {}, &block) + blob = blob_for(key) + + chunk_size = 5.megabytes + offset = 0 + + while offset < blob.properties[:content_length] + _, io = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) + yield io + offset += chunk_size + end + end +end diff --git a/test/controllers/direct_uploads_controller_test.rb b/test/controllers/direct_uploads_controller_test.rb index 7ffa77ea73..7185bf2737 100644 --- a/test/controllers/direct_uploads_controller_test.rb +++ b/test/controllers/direct_uploads_controller_test.rb @@ -67,6 +67,40 @@ class ActiveStorage::GCSDirectUploadsControllerTest < ActionDispatch::Integratio puts "Skipping GCS Direct Upload tests because no GCS configuration was supplied" end +if SERVICE_CONFIGURATIONS[:azure] + class ActiveStorage::AzureDirectUploadsControllerTest < ActionDispatch::IntegrationTest + setup do + @config = SERVICE_CONFIGURATIONS[:azure] + + @old_service = ActiveStorage::Blob.service + ActiveStorage::Blob.service = ActiveStorage::Service.configure(:azure, SERVICE_CONFIGURATIONS) + end + + teardown do + ActiveStorage::Blob.service = @old_service + end + + test "creating new direct upload" do + checksum = Digest::MD5.base64digest("Hello") + + post rails_direct_uploads_url, params: { blob: { + filename: "hello.txt", byte_size: 6, checksum: checksum, content_type: "text/plain" } } + + @response.parsed_body.tap do |details| + assert_equal ActiveStorage::Blob.find(details["id"]), ActiveStorage::Blob.find_signed(details["signed_id"]) + assert_equal "hello.txt", details["filename"] + assert_equal 6, details["byte_size"] + assert_equal checksum, details["checksum"] + assert_equal "text/plain", details["content_type"] + assert_match %r{#{@config[:storage_account_name]}\.blob\.core\.windows\.net/#{@config[:container]}}, details["direct_upload"]["url"] + assert_equal({ "Content-Type" => "text/plain", "Content-MD5" => checksum, "x-ms-blob-type" => "BlockBlob" }, details["direct_upload"]["headers"]) + end + end + end +else + puts "Skipping Azure Direct Upload tests because no Azure configuration was supplied" +end + class ActiveStorage::DiskDirectUploadsControllerTest < ActionDispatch::IntegrationTest test "creating new direct upload" do checksum = Digest::MD5.base64digest("Hello") diff --git a/test/service/azure_service_test.rb b/test/service/azure_service_test.rb new file mode 100644 index 0000000000..0ddbac83e7 --- /dev/null +++ b/test/service/azure_service_test.rb @@ -0,0 +1,14 @@ +require "service/shared_service_tests" +require "httparty" +require "uri" + +if SERVICE_CONFIGURATIONS[:azure] + class ActiveStorage::Service::AzureServiceTest < ActiveSupport::TestCase + SERVICE = ActiveStorage::Service.configure(:azure, SERVICE_CONFIGURATIONS) + + include ActiveStorage::Service::SharedServiceTests + end + +else + puts "Skipping Azure Storage Service tests because no Azure configuration was supplied" +end diff --git a/test/service/configurations-example.yml b/test/service/configurations-example.yml index 8bcc57f05a..68f6ae4224 100644 --- a/test/service/configurations-example.yml +++ b/test/service/configurations-example.yml @@ -22,3 +22,10 @@ gcs: } project: bucket: + +azure: + service: Azure + path: "" + storage_account_name: "" + storage_access_key: "" + container: "" -- GitLab