diff --git a/.gitignore b/.gitignore index d723847..3a4d0ac 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,7 @@ devenv.local.nix terraform.tfstate terraform.tfstate.backup .terraform.tfstate.lock.info -*.tfvars \ No newline at end of file +*.tfvars + +__azurite_db* +__blobstorage__/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 78eef9a..6a841e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## [Unreleased] +- Allow creating public container +- Add Azurite support ## [0.5.3] 2024-10-31 diff --git a/README.md b/README.md index a36da33..550d1b0 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,30 @@ prod: principal_id: 71b34410-4c50-451d-b456-95ead1b18cce ``` +### Azurite + +To use Azurite, pass the `storage_blob_host` config key with the Azurite URL (`http://127.0.0.1:10000/devstoreaccount1` by default) +and the Azurite credentials (`devstoreaccount1` and `Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==` by default). + +Example: + +``` +dev: + service: AzureBlob + container: container_name + storage_account_name: devstoreaccount1 + storage_access_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + storage_blob_host: http://127.0.0.1:10000/devstoreaccount1 +``` + +You'll have to create the container before you can start uploading files. +You can do so using Azure CLI, Azure Storage Explorer, or by running: + +`bin/rails runner "ActiveStorage::Blob.service.client.tap{|client| client.create_container unless client.get_container_properties.present?}.tap { |client| puts 'done!' if client.get_container_properties.present?}"` + +Make sure that `config.active_storage.service = :dev` is set to your azurite configuration. +Container names can't have any special characters, or you'll get an error. + ## Standalone Instantiate a client with your account name, an access key and the container name: diff --git a/Rakefile b/Rakefile index 7536975..50c918b 100644 --- a/Rakefile +++ b/Rakefile @@ -5,6 +5,7 @@ require "minitest/test_task" require "azure_blob" require_relative "test/support/app_service_vpn" require_relative "test/support/azure_vm_vpn" +require_relative "test/support/azurite" Minitest::TestTask.create(:test_rails) do self.test_globs = [ "test/rails/**/test_*.rb", @@ -39,6 +40,28 @@ ensure vpn.kill end +task :test_azurite do |t| + azurite = Azurite.new + # Azurite well-known credentials + # https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio%2Cblob-storage#well-known-storage-account-and-key + account_name = ENV["AZURE_ACCOUNT_NAME"] = "devstoreaccount1" + access_key = ENV["AZURE_ACCESS_KEY"] = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + host = ENV["STORAGE_BLOB_HOST"] = "http://127.0.0.1:10000/devstoreaccount1" + ENV["TESTING_AZURITE"] = "true" + + # Create containers + private_container = AzureBlob::Client.new(account_name:, access_key:, host:, container: ENV["AZURE_PRIVATE_CONTAINER"]) + public_container = AzureBlob::Client.new(account_name:, access_key:, host:, container: ENV["AZURE_PUBLIC_CONTAINER"]) + # public_container.delete_container + private_container.create_container unless private_container.get_container_properties.present? + public_container.create_container(public_access: true) unless public_container.get_container_properties.present? + + Rake::Task["test_client"].execute + Rake::Task["test_rails"].execute +ensure + azurite.kill +end + task :test_entra_id do |t| ENV["AZURE_ACCESS_KEY"] = nil Rake::Task["test"].execute diff --git a/devenv.nix b/devenv.nix index 6b8d95d..d24045a 100644 --- a/devenv.nix +++ b/devenv.nix @@ -15,6 +15,7 @@ sshuttle sshpass rsync + azurite ]; languages.ruby.enable = true; diff --git a/lib/active_storage/service/azure_blob_service.rb b/lib/active_storage/service/azure_blob_service.rb index bc56c42..a6836df 100644 --- a/lib/active_storage/service/azure_blob_service.rb +++ b/lib/active_storage/service/azure_blob_service.rb @@ -35,13 +35,14 @@ module ActiveStorage class Service::AzureBlobService < Service attr_reader :client, :container, :signer - def initialize(storage_account_name:, storage_access_key: nil, container:, public: false, **options) + def initialize(storage_account_name:, storage_access_key: nil, container:, storage_blob_host: nil, public: false, **options) @container = container @public = public @client = AzureBlob::Client.new( account_name: storage_account_name, access_key: storage_access_key, container: container, + host: storage_blob_host, **options) end diff --git a/lib/azure_blob/client.rb b/lib/azure_blob/client.rb index e89cf7c..c6235ba 100644 --- a/lib/azure_blob/client.rb +++ b/lib/azure_blob/client.rb @@ -15,9 +15,10 @@ module AzureBlob # AzureBlob Client class. You interact with the Azure Blob api # through an instance of this class. class Client - def initialize(account_name:, access_key: nil, principal_id: nil, container:, **options) + def initialize(account_name:, access_key: nil, principal_id: nil, container:, host: nil, **options) @account_name = account_name @container = container + @host = host @cloud_regions = options[:cloud_regions]&.to_sym || :global no_access_key = access_key.nil? || access_key&.empty? @@ -29,8 +30,8 @@ def initialize(account_name:, access_key: nil, principal_id: nil, container:, ** ) end @signer = using_managed_identities ? - AzureBlob::EntraIdSigner.new(account_name:, host:, principal_id:) : - AzureBlob::SharedKeySigner.new(account_name:, access_key:) + AzureBlob::EntraIdSigner.new(account_name:, host: self.host, principal_id:) : + AzureBlob::SharedKeySigner.new(account_name:, access_key:, host: self.host) end # Create a blob of type block. Will automatically split the the blob in multiple block and send the blob in pieces (blocks) if the blob is too big. @@ -190,8 +191,12 @@ def get_container_properties(options = {}) # Calls to {Create Container}[https://learn.microsoft.com/en-us/rest/api/storageservices/create-container] def create_container(options = {}) uri = generate_uri(container) + headers = {} + headers[:"x-ms-blob-public-access"] = "blob" if options[:public_access] + headers[:"x-ms-blob-public-access"] = options[:public_access] if ["container","blob"].include?(options[:public_access]) + uri.query = URI.encode_www_form(restype: "container") - response = Http.new(uri, signer:).put + response = Http.new(uri, headers, signer:).put end # Delete the container diff --git a/lib/azure_blob/shared_key_signer.rb b/lib/azure_blob/shared_key_signer.rb index 907954c..dc00196 100644 --- a/lib/azure_blob/shared_key_signer.rb +++ b/lib/azure_blob/shared_key_signer.rb @@ -7,9 +7,11 @@ module AzureBlob class SharedKeySigner # :nodoc: - def initialize(account_name:, access_key:) + def initialize(account_name:, access_key:, host:) @account_name = account_name @access_key = Base64.decode64(access_key) + @host = host + @remove_prefix = @host.include?("/#{@account_name}") end def authorization_header(uri:, verb:, headers: {}) @@ -39,6 +41,11 @@ def authorization_header(uri:, verb:, headers: {}) end def sas_token(uri, options = {}) + if remove_prefix + uri = uri.clone + uri.path = uri.path.delete_prefix("/#{account_name}") + end + to_sign = [ options[:permissions], options[:start], @@ -99,6 +106,6 @@ module Resources # :nodoc: end end - attr_reader :access_key, :account_name + attr_reader :access_key, :account_name, :remove_prefix end end diff --git a/test/client/test_client.rb b/test/client/test_client.rb index 89ea6de..6585723 100644 --- a/test/client/test_client.rb +++ b/test/client/test_client.rb @@ -13,11 +13,13 @@ def setup @access_key = ENV["AZURE_ACCESS_KEY"] @container = ENV["AZURE_PRIVATE_CONTAINER"] @principal_id = ENV["AZURE_PRINCIPAL_ID"] + @host = ENV["STORAGE_BLOB_HOST"] @client = AzureBlob::Client.new( account_name: @account_name, access_key: @access_key, container: @container, principal_id: @principal_id, + host: @host, ) @key = "test client##{name}" @content = "Some random content #{Random.rand(200)}" @@ -104,11 +106,13 @@ def test_upload_integrity_block end def test_upload_raise_on_invalid_checksum_blob + skip if ENV["TESTING_AZURITE"] checksum = OpenSSL::Digest::MD5.base64digest(content + "a") assert_raises(AzureBlob::Http::IntegrityError) { client.create_block_blob(key, content, content_md5: checksum) } end def test_upload_raise_on_invalid_checksum_block + skip if ENV["TESTING_AZURITE"] checksum = OpenSSL::Digest::MD5.base64digest(content + "a") assert_raises(AzureBlob::Http::IntegrityError) { client.put_blob_block(key, 0, content, content_md5: checksum) } end @@ -338,6 +342,7 @@ def test_create_container access_key: @access_key, container: Random.alphanumeric(20).tr("0-9", "").downcase, principal_id: @principal_id, + host: @host, ) container = client.get_container_properties refute container.present? diff --git a/test/rails/controllers/direct_uploads_controller_test.rb b/test/rails/controllers/direct_uploads_controller_test.rb index b40f2ec..f307cd8 100644 --- a/test/rails/controllers/direct_uploads_controller_test.rb +++ b/test/rails/controllers/direct_uploads_controller_test.rb @@ -23,6 +23,8 @@ class ActiveStorage::AzureBlobDirectUploadsControllerTest < ActionDispatch::Inte post rails_direct_uploads_url, params: { blob: { filename: "hello.txt", byte_size: 6, checksum: checksum, content_type: "text/plain", metadata: metadata, } } + host = @config[:host] || "https://#{@config[:storage_account_name]}.blob.core.windows.net" + response.parsed_body.tap do |details| assert_equal ActiveStorage::Blob.find(details["id"]), ActiveStorage::Blob.find_signed!(details["signed_id"]) assert_equal "hello.txt", details["filename"] @@ -30,7 +32,7 @@ class ActiveStorage::AzureBlobDirectUploadsControllerTest < ActionDispatch::Inte assert_equal checksum, details["checksum"] assert_equal metadata, details["metadata"] assert_equal "text/plain", details["content_type"] - assert_match %r{#{@config[:storage_account_name]}\.blob\.core\.windows\.net/#{@config[:container]}}, details["direct_upload"]["url"] + assert details["direct_upload"]["url"].start_with?("#{host}/#{@config[:container]}") assert_equal({ "Content-Type" => "text/plain", "Content-MD5" => checksum, "x-ms-blob-content-disposition" => "inline; filename=\"hello.txt\"; filename*=UTF-8''hello.txt", "x-ms-blob-type" => "BlockBlob" }, details["direct_upload"]["headers"]) end end diff --git a/test/rails/service/azure_blob_public_service_test.rb b/test/rails/service/azure_blob_public_service_test.rb index e30295d..66c629d 100644 --- a/test/rails/service/azure_blob_public_service_test.rb +++ b/test/rails/service/azure_blob_public_service_test.rb @@ -8,10 +8,15 @@ class ActiveStorage::Service::AzureBlobPublicServiceTest < ActiveSupport::TestCa include ActiveStorage::Service::SharedServiceTests + setup do + @config = SERVICE_CONFIGURATIONS[:azure_public] + end + test "public URL generation" do url = @service.url(@key, filename: ActiveStorage::Filename.new("avatar.png")) + host = @config[:host] || "https://#{@config[:storage_account_name]}.blob.core.windows.net" - assert_match(/.*\.blob\.core\.windows\.net\/.*\/#{@key}/, url) + assert url.start_with?("#{host}/#{@config[:container]}/#{@key}") response = Net::HTTP.get_response(URI(url)) assert_equal "200", response.code @@ -30,7 +35,7 @@ class ActiveStorage::Service::AzureBlobPublicServiceTest < ActiveSupport::TestCa @service.headers_for_direct_upload(key, checksum: checksum, content_type: content_type, filename: ActiveStorage::Filename.new("test.txt")).each do |k, v| request.add_field k, v end - Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http| + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.port == 443) do |http| http.request request end diff --git a/test/rails/service/azure_blob_service_test.rb b/test/rails/service/azure_blob_service_test.rb index 1277e83..3e5b6c5 100644 --- a/test/rails/service/azure_blob_service_test.rb +++ b/test/rails/service/azure_blob_service_test.rb @@ -21,7 +21,8 @@ class ActiveStorage::Service::AzureBlobServiceTest < ActiveSupport::TestCase @service.headers_for_direct_upload(key, checksum: checksum, content_type: content_type, filename: ActiveStorage::Filename.new("test.txt")).each do |k, v| request.add_field k, v end - Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http| + + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.port == 443) do |http| http.request request end @@ -42,7 +43,7 @@ class ActiveStorage::Service::AzureBlobServiceTest < ActiveSupport::TestCase @service.headers_for_direct_upload(key, checksum: checksum, content_type: "text/plain", filename: ActiveStorage::Filename.new("test.txt"), disposition: :attachment).each do |k, v| request.add_field k, v end - Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http| + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.port == 443) do |http| http.request request end diff --git a/test/rails/service/configurations.yml b/test/rails/service/configurations.yml index 2acc30d..aa332ff 100644 --- a/test/rails/service/configurations.yml +++ b/test/rails/service/configurations.yml @@ -3,6 +3,7 @@ DEFAULT: &default storage_account_name: <%= ENV["AZURE_ACCOUNT_NAME"] %> storage_access_key: <%= ENV["AZURE_ACCESS_KEY"] %> principal_id: <%= ENV["AZURE_PRINCIPAL_ID"]%> + storage_blob_host: <%= ENV["STORAGE_BLOB_HOST"] %> azure: <<: *default diff --git a/test/rails/service/shared_service_tests.rb b/test/rails/service/shared_service_tests.rb index c910d96..fbe2d2d 100644 --- a/test/rails/service/shared_service_tests.rb +++ b/test/rails/service/shared_service_tests.rb @@ -30,6 +30,7 @@ module ActiveStorage::Service::SharedServiceTests end test "uploading without integrity" do + skip if ENV["TESTING_AZURITE"] key = SecureRandom.base58(24) data = "Something else entirely!" @@ -39,7 +40,7 @@ module ActiveStorage::Service::SharedServiceTests assert_not @service.exist?(key) ensure - @service.delete key + @service.delete key unless ENV["TESTING_AZURITE"] end test "uploading with integrity and multiple keys" do diff --git a/test/support/azurite.rb b/test/support/azurite.rb new file mode 100644 index 0000000..5bb8f52 --- /dev/null +++ b/test/support/azurite.rb @@ -0,0 +1,20 @@ +require "open3" +require "shellwords" + +class Azurite + def initialize(verbose: false) + @verbose = verbose + stdin, stdout, @wait_thread = Open3.popen2e("azurite") + stdout.each do |line| + break if line.include?("Azurite Blob service is successfully listening at http://127.0.0.1:10000") + end + end + + def kill + Process.kill("INT", wait_thread.pid) + end + + private + + attr_reader :wait_thread +end