Merge pull request #15 from phac-nml/copy-blob

Add Copy blob to client and use in compose when composing a new blob from a single blob

Author: JoeDupuis

CHANGELOG.md

@@ -1,5 +1,8 @@
 ## [Unreleased]
 
+- Add `copy_blob`
+- Update `compose` to use `copy_blob` if 1 source key and blob is <= 256MiB
+
 ## [0.5.6] 2025-01-17
 
 - Fix user delegation key not refreshing (#14)

lib/active_storage/service/azure_blob_service.rb

@@ -123,16 +123,22 @@ def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disp
     def compose(source_keys, destination_key, filename: nil, content_type: nil, disposition: nil, custom_metadata: {})
       content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
 
-      client.create_append_blob(
-        destination_key,
-        content_type: content_type,
-        content_disposition: content_disposition,
-        metadata: custom_metadata,
-      )
-
-      source_keys.each do |source_key|
-        stream(source_key) do |chunk|
-          client.append_blob_block(destination_key, chunk)
+      # use copy_blob operation if composing a new blob from a single existing blob
+      # and that single blob is <= 256 MiB which is the upper limit for copy_blob operation
+      if source_keys.length == 1 && client.get_blob_properties(source_keys[0]).size <= 256.megabytes
+        client.copy_blob(destination_key, source_keys[0], metadata: custom_metadata)
+      else
+        client.create_append_blob(
+          destination_key,
+          content_type: content_type,
+          content_disposition: content_disposition,
+          metadata: custom_metadata,
+        )
+
+        source_keys.each do |source_key|
+          stream(source_key) do |chunk|
+            client.append_blob_block(destination_key, chunk)
+          end
         end
       end
     end

lib/azure_blob/client.rb

@@ -77,6 +77,25 @@ def get_blob(key, options = {})
       Http.new(uri, headers, signer:).get
     end
 
+    # Copy a blob
+    #
+    # Calls to {Copy Blob From URL}[https://learn.microsoft.com/en-us/rest/api/storageservices/copy-blob-from-url]
+    #
+    # Takes a key (path) and a source_key (path).
+    #
+    def copy_blob(key, source_key, options = {})
+      uri = generate_uri("#{container}/#{key}")
+
+      source_uri = signed_uri(source_key, permissions: "r", expiry: Time.at(Time.now.to_i + 300).utc.iso8601)
+
+      headers = {
+        "x-ms-copy-source": source_uri.to_s,
+        "x-ms-requires-sync": "true",
+      }
+
+      Http.new(uri, headers, signer:, **options.slice(:metadata, :tags)).put
+    end
+
     # Delete a blob
     #
     # Calls to {Delete Blob}[https://learn.microsoft.com/en-us/rest/api/storageservices/delete-blob]
@@ -202,7 +221,7 @@ def create_container(options = {})
       uri = generate_uri(container)
       headers = {}
       headers[:"x-ms-blob-public-access"] = "blob" if options[:public_access]
-      headers[:"x-ms-blob-public-access"] = options[:public_access] if ["container","blob"].include?(options[:public_access])
+      headers[:"x-ms-blob-public-access"] = options[:public_access] if [ "container", "blob" ].include?(options[:public_access])
 
       uri.query = URI.encode_www_form(restype: "container")
       response = Http.new(uri, headers, signer:).put

test/client/test_client.rb

@@ -175,6 +175,17 @@ def test_download_404
     assert_raises(AzureBlob::Http::FileNotFoundError) { client.get_blob(key) }
   end
 
+  def test_copy
+    client.create_block_blob(key, content)
+    assert_equal content, client.get_blob(key)
+
+    copy_key = "#{key}_copy"
+
+    client.copy_blob(copy_key, key)
+
+    assert_equal content, client.get_blob(copy_key)
+  end
+
   def test_delete
     client.create_block_blob(key, content)
     assert_equal content, client.get_blob(key)

test/rails/service/azure_blob_service_test.rb

@@ -116,4 +116,25 @@ class ActiveStorage::Service::AzureBlobServiceTest < ActiveSupport::TestCase
   ensure
     @service.delete(key)
   end
+
+  test "composing a blob from one source blob" do
+    key  = SecureRandom.base58(24)
+    data = "Something else entirely!"
+
+    Tempfile.open do |file|
+      file.write(data)
+      file.rewind
+      @service.upload(key, file)
+    end
+
+    assert_equal data, @service.download(key)
+
+    copy_key = SecureRandom.base58(24)
+    @service.compose([ key ], copy_key)
+
+    assert_equal data, @service.download(copy_key)
+  ensure
+    @service.delete key
+    @service.delete copy_key
+  end
 end

test/rails/service/shared_service_tests.rb

@@ -158,5 +158,24 @@ module ActiveStorage::Service::SharedServiceTests
 
       assert_equal "Together", @service.download(destination_key)
     end
+
+    test "compose from single blob" do
+      keys = [ SecureRandom.base58(24) ]
+      data = %w[Together]
+      keys.zip(data).each do |key, data|
+        @service.upload(
+          key,
+          StringIO.new(data),
+          checksum: Digest::MD5.base64digest(data),
+          disposition: :attachment,
+          filename: ActiveStorage::Filename.new("test.html"),
+          content_type: "text/html",
+        )
+      end
+      destination_key = SecureRandom.base58(24)
+      @service.compose(keys, destination_key)
+
+      assert_equal "Together", @service.download(destination_key)
+    end
   end
 end