diff options
Diffstat (limited to 'activestorage/lib/active_storage/service')
5 files changed, 110 insertions, 47 deletions
diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index 2867a4e441..66aabc1f9f 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -20,7 +20,7 @@ module ActiveStorage def upload(key, io, checksum: nil) instrument :upload, key: key, checksum: checksum do begin - blobs.create_block_blob(container, key, io, content_md5: checksum) + blobs.create_block_blob(container, key, IO.try_convert(io) || io, content_md5: checksum) rescue Azure::Core::Http::HTTPError raise ActiveStorage::IntegrityError end @@ -34,16 +34,20 @@ module ActiveStorage end else instrument :download, key: key do - _, io = blobs.get_blob(container, key) - io.force_encoding(Encoding::BINARY) + handle_errors do + _, io = blobs.get_blob(container, key) + io.force_encoding(Encoding::BINARY) + end end end end def download_chunk(key, range) instrument :download_chunk, key: key, range: range do - _, io = blobs.get_blob(container, key, start_range: range.begin, end_range: range.exclude_end? ? range.end - 1 : range.end) - io.force_encoding(Encoding::BINARY) + handle_errors do + _, io = blobs.get_blob(container, key, start_range: range.begin, end_range: range.exclude_end? ? range.end - 1 : range.end) + io.force_encoding(Encoding::BINARY) + end end end @@ -139,11 +143,23 @@ module ActiveStorage chunk_size = 5.megabytes offset = 0 + raise ActiveStorage::FileNotFoundError unless blob.present? + while offset < blob.properties[:content_length] _, chunk = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) yield chunk.force_encoding(Encoding::BINARY) offset += chunk_size end end + + def handle_errors + yield + rescue Azure::Core::Http::HTTPError => e + if e.type == "BlobNotFound" + raise ActiveStorage::FileNotFoundError + else + raise + end + end end end diff --git a/activestorage/lib/active_storage/service/configurator.rb b/activestorage/lib/active_storage/service/configurator.rb index 39951fd026..fa80c66c3b 100644 --- a/activestorage/lib/active_storage/service/configurator.rb +++ b/activestorage/lib/active_storage/service/configurator.rb @@ -26,7 +26,9 @@ module ActiveStorage def resolve(class_name) require "active_storage/service/#{class_name.to_s.underscore}_service" - ActiveStorage::Service.const_get(:"#{class_name}Service") + ActiveStorage::Service.const_get(:"#{class_name.camelize}Service") + rescue LoadError + raise "Missing service adapter for #{class_name.inspect}" end end end diff --git a/activestorage/lib/active_storage/service/disk_service.rb b/activestorage/lib/active_storage/service/disk_service.rb index 75b66081c3..52f3a3df16 100644 --- a/activestorage/lib/active_storage/service/disk_service.rb +++ b/activestorage/lib/active_storage/service/disk_service.rb @@ -22,27 +22,31 @@ module ActiveStorage end end - def download(key) + def download(key, &block) if block_given? instrument :streaming_download, key: key do - File.open(path_for(key), "rb") do |file| - while data = file.read(64.kilobytes) - yield data - end - end + stream key, &block end else instrument :download, key: key do - File.binread path_for(key) + begin + File.binread path_for(key) + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError + end end end end def download_chunk(key, range) instrument :download_chunk, key: key, range: range do - File.open(path_for(key), "rb") do |file| - file.seek range.begin - file.read range.size + begin + File.open(path_for(key), "rb") do |file| + file.seek range.begin + file.read range.size + end + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError end end end @@ -78,8 +82,9 @@ module ActiveStorage verified_key_with_expiration = ActiveStorage.verifier.generate(key, expires_in: expires_in, purpose: :blob_key) generated_url = - url_helpers.rails_disk_service_path( + url_helpers.rails_disk_service_url( verified_key_with_expiration, + host: current_host, filename: filename, disposition: content_disposition_with(type: disposition, filename: filename), content_type: content_type @@ -104,7 +109,7 @@ module ActiveStorage purpose: :blob_token } ) - generated_url = url_helpers.update_rails_disk_service_path(verified_token_with_expiration) + generated_url = url_helpers.update_rails_disk_service_url(verified_token_with_expiration, host: current_host) payload[:url] = generated_url @@ -116,9 +121,19 @@ module ActiveStorage { "Content-Type" => content_type } end + def path_for(key) #:nodoc: + File.join root, folder_for(key), key + end + private - def path_for(key) - File.join root, folder_for(key), key + def stream(key) + File.open(path_for(key), "rb") do |file| + while data = file.read(5.megabytes) + yield data + end + end + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError end def folder_for(key) @@ -129,7 +144,6 @@ module ActiveStorage path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } end - def ensure_integrity_of(key, checksum) unless Digest::MD5.file(path_for(key)).base64digest == checksum delete key @@ -137,9 +151,12 @@ module ActiveStorage end end - def url_helpers @url_helpers ||= Rails.application.routes.url_helpers end + + def current_host + ActiveStorage::Current.host + end end end diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb index 369c33cbdb..18c0f14cfc 100644 --- a/activestorage/lib/active_storage/service/gcs_service.rb +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -1,12 +1,7 @@ # frozen_string_literal: true -gem "google-cloud-storage", "~> 1.8" - +gem "google-cloud-storage", "~> 1.11" require "google/cloud/storage" -require "net/http" - -require "active_support/core_ext/object/to_query" -require "active_storage/filename" module ActiveStorage # Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API @@ -32,26 +27,28 @@ module ActiveStorage end end - # FIXME: Download in chunks when given a block. - def download(key) - instrument :download, key: key do - io = file_for(key).download - io.rewind - - if block_given? - yield io.read - else - io.read + def download(key, &block) + if block_given? + instrument :streaming_download, key: key do + stream(key, &block) + end + else + instrument :download, key: key do + begin + file_for(key).download.string + rescue Google::Cloud::NotFoundError + raise ActiveStorage::FileNotFoundError + end end end end def download_chunk(key, range) instrument :download_chunk, key: key, range: range do - uri = URI(url(key, expires_in: 30.seconds, filename: ActiveStorage::Filename.new(""), content_type: "application/octet-stream", disposition: "inline")) - - Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |client| - client.get(uri, "Range" => "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body + begin + file_for(key).download(range: range).string + rescue Google::Cloud::NotFoundError + raise ActiveStorage::FileNotFoundError end end end @@ -68,7 +65,13 @@ module ActiveStorage def delete_prefixed(prefix) instrument :delete_prefixed, prefix: prefix do - bucket.files(prefix: prefix).all(&:delete) + bucket.files(prefix: prefix).all do |file| + begin + file.delete + rescue Google::Cloud::NotFoundError + # Ignore concurrently-deleted files + end + end end end @@ -110,8 +113,23 @@ module ActiveStorage private attr_reader :config - def file_for(key) - bucket.file(key, skip_lookup: true) + def file_for(key, skip_lookup: true) + bucket.file(key, skip_lookup: skip_lookup) + end + + # Reads the file for the given key in chunks, yielding each to the block. + def stream(key) + file = file_for(key, skip_lookup: false) + + chunk_size = 5.megabytes + offset = 0 + + raise ActiveStorage::FileNotFoundError unless file.present? + + while offset < file.size + yield file.download(range: offset..(offset + chunk_size - 1)).string + offset += chunk_size + end end def bucket diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index 5e489f4be1..89a9e54158 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -33,14 +33,22 @@ module ActiveStorage end else instrument :download, key: key do - object_for(key).get.body.read.force_encoding(Encoding::BINARY) + begin + object_for(key).get.body.string.force_encoding(Encoding::BINARY) + rescue Aws::S3::Errors::NoSuchKey + raise ActiveStorage::FileNotFoundError + end end end end def download_chunk(key, range) instrument :download_chunk, key: key, range: range do - object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.read.force_encoding(Encoding::BINARY) + begin + object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.read.force_encoding(Encoding::BINARY) + rescue Aws::S3::Errors::NoSuchKey + raise ActiveStorage::FileNotFoundError + end end end @@ -103,6 +111,8 @@ module ActiveStorage chunk_size = 5.megabytes offset = 0 + raise ActiveStorage::FileNotFoundError unless object.exists? + while offset < object.content_length yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.read.force_encoding(Encoding::BINARY) offset += chunk_size |