diff options
Diffstat (limited to 'activestorage/lib')
22 files changed, 355 insertions, 105 deletions
diff --git a/activestorage/lib/active_storage.rb b/activestorage/lib/active_storage.rb index 928db32691..5c5da551ae 100644 --- a/activestorage/lib/active_storage.rb +++ b/activestorage/lib/active_storage.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true #-- -# Copyright (c) 2017-2018 David Heinemeier Hansson, Basecamp +# Copyright (c) 2017-2019 David Heinemeier Hansson, Basecamp # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -26,6 +26,7 @@ require "active_record" require "active_support" require "active_support/rails" +require "active_support/core_ext/numeric/time" require "active_storage/version" require "active_storage/errors" @@ -42,12 +43,23 @@ module ActiveStorage mattr_accessor :logger mattr_accessor :verifier - mattr_accessor :queue + mattr_accessor :queues, default: {} mattr_accessor :previewers, default: [] mattr_accessor :analyzers, default: [] mattr_accessor :variant_processor, default: :mini_magick mattr_accessor :paths, default: {} mattr_accessor :variable_content_types, default: [] mattr_accessor :content_types_to_serve_as_binary, default: [] + mattr_accessor :content_types_allowed_inline, default: [] + mattr_accessor :binary_content_type, default: "application/octet-stream" mattr_accessor :service_urls_expire_in, default: 5.minutes + mattr_accessor :routes_prefix, default: "/rails/active_storage" + + module Transformers + extend ActiveSupport::Autoload + + autoload :Transformer + autoload :ImageProcessingTransformer + autoload :MiniMagickTransformer + end end diff --git a/activestorage/lib/active_storage/analyzer.rb b/activestorage/lib/active_storage/analyzer.rb index caa25418a5..26414ffbc2 100644 --- a/activestorage/lib/active_storage/analyzer.rb +++ b/activestorage/lib/active_storage/analyzer.rb @@ -24,14 +24,14 @@ module ActiveStorage private # Downloads the blob to a tempfile on disk. Yields the tempfile. def download_blob_to_tempfile(&block) #:doc: - blob.open tempdir: tempdir, &block + blob.open tmpdir: tmpdir, &block end def logger #:doc: ActiveStorage.logger end - def tempdir #:doc: + def tmpdir #:doc: Dir.tmpdir end end diff --git a/activestorage/lib/active_storage/analyzer/video_analyzer.rb b/activestorage/lib/active_storage/analyzer/video_analyzer.rb index 18d8ff8237..e56c97f4f5 100644 --- a/activestorage/lib/active_storage/analyzer/video_analyzer.rb +++ b/activestorage/lib/active_storage/analyzer/video_analyzer.rb @@ -11,7 +11,7 @@ module ActiveStorage # # Example: # - # ActiveStorage::VideoAnalyzer.new(blob).metadata + # ActiveStorage::Analyzer::VideoAnalyzer.new(blob).metadata # # => { width: 640.0, height: 480.0, duration: 5.0, angle: 0, display_aspect_ratio: [4, 3] } # # When a video's angle is 90 or 270 degrees, its width and height are automatically swapped for convenience. diff --git a/activestorage/lib/active_storage/attached/changes/create_one.rb b/activestorage/lib/active_storage/attached/changes/create_one.rb index 5812fd2b08..89cccfb58a 100644 --- a/activestorage/lib/active_storage/attached/changes/create_one.rb +++ b/activestorage/lib/active_storage/attached/changes/create_one.rb @@ -30,6 +30,7 @@ module ActiveStorage def save record.public_send("#{name}_attachment=", attachment) + record.public_send("#{name}_blob=", blob) end private diff --git a/activestorage/lib/active_storage/downloader.rb b/activestorage/lib/active_storage/downloader.rb index 87be6efb05..4d7e832af5 100644 --- a/activestorage/lib/active_storage/downloader.rb +++ b/activestorage/lib/active_storage/downloader.rb @@ -2,24 +2,23 @@ module ActiveStorage class Downloader #:nodoc: - def initialize(blob, tempdir: nil) - @blob = blob - @tempdir = tempdir + attr_reader :service + + def initialize(service) + @service = service end - def download_blob_to_tempfile - open_tempfile do |file| - download_blob_to file - verify_integrity_of file + def open(key, checksum:, name: "ActiveStorage-", tmpdir: nil) + open_tempfile(name, tmpdir) do |file| + download key, file + verify_integrity_of file, checksum: checksum yield file end end private - attr_reader :blob, :tempdir - - def open_tempfile - file = Tempfile.open([ "ActiveStorage-#{blob.id}-", blob.filename.extension_with_delimiter ], tempdir) + def open_tempfile(name, tmpdir = nil) + file = Tempfile.open(name, tmpdir) begin yield file @@ -28,15 +27,15 @@ module ActiveStorage end end - def download_blob_to(file) + def download(key, file) file.binmode - blob.download { |chunk| file.write(chunk) } + service.download(key) { |chunk| file.write(chunk) } file.flush file.rewind end - def verify_integrity_of(file) - unless Digest::MD5.file(file).base64digest == blob.checksum + def verify_integrity_of(file, checksum:) + unless Digest::MD5.file(file).base64digest == checksum raise ActiveStorage::IntegrityError end end diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb index 9d6a27eabe..fc75a8f816 100644 --- a/activestorage/lib/active_storage/engine.rb +++ b/activestorage/lib/active_storage/engine.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true require "rails" +require "action_controller/railtie" +require "active_job/railtie" +require "active_record/railtie" + require "active_storage" require "active_storage/previewer/poppler_pdf_previewer" @@ -20,12 +24,15 @@ module ActiveStorage config.active_storage.previewers = [ ActiveStorage::Previewer::PopplerPDFPreviewer, ActiveStorage::Previewer::MuPDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ] config.active_storage.analyzers = [ ActiveStorage::Analyzer::ImageAnalyzer, ActiveStorage::Analyzer::VideoAnalyzer ] config.active_storage.paths = ActiveSupport::OrderedOptions.new + config.active_storage.queues = ActiveSupport::OrderedOptions.new config.active_storage.variable_content_types = %w( image/png image/gif image/jpg image/jpeg + image/pjpeg + image/tiff image/vnd.adobe.photoshop image/vnd.microsoft.icon ) @@ -39,6 +46,19 @@ module ActiveStorage text/xml application/xml application/xhtml+xml + application/mathml+xml + text/cache-manifest + ) + + config.active_storage.content_types_allowed_inline = %w( + image/png + image/gif + image/jpg + image/jpeg + image/tiff + image/vnd.adobe.photoshop + image/vnd.microsoft.icon + application/pdf ) config.eager_load_namespaces << ActiveStorage @@ -46,15 +66,17 @@ module ActiveStorage initializer "active_storage.configs" do config.after_initialize do |app| ActiveStorage.logger = app.config.active_storage.logger || Rails.logger - ActiveStorage.queue = app.config.active_storage.queue ActiveStorage.variant_processor = app.config.active_storage.variant_processor || :mini_magick ActiveStorage.previewers = app.config.active_storage.previewers || [] ActiveStorage.analyzers = app.config.active_storage.analyzers || [] ActiveStorage.paths = app.config.active_storage.paths || {} + ActiveStorage.routes_prefix = app.config.active_storage.routes_prefix || "/rails/active_storage" ActiveStorage.variable_content_types = app.config.active_storage.variable_content_types || [] ActiveStorage.content_types_to_serve_as_binary = app.config.active_storage.content_types_to_serve_as_binary || [] ActiveStorage.service_urls_expire_in = app.config.active_storage.service_urls_expire_in || 5.minutes + ActiveStorage.content_types_allowed_inline = app.config.active_storage.content_types_allowed_inline || [] + ActiveStorage.binary_content_type = app.config.active_storage.binary_content_type || "application/octet-stream" end end @@ -99,6 +121,20 @@ module ActiveStorage end end + initializer "active_storage.queues" do + config.after_initialize do |app| + if queue = app.config.active_storage.queue + ActiveSupport::Deprecation.warn \ + "config.active_storage.queue is deprecated and will be removed in Rails 6.1. " \ + "Set config.active_storage.queues.purge and config.active_storage.queues.analysis instead." + + ActiveStorage.queues = { purge: queue, analysis: queue } + else + ActiveStorage.queues = app.config.active_storage.queues || {} + end + end + end + initializer "active_storage.reflection" do ActiveSupport.on_load(:active_record) do include Reflection::ActiveRecordExtensions diff --git a/activestorage/lib/active_storage/errors.rb b/activestorage/lib/active_storage/errors.rb index bedcd080c4..6475c1d076 100644 --- a/activestorage/lib/active_storage/errors.rb +++ b/activestorage/lib/active_storage/errors.rb @@ -1,11 +1,26 @@ # frozen_string_literal: true module ActiveStorage - class InvariableError < StandardError; end - class UnpreviewableError < StandardError; end - class UnrepresentableError < StandardError; end + # Generic base class for all Active Storage exceptions. + class Error < StandardError; end + + # Raised when ActiveStorage::Blob#variant is called on a blob that isn't variable. + # Use ActiveStorage::Blob#variable? to determine whether a blob is variable. + class InvariableError < Error; end + + # Raised when ActiveStorage::Blob#preview is called on a blob that isn't previewable. + # Use ActiveStorage::Blob#previewable? to determine whether a blob is previewable. + class UnpreviewableError < Error; end + + # Raised when ActiveStorage::Blob#representation is called on a blob that isn't representable. + # Use ActiveStorage::Blob#representable? to determine whether a blob is representable. + class UnrepresentableError < Error; end # Raised when uploaded or downloaded data does not match a precomputed checksum. # Indicates that a network error or a software bug caused data corruption. - class IntegrityError < StandardError; end + class IntegrityError < Error; end + + # Raised when ActiveStorage::Blob#download is called on a blob where the + # backing file is no longer present in its service. + class FileNotFoundError < Error; end end diff --git a/activestorage/lib/active_storage/gem_version.rb b/activestorage/lib/active_storage/gem_version.rb index 492620731b..d8f8577cc3 100644 --- a/activestorage/lib/active_storage/gem_version.rb +++ b/activestorage/lib/active_storage/gem_version.rb @@ -10,7 +10,7 @@ module ActiveStorage MAJOR = 6 MINOR = 0 TINY = 0 - PRE = "alpha" + PRE = "beta3" STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".") end diff --git a/activestorage/lib/active_storage/log_subscriber.rb b/activestorage/lib/active_storage/log_subscriber.rb index a4e148c1a5..6c0b4c30e7 100644 --- a/activestorage/lib/active_storage/log_subscriber.rb +++ b/activestorage/lib/active_storage/log_subscriber.rb @@ -14,6 +14,8 @@ module ActiveStorage info event, color("Downloaded file from key: #{key_in(event)}", BLUE) end + alias_method :service_streaming_download, :service_download + def service_delete(event) info event, color("Deleted file from key: #{key_in(event)}", RED) end diff --git a/activestorage/lib/active_storage/previewer.rb b/activestorage/lib/active_storage/previewer.rb index fb202f029a..af6bcadd4c 100644 --- a/activestorage/lib/active_storage/previewer.rb +++ b/activestorage/lib/active_storage/previewer.rb @@ -2,8 +2,8 @@ module ActiveStorage # This is an abstract base class for previewers, which generate images from blobs. See - # ActiveStorage::Previewer::PDFPreviewer and ActiveStorage::Previewer::VideoPreviewer for examples of - # concrete subclasses. + # ActiveStorage::Previewer::MuPDFPreviewer and ActiveStorage::Previewer::VideoPreviewer for + # examples of concrete subclasses. class Previewer attr_reader :blob @@ -26,7 +26,7 @@ module ActiveStorage private # Downloads the blob to a tempfile on disk. Yields the tempfile. def download_blob_to_tempfile(&block) #:doc: - blob.open tempdir: tempdir, &block + blob.open tmpdir: tmpdir, &block end # Executes a system command, capturing its binary output in a tempfile. Yields the tempfile. @@ -42,7 +42,7 @@ module ActiveStorage # end # end # - # The output tempfile is opened in the directory returned by #tempdir. + # The output tempfile is opened in the directory returned by #tmpdir. def draw(*argv) #:doc: open_tempfile do |file| instrument :preview, key: blob.key do @@ -54,7 +54,7 @@ module ActiveStorage end def open_tempfile - tempfile = Tempfile.open("ActiveStorage-", tempdir) + tempfile = Tempfile.open("ActiveStorage-", tmpdir) begin yield tempfile @@ -77,7 +77,7 @@ module ActiveStorage ActiveStorage.logger end - def tempdir #:doc: + def tmpdir #:doc: Dir.tmpdir end end diff --git a/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb b/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb index 69eb617d7b..6bf501a607 100644 --- a/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb +++ b/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb @@ -28,7 +28,7 @@ module ActiveStorage private def draw_first_page_from(file, &block) - # use 72 dpi to match thumbnail dimesions of the PDF + # use 72 dpi to match thumbnail dimensions of the PDF draw self.class.pdftoppm_path, "-singlefile", "-r", "72", "-png", file.path, &block end end diff --git a/activestorage/lib/active_storage/service.rb b/activestorage/lib/active_storage/service.rb index f915518f52..aac1e62e7f 100644 --- a/activestorage/lib/active_storage/service.rb +++ b/activestorage/lib/active_storage/service.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true require "active_storage/log_subscriber" +require "action_dispatch" +require "action_dispatch/http/content_disposition" module ActiveStorage # Abstract class serving as an interface for concrete services. @@ -60,10 +62,16 @@ module ActiveStorage # Upload the +io+ to the +key+ specified. If a +checksum+ is provided, the service will # ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError. - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, **options) raise NotImplementedError end + # Update metadata for the file identified by +key+ in the service. + # Override in subclasses only if the service needs to store specific + # metadata that has to be updated upon identification. + def update_metadata(key, **metadata) + end + # Return the content of the file at the +key+. def download(key) raise NotImplementedError @@ -74,6 +82,10 @@ module ActiveStorage raise NotImplementedError end + def open(*args, &block) + ActiveStorage::Downloader.new(self).open(*args, &block) + end + # Delete the file at the +key+. def delete(key) raise NotImplementedError @@ -122,7 +134,8 @@ module ActiveStorage end def content_disposition_with(type: "inline", filename:) - (type.to_s.presence_in(%w( attachment inline )) || "inline") + "; #{filename.parameters}" + disposition = (type.to_s.presence_in(%w( attachment inline )) || "inline") + ActionDispatch::Http::ContentDisposition.format(disposition: disposition, filename: filename.sanitized) end end end diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index 2867a4e441..993cc0e5f7 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -10,19 +10,17 @@ module ActiveStorage class Service::AzureStorageService < Service attr_reader :client, :blobs, :container, :signer - def initialize(storage_account_name:, storage_access_key:, container:) - @client = Azure::Storage::Client.create(storage_account_name: storage_account_name, storage_access_key: storage_access_key) + def initialize(storage_account_name:, storage_access_key:, container:, **options) + @client = Azure::Storage::Client.create(storage_account_name: storage_account_name, storage_access_key: storage_access_key, **options) @signer = Azure::Storage::Core::Auth::SharedAccessSignature.new(storage_account_name, storage_access_key) @blobs = client.blob_client @container = container end - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, **) instrument :upload, key: key, checksum: checksum do - begin - blobs.create_block_blob(container, key, io, content_md5: checksum) - rescue Azure::Core::Http::HTTPError - raise ActiveStorage::IntegrityError + handle_errors do + blobs.create_block_blob(container, key, IO.try_convert(io) || io, content_md5: checksum) end end end @@ -34,26 +32,29 @@ module ActiveStorage end else instrument :download, key: key do - _, io = blobs.get_blob(container, key) - io.force_encoding(Encoding::BINARY) + handle_errors do + _, io = blobs.get_blob(container, key) + io.force_encoding(Encoding::BINARY) + end end end end def download_chunk(key, range) instrument :download_chunk, key: key, range: range do - _, io = blobs.get_blob(container, key, start_range: range.begin, end_range: range.exclude_end? ? range.end - 1 : range.end) - io.force_encoding(Encoding::BINARY) + handle_errors do + _, io = blobs.get_blob(container, key, start_range: range.begin, end_range: range.exclude_end? ? range.end - 1 : range.end) + io.force_encoding(Encoding::BINARY) + end end end def delete(key) instrument :delete, key: key do - begin - blobs.delete_blob(container, key) - rescue Azure::Core::Http::HTTPError - # Ignore files already deleted - end + blobs.delete_blob(container, key) + rescue Azure::Core::Http::HTTPError => e + raise unless e.type == "BlobNotFound" + # Ignore files already deleted end end @@ -139,11 +140,26 @@ module ActiveStorage chunk_size = 5.megabytes offset = 0 + raise ActiveStorage::FileNotFoundError unless blob.present? + while offset < blob.properties[:content_length] _, chunk = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) yield chunk.force_encoding(Encoding::BINARY) offset += chunk_size end end + + def handle_errors + yield + rescue Azure::Core::Http::HTTPError => e + case e.type + when "BlobNotFound" + raise ActiveStorage::FileNotFoundError + when "Md5Mismatch" + raise ActiveStorage::IntegrityError + else + raise + end + end end end diff --git a/activestorage/lib/active_storage/service/configurator.rb b/activestorage/lib/active_storage/service/configurator.rb index 39951fd026..fa80c66c3b 100644 --- a/activestorage/lib/active_storage/service/configurator.rb +++ b/activestorage/lib/active_storage/service/configurator.rb @@ -26,7 +26,9 @@ module ActiveStorage def resolve(class_name) require "active_storage/service/#{class_name.to_s.underscore}_service" - ActiveStorage::Service.const_get(:"#{class_name}Service") + ActiveStorage::Service.const_get(:"#{class_name.camelize}Service") + rescue LoadError + raise "Missing service adapter for #{class_name.inspect}" end end end diff --git a/activestorage/lib/active_storage/service/disk_service.rb b/activestorage/lib/active_storage/service/disk_service.rb index 9f304b7e01..67892d43b2 100644 --- a/activestorage/lib/active_storage/service/disk_service.rb +++ b/activestorage/lib/active_storage/service/disk_service.rb @@ -15,25 +15,23 @@ module ActiveStorage @root = root end - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, **) instrument :upload, key: key, checksum: checksum do IO.copy_stream(io, make_path_for(key)) ensure_integrity_of(key, checksum) if checksum end end - def download(key) + def download(key, &block) if block_given? instrument :streaming_download, key: key do - File.open(path_for(key), "rb") do |file| - while data = file.read(5.megabytes) - yield data - end - end + stream key, &block end else instrument :download, key: key do File.binread path_for(key) + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError end end end @@ -44,16 +42,16 @@ module ActiveStorage file.seek range.begin file.read range.size end + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError end end def delete(key) instrument :delete, key: key do - begin - File.delete path_for(key) - rescue Errno::ENOENT - # Ignore files already deleted - end + File.delete path_for(key) + rescue Errno::ENOENT + # Ignore files already deleted end end @@ -75,17 +73,23 @@ module ActiveStorage def url(key, expires_in:, filename:, disposition:, content_type:) instrument :url, key: key do |payload| - verified_key_with_expiration = ActiveStorage.verifier.generate(key, expires_in: expires_in, purpose: :blob_key) - - generated_url = - url_helpers.rails_disk_service_url( - verified_key_with_expiration, - host: current_host, - filename: filename, - disposition: content_disposition_with(type: disposition, filename: filename), + content_disposition = content_disposition_with(type: disposition, filename: filename) + verified_key_with_expiration = ActiveStorage.verifier.generate( + { + key: key, + disposition: content_disposition, content_type: content_type - ) + }, + { expires_in: expires_in, + purpose: :blob_key } + ) + generated_url = url_helpers.rails_disk_service_url(verified_key_with_expiration, + host: current_host, + disposition: content_disposition, + content_type: content_type, + filename: filename + ) payload[:url] = generated_url generated_url @@ -122,6 +126,16 @@ module ActiveStorage end private + def stream(key) + File.open(path_for(key), "rb") do |file| + while data = file.read(5.megabytes) + yield data + end + end + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError + end + def folder_for(key) [ key[0..1], key[2..3] ].join("/") end diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb index fdde01d4a3..09abc613f3 100644 --- a/activestorage/lib/active_storage/service/gcs_service.rb +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -1,11 +1,7 @@ # frozen_string_literal: true gem "google-cloud-storage", "~> 1.11" - require "google/cloud/storage" -require "net/http" - -require "active_support/core_ext/object/to_query" module ActiveStorage # Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API @@ -15,19 +11,16 @@ module ActiveStorage @config = config end - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, content_type: nil, disposition: nil, filename: nil) instrument :upload, key: key, checksum: checksum do - begin - # The official GCS client library doesn't allow us to create a file with no Content-Type metadata. - # We need the file we create to have no Content-Type so we can control it via the response-content-type - # param in signed URLs. Workaround: let the GCS client create the file with an inferred - # Content-Type (usually "application/octet-stream") then clear it. - bucket.create_file(io, key, md5: checksum).update do |file| - file.content_type = nil - end - rescue Google::Cloud::InvalidArgumentError - raise ActiveStorage::IntegrityError - end + # GCS's signed URLs don't include params such as response-content-type response-content_disposition + # in the signature, which means an attacker can modify them and bypass our effort to force these to + # binary and attachment when the file's content type requires it. The only way to force them is to + # store them as object's metadata. + content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename + bucket.create_file(io, key, md5: checksum, content_type: content_type, content_disposition: content_disposition) + rescue Google::Cloud::InvalidArgumentError + raise ActiveStorage::IntegrityError end end @@ -39,6 +32,17 @@ module ActiveStorage else instrument :download, key: key do file_for(key).download.string + rescue Google::Cloud::NotFoundError + raise ActiveStorage::FileNotFoundError + end + end + end + + def update_metadata(key, content_type:, disposition: nil, filename: nil) + instrument :update_metadata, key: key, content_type: content_type, disposition: disposition do + file_for(key).update do |file| + file.content_type = content_type + file.content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename end end end @@ -46,22 +50,26 @@ module ActiveStorage def download_chunk(key, range) instrument :download_chunk, key: key, range: range do file_for(key).download(range: range).string + rescue Google::Cloud::NotFoundError + raise ActiveStorage::FileNotFoundError end end def delete(key) instrument :delete, key: key do - begin - file_for(key).delete - rescue Google::Cloud::NotFoundError - # Ignore files already deleted - end + file_for(key).delete + rescue Google::Cloud::NotFoundError + # Ignore files already deleted end end def delete_prefixed(prefix) instrument :delete_prefixed, prefix: prefix do - bucket.files(prefix: prefix).all(&:delete) + bucket.files(prefix: prefix).all do |file| + file.delete + rescue Google::Cloud::NotFoundError + # Ignore concurrently-deleted files + end end end @@ -114,6 +122,8 @@ module ActiveStorage chunk_size = 5.megabytes offset = 0 + raise ActiveStorage::FileNotFoundError unless file.present? + while offset < file.size yield file.download(range: offset..(offset + chunk_size - 1)).string offset += chunk_size @@ -121,7 +131,7 @@ module ActiveStorage end def bucket - @bucket ||= client.bucket(config.fetch(:bucket)) + @bucket ||= client.bucket(config.fetch(:bucket), skip_lookup: true) end def client diff --git a/activestorage/lib/active_storage/service/mirror_service.rb b/activestorage/lib/active_storage/service/mirror_service.rb index 6002ef5a00..aa41df304e 100644 --- a/activestorage/lib/active_storage/service/mirror_service.rb +++ b/activestorage/lib/active_storage/service/mirror_service.rb @@ -9,7 +9,7 @@ module ActiveStorage class Service::MirrorService < Service attr_reader :primary, :mirrors - delegate :download, :download_chunk, :exist?, :url, to: :primary + delegate :download, :download_chunk, :exist?, :url, :path_for, to: :primary # Stitch together from named services. def self.build(primary:, mirrors:, configurator:, **options) #:nodoc: @@ -24,9 +24,9 @@ module ActiveStorage # Upload the +io+ to the +key+ specified to all services. If a +checksum+ is provided, all services will # ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError. - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, **options) each_service.collect do |service| - service.upload key, io.tap(&:rewind), checksum: checksum + service.upload key, io.tap(&:rewind), checksum: checksum, **options end end diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index 0286e7ff21..bf94f3f49e 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -16,13 +16,11 @@ module ActiveStorage @upload_options = upload end - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, content_type: nil, **) instrument :upload, key: key, checksum: checksum do - begin - object_for(key).put(upload_options.merge(body: io, content_md5: checksum)) - rescue Aws::S3::Errors::BadDigest - raise ActiveStorage::IntegrityError - end + object_for(key).put(upload_options.merge(body: io, content_md5: checksum, content_type: content_type)) + rescue Aws::S3::Errors::BadDigest + raise ActiveStorage::IntegrityError end end @@ -34,6 +32,8 @@ module ActiveStorage else instrument :download, key: key do object_for(key).get.body.string.force_encoding(Encoding::BINARY) + rescue Aws::S3::Errors::NoSuchKey + raise ActiveStorage::FileNotFoundError end end end @@ -41,6 +41,8 @@ module ActiveStorage def download_chunk(key, range) instrument :download_chunk, key: key, range: range do object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.read.force_encoding(Encoding::BINARY) + rescue Aws::S3::Errors::NoSuchKey + raise ActiveStorage::FileNotFoundError end end @@ -103,6 +105,8 @@ module ActiveStorage chunk_size = 5.megabytes offset = 0 + raise ActiveStorage::FileNotFoundError unless object.exists? + while offset < object.content_length yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.read.force_encoding(Encoding::BINARY) offset += chunk_size diff --git a/activestorage/lib/active_storage/transformers/image_processing_transformer.rb b/activestorage/lib/active_storage/transformers/image_processing_transformer.rb new file mode 100644 index 0000000000..506150576c --- /dev/null +++ b/activestorage/lib/active_storage/transformers/image_processing_transformer.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require "image_processing" + +module ActiveStorage + module Transformers + class ImageProcessingTransformer < Transformer + private + def process(file, format:) + processor. + source(file). + loader(page: 0). + convert(format). + apply(operations). + call + end + + def processor + ImageProcessing.const_get(ActiveStorage.variant_processor.to_s.camelize) + end + + def operations + transformations.each_with_object([]) do |(name, argument), list| + if name.to_s == "combine_options" + ActiveSupport::Deprecation.warn <<~WARNING.squish + Active Storage's ImageProcessing transformer doesn't support :combine_options, + as it always generates a single ImageMagick command. Passing :combine_options will + not be supported in Rails 6.1. + WARNING + + list.concat argument.keep_if { |key, value| value.present? }.to_a + elsif argument.present? + list << [ name, argument ] + end + end + end + end + end +end diff --git a/activestorage/lib/active_storage/transformers/mini_magick_transformer.rb b/activestorage/lib/active_storage/transformers/mini_magick_transformer.rb new file mode 100644 index 0000000000..e8e99cea9e --- /dev/null +++ b/activestorage/lib/active_storage/transformers/mini_magick_transformer.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require "mini_magick" + +module ActiveStorage + module Transformers + class MiniMagickTransformer < Transformer + private + def process(file, format:) + image = MiniMagick::Image.new(file.path, file) + + transformations.each do |name, argument_or_subtransformations| + image.mogrify do |command| + if name.to_s == "combine_options" + argument_or_subtransformations.each do |subtransformation_name, subtransformation_argument| + pass_transform_argument(command, subtransformation_name, subtransformation_argument) + end + else + pass_transform_argument(command, name, argument_or_subtransformations) + end + end + end + + image.format(format) if format + + image.tempfile.tap(&:open) + end + + def pass_transform_argument(command, method, argument) + if argument == true + command.public_send(method) + elsif argument.present? + command.public_send(method, argument) + end + end + end + end +end diff --git a/activestorage/lib/active_storage/transformers/transformer.rb b/activestorage/lib/active_storage/transformers/transformer.rb new file mode 100644 index 0000000000..2e21201004 --- /dev/null +++ b/activestorage/lib/active_storage/transformers/transformer.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module ActiveStorage + module Transformers + # A Transformer applies a set of transformations to an image. + # + # The following concrete subclasses are included in Active Storage: + # + # * ActiveStorage::Transformers::ImageProcessingTransformer: + # backed by ImageProcessing, a common interface for MiniMagick and ruby-vips + # + # * ActiveStorage::Transformers::MiniMagickTransformer: + # backed by MiniMagick, a wrapper around the ImageMagick CLI + class Transformer + attr_reader :transformations + + def initialize(transformations) + @transformations = transformations + end + + # Applies the transformations to the source image in +file+, producing a target image in the + # specified +format+. Yields an open Tempfile containing the target image. Closes and unlinks + # the output tempfile after yielding to the given block. Returns the result of the block. + def transform(file, format:) + output = process(file, format: format) + + begin + yield output + ensure + output.close! + end + end + + private + # Returns an open Tempfile containing a transformed image in the given +format+. + # All subclasses implement this method. + def process(file, format:) #:doc: + raise NotImplementedError + end + end + end +end diff --git a/activestorage/lib/tasks/activestorage.rake b/activestorage/lib/tasks/activestorage.rake index ac254d717f..6b0469636c 100644 --- a/activestorage/lib/tasks/activestorage.rake +++ b/activestorage/lib/tasks/activestorage.rake @@ -12,4 +12,11 @@ namespace :active_storage do Rake::Task["app:active_storage:install:migrations"].invoke end end + + # desc "Copy over the migrations needed to the application upgrading" + task update: :environment do + ENV["MIGRATIONS_PATH"] = "db/update_migrate" + + Rake::Task["active_storage:install"].invoke + end end |