diff options
Diffstat (limited to 'activestorage/lib/active_storage')
33 files changed, 873 insertions, 294 deletions
diff --git a/activestorage/lib/active_storage/analyzer.rb b/activestorage/lib/active_storage/analyzer.rb index 7c4168c1a0..26414ffbc2 100644 --- a/activestorage/lib/active_storage/analyzer.rb +++ b/activestorage/lib/active_storage/analyzer.rb @@ -1,13 +1,9 @@ # frozen_string_literal: true -require "active_storage/downloading" - module ActiveStorage # This is an abstract base class for analyzers, which extract metadata from blobs. See # ActiveStorage::Analyzer::ImageAnalyzer for an example of a concrete subclass. class Analyzer - include Downloading - attr_reader :blob # Implement this method in a concrete subclass. Have it return true when given a blob from which @@ -26,8 +22,17 @@ module ActiveStorage end private + # Downloads the blob to a tempfile on disk. Yields the tempfile. + def download_blob_to_tempfile(&block) #:doc: + blob.open tmpdir: tmpdir, &block + end + def logger #:doc: ActiveStorage.logger end + + def tmpdir #:doc: + Dir.tmpdir + end end end diff --git a/activestorage/lib/active_storage/analyzer/video_analyzer.rb b/activestorage/lib/active_storage/analyzer/video_analyzer.rb index e31bdb0edb..e56c97f4f5 100644 --- a/activestorage/lib/active_storage/analyzer/video_analyzer.rb +++ b/activestorage/lib/active_storage/analyzer/video_analyzer.rb @@ -11,12 +11,12 @@ module ActiveStorage # # Example: # - # ActiveStorage::VideoAnalyzer.new(blob).metadata + # ActiveStorage::Analyzer::VideoAnalyzer.new(blob).metadata # # => { width: 640.0, height: 480.0, duration: 5.0, angle: 0, display_aspect_ratio: [4, 3] } # # When a video's angle is 90 or 270 degrees, its width and height are automatically swapped for convenience. # - # This analyzer requires the {ffmpeg}[https://www.ffmpeg.org] system library, which is not provided by Rails. + # This analyzer requires the {FFmpeg}[https://www.ffmpeg.org] system library, which is not provided by Rails. class Analyzer::VideoAnalyzer < Analyzer def self.accept?(blob) blob.video? @@ -107,7 +107,7 @@ module ActiveStorage JSON.parse(output.read) end rescue Errno::ENOENT - logger.info "Skipping video analysis because ffmpeg isn't installed" + logger.info "Skipping video analysis because FFmpeg isn't installed" {} end diff --git a/activestorage/lib/active_storage/attached.rb b/activestorage/lib/active_storage/attached.rb index c08fd56652..b540f85fbe 100644 --- a/activestorage/lib/active_storage/attached.rb +++ b/activestorage/lib/active_storage/attached.rb @@ -1,40 +1,25 @@ # frozen_string_literal: true -require "action_dispatch" -require "action_dispatch/http/upload" require "active_support/core_ext/module/delegation" module ActiveStorage # Abstract base class for the concrete ActiveStorage::Attached::One and ActiveStorage::Attached::Many # classes that both provide proxy access to the blob association for a record. class Attached - attr_reader :name, :record, :dependent + attr_reader :name, :record - def initialize(name, record, dependent:) - @name, @record, @dependent = name, record, dependent + def initialize(name, record) + @name, @record = name, record end private - def create_blob_from(attachable) - case attachable - when ActiveStorage::Blob - attachable - when ActionDispatch::Http::UploadedFile, Rack::Test::UploadedFile - ActiveStorage::Blob.create_after_upload! \ - io: attachable.open, - filename: attachable.original_filename, - content_type: attachable.content_type - when Hash - ActiveStorage::Blob.create_after_upload!(attachable) - when String - ActiveStorage::Blob.find_signed(attachable) - else - nil - end + def change + record.attachment_changes[name] end end end +require "active_storage/attached/model" require "active_storage/attached/one" require "active_storage/attached/many" -require "active_storage/attached/macros" +require "active_storage/attached/changes" diff --git a/activestorage/lib/active_storage/attached/changes.rb b/activestorage/lib/active_storage/attached/changes.rb new file mode 100644 index 0000000000..1db3906a63 --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module ActiveStorage + module Attached::Changes #:nodoc: + extend ActiveSupport::Autoload + + eager_autoload do + autoload :CreateOne + autoload :CreateMany + autoload :CreateOneOfMany + + autoload :DeleteOne + autoload :DeleteMany + end + end +end diff --git a/activestorage/lib/active_storage/attached/changes/create_many.rb b/activestorage/lib/active_storage/attached/changes/create_many.rb new file mode 100644 index 0000000000..a7a8553e0f --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes/create_many.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module ActiveStorage + class Attached::Changes::CreateMany #:nodoc: + attr_reader :name, :record, :attachables + + def initialize(name, record, attachables) + @name, @record, @attachables = name, record, Array(attachables) + end + + def attachments + @attachments ||= subchanges.collect(&:attachment) + end + + def blobs + @blobs ||= subchanges.collect(&:blob) + end + + def upload + subchanges.each(&:upload) + end + + def save + assign_associated_attachments + reset_associated_blobs + end + + private + def subchanges + @subchanges ||= attachables.collect { |attachable| build_subchange_from(attachable) } + end + + def build_subchange_from(attachable) + ActiveStorage::Attached::Changes::CreateOneOfMany.new(name, record, attachable) + end + + + def assign_associated_attachments + record.public_send("#{name}_attachments=", attachments) + end + + def reset_associated_blobs + record.public_send("#{name}_blobs").reset + end + end +end diff --git a/activestorage/lib/active_storage/attached/changes/create_one.rb b/activestorage/lib/active_storage/attached/changes/create_one.rb new file mode 100644 index 0000000000..89cccfb58a --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes/create_one.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require "action_dispatch" +require "action_dispatch/http/upload" + +module ActiveStorage + class Attached::Changes::CreateOne #:nodoc: + attr_reader :name, :record, :attachable + + def initialize(name, record, attachable) + @name, @record, @attachable = name, record, attachable + end + + def attachment + @attachment ||= find_or_build_attachment + end + + def blob + @blob ||= find_or_build_blob + end + + def upload + case attachable + when ActionDispatch::Http::UploadedFile, Rack::Test::UploadedFile + blob.upload_without_unfurling(attachable.open) + when Hash + blob.upload_without_unfurling(attachable.fetch(:io)) + end + end + + def save + record.public_send("#{name}_attachment=", attachment) + record.public_send("#{name}_blob=", blob) + end + + private + def find_or_build_attachment + find_attachment || build_attachment + end + + def find_attachment + if record.public_send("#{name}_blob") == blob + record.public_send("#{name}_attachment") + end + end + + def build_attachment + ActiveStorage::Attachment.new(record: record, name: name, blob: blob) + end + + def find_or_build_blob + case attachable + when ActiveStorage::Blob + attachable + when ActionDispatch::Http::UploadedFile, Rack::Test::UploadedFile + ActiveStorage::Blob.build_after_unfurling \ + io: attachable.open, + filename: attachable.original_filename, + content_type: attachable.content_type + when Hash + ActiveStorage::Blob.build_after_unfurling(attachable) + when String + ActiveStorage::Blob.find_signed(attachable) + else + raise ArgumentError, "Could not find or build blob: expected attachable, got #{attachable.inspect}" + end + end + end +end diff --git a/activestorage/lib/active_storage/attached/changes/create_one_of_many.rb b/activestorage/lib/active_storage/attached/changes/create_one_of_many.rb new file mode 100644 index 0000000000..7268e87316 --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes/create_one_of_many.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module ActiveStorage + class Attached::Changes::CreateOneOfMany < Attached::Changes::CreateOne #:nodoc: + private + def find_attachment + record.public_send("#{name}_attachments").detect { |attachment| attachment.blob_id == blob.id } + end + end +end diff --git a/activestorage/lib/active_storage/attached/changes/delete_many.rb b/activestorage/lib/active_storage/attached/changes/delete_many.rb new file mode 100644 index 0000000000..6cbd1158dc --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes/delete_many.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module ActiveStorage + class Attached::Changes::DeleteMany #:nodoc: + attr_reader :name, :record + + def initialize(name, record) + @name, @record = name, record + end + + def attachments + ActiveStorage::Attachment.none + end + + def blobs + ActiveStorage::Blob.none + end + + def save + record.public_send("#{name}_attachments=", []) + end + end +end diff --git a/activestorage/lib/active_storage/attached/changes/delete_one.rb b/activestorage/lib/active_storage/attached/changes/delete_one.rb new file mode 100644 index 0000000000..2f7d356613 --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes/delete_one.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module ActiveStorage + class Attached::Changes::DeleteOne #:nodoc: + attr_reader :name, :record + + def initialize(name, record) + @name, @record = name, record + end + + def attachment + nil + end + + def save + record.public_send("#{name}_attachment=", nil) + end + end +end diff --git a/activestorage/lib/active_storage/attached/macros.rb b/activestorage/lib/active_storage/attached/macros.rb deleted file mode 100644 index 819f00cc06..0000000000 --- a/activestorage/lib/active_storage/attached/macros.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: true - -module ActiveStorage - # Provides the class-level DSL for declaring that an Active Record model has attached blobs. - module Attached::Macros - # Specifies the relation between a single attachment and the model. - # - # class User < ActiveRecord::Base - # has_one_attached :avatar - # end - # - # There is no column defined on the model side, Active Storage takes - # care of the mapping between your records and the attachment. - # - # To avoid N+1 queries, you can include the attached blobs in your query like so: - # - # User.with_attached_avatar - # - # Under the covers, this relationship is implemented as a +has_one+ association to a - # ActiveStorage::Attachment record and a +has_one-through+ association to a - # ActiveStorage::Blob record. These associations are available as +avatar_attachment+ - # and +avatar_blob+. But you shouldn't need to work with these associations directly in - # most circumstances. - # - # The system has been designed to having you go through the ActiveStorage::Attached::One - # proxy that provides the dynamic proxy to the associations and factory methods, like +attach+. - # - # If the +:dependent+ option isn't set, the attachment will be purged - # (i.e. destroyed) whenever the record is destroyed. - def has_one_attached(name, dependent: :purge_later) - class_eval <<-CODE, __FILE__, __LINE__ + 1 - def #{name} - @active_storage_attached_#{name} ||= ActiveStorage::Attached::One.new("#{name}", self, dependent: #{dependent == :purge_later ? ":purge_later" : "false"}) - end - - def #{name}=(attachable) - #{name}.attach(attachable) - end - CODE - - has_one :"#{name}_attachment", -> { where(name: name) }, class_name: "ActiveStorage::Attachment", as: :record, inverse_of: :record, dependent: false - has_one :"#{name}_blob", through: :"#{name}_attachment", class_name: "ActiveStorage::Blob", source: :blob - - scope :"with_attached_#{name}", -> { includes("#{name}_attachment": :blob) } - - if dependent == :purge_later - after_destroy_commit { public_send(name).purge_later } - else - before_destroy { public_send(name).detach } - end - end - - # Specifies the relation between multiple attachments and the model. - # - # class Gallery < ActiveRecord::Base - # has_many_attached :photos - # end - # - # There are no columns defined on the model side, Active Storage takes - # care of the mapping between your records and the attachments. - # - # To avoid N+1 queries, you can include the attached blobs in your query like so: - # - # Gallery.where(user: Current.user).with_attached_photos - # - # Under the covers, this relationship is implemented as a +has_many+ association to a - # ActiveStorage::Attachment record and a +has_many-through+ association to a - # ActiveStorage::Blob record. These associations are available as +photos_attachments+ - # and +photos_blobs+. But you shouldn't need to work with these associations directly in - # most circumstances. - # - # The system has been designed to having you go through the ActiveStorage::Attached::Many - # proxy that provides the dynamic proxy to the associations and factory methods, like +#attach+. - # - # If the +:dependent+ option isn't set, all the attachments will be purged - # (i.e. destroyed) whenever the record is destroyed. - def has_many_attached(name, dependent: :purge_later) - class_eval <<-CODE, __FILE__, __LINE__ + 1 - def #{name} - @active_storage_attached_#{name} ||= ActiveStorage::Attached::Many.new("#{name}", self, dependent: #{dependent == :purge_later ? ":purge_later" : "false"}) - end - - def #{name}=(attachables) - #{name}.attach(attachables) - end - CODE - - has_many :"#{name}_attachments", -> { where(name: name) }, as: :record, class_name: "ActiveStorage::Attachment", inverse_of: :record, dependent: false do - def purge - each(&:purge) - reset - end - - def purge_later - each(&:purge_later) - reset - end - end - has_many :"#{name}_blobs", through: :"#{name}_attachments", class_name: "ActiveStorage::Blob", source: :blob - - scope :"with_attached_#{name}", -> { includes("#{name}_attachments": :blob) } - - if dependent == :purge_later - after_destroy_commit { public_send(name).purge_later } - else - before_destroy { public_send(name).detach } - end - end - end -end diff --git a/activestorage/lib/active_storage/attached/many.rb b/activestorage/lib/active_storage/attached/many.rb index d61acb6fad..25f88284df 100644 --- a/activestorage/lib/active_storage/attached/many.rb +++ b/activestorage/lib/active_storage/attached/many.rb @@ -9,22 +9,29 @@ module ActiveStorage # # All methods called on this proxy object that aren't listed here will automatically be delegated to +attachments+. def attachments - record.public_send("#{name}_attachments") + change.present? ? change.attachments : record.public_send("#{name}_attachments") end - # Associates one or several attachments with the current record, saving them to the database. + # Returns all attached blobs. + def blobs + change.present? ? change.blobs : record.public_send("#{name}_blobs") + end + + # Attaches one or more +attachables+ to the record. + # + # If the record is persisted and unchanged, the attachments are saved to + # the database immediately. Otherwise, they'll be saved to the DB when the + # record is next saved. # # document.images.attach(params[:images]) # Array of ActionDispatch::Http::UploadedFile objects # document.images.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload # document.images.attach(io: File.open("/path/to/racecar.jpg"), filename: "racecar.jpg", content_type: "image/jpg") # document.images.attach([ first_blob, second_blob ]) def attach(*attachables) - attachables.flatten.collect do |attachable| - if record.new_record? - attachments.build(record: record, blob: create_blob_from(attachable)) - else - attachments.create!(record: record, blob: create_blob_from(attachable)) - end + if record.persisted? && !record.changed? + record.update(name => blobs + attachables.flatten) + else + record.public_send("#{name}=", blobs + attachables.flatten) end end @@ -41,7 +48,7 @@ module ActiveStorage # Deletes associated attachments without purging them, leaving their respective blobs in place. def detach - attachments.destroy_all if attached? + attachments.delete_all if attached? end ## @@ -50,7 +57,6 @@ module ActiveStorage # Directly purges each associated attachment (i.e. destroys the blobs and # attachments and deletes the files on the service). - ## # :method: purge_later # diff --git a/activestorage/lib/active_storage/attached/model.rb b/activestorage/lib/active_storage/attached/model.rb new file mode 100644 index 0000000000..ae7f0685f2 --- /dev/null +++ b/activestorage/lib/active_storage/attached/model.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module ActiveStorage + # Provides the class-level DSL for declaring an Active Record model's attachments. + module Attached::Model + extend ActiveSupport::Concern + + class_methods do + # Specifies the relation between a single attachment and the model. + # + # class User < ActiveRecord::Base + # has_one_attached :avatar + # end + # + # There is no column defined on the model side, Active Storage takes + # care of the mapping between your records and the attachment. + # + # To avoid N+1 queries, you can include the attached blobs in your query like so: + # + # User.with_attached_avatar + # + # Under the covers, this relationship is implemented as a +has_one+ association to a + # ActiveStorage::Attachment record and a +has_one-through+ association to a + # ActiveStorage::Blob record. These associations are available as +avatar_attachment+ + # and +avatar_blob+. But you shouldn't need to work with these associations directly in + # most circumstances. + # + # The system has been designed to having you go through the ActiveStorage::Attached::One + # proxy that provides the dynamic proxy to the associations and factory methods, like +attach+. + # + # If the +:dependent+ option isn't set, the attachment will be purged + # (i.e. destroyed) whenever the record is destroyed. + def has_one_attached(name, dependent: :purge_later) + generated_association_methods.class_eval <<-CODE, __FILE__, __LINE__ + 1 + def #{name} + @active_storage_attached_#{name} ||= ActiveStorage::Attached::One.new("#{name}", self) + end + + def #{name}=(attachable) + attachment_changes["#{name}"] = + if attachable.nil? + ActiveStorage::Attached::Changes::DeleteOne.new("#{name}", self) + else + ActiveStorage::Attached::Changes::CreateOne.new("#{name}", self, attachable) + end + end + CODE + + has_one :"#{name}_attachment", -> { where(name: name) }, class_name: "ActiveStorage::Attachment", as: :record, inverse_of: :record, dependent: :destroy + has_one :"#{name}_blob", through: :"#{name}_attachment", class_name: "ActiveStorage::Blob", source: :blob + + scope :"with_attached_#{name}", -> { includes("#{name}_attachment": :blob) } + + after_save { attachment_changes[name.to_s]&.save } + + after_commit(on: %i[ create update ]) { attachment_changes.delete(name.to_s).try(:upload) } + + ActiveRecord::Reflection.add_attachment_reflection( + self, + name, + ActiveRecord::Reflection.create(:has_one_attached, name, nil, { dependent: dependent }, self) + ) + end + + # Specifies the relation between multiple attachments and the model. + # + # class Gallery < ActiveRecord::Base + # has_many_attached :photos + # end + # + # There are no columns defined on the model side, Active Storage takes + # care of the mapping between your records and the attachments. + # + # To avoid N+1 queries, you can include the attached blobs in your query like so: + # + # Gallery.where(user: Current.user).with_attached_photos + # + # Under the covers, this relationship is implemented as a +has_many+ association to a + # ActiveStorage::Attachment record and a +has_many-through+ association to a + # ActiveStorage::Blob record. These associations are available as +photos_attachments+ + # and +photos_blobs+. But you shouldn't need to work with these associations directly in + # most circumstances. + # + # The system has been designed to having you go through the ActiveStorage::Attached::Many + # proxy that provides the dynamic proxy to the associations and factory methods, like +#attach+. + # + # If the +:dependent+ option isn't set, all the attachments will be purged + # (i.e. destroyed) whenever the record is destroyed. + def has_many_attached(name, dependent: :purge_later) + generated_association_methods.class_eval <<-CODE, __FILE__, __LINE__ + 1 + def #{name} + @active_storage_attached_#{name} ||= ActiveStorage::Attached::Many.new("#{name}", self) + end + + def #{name}=(attachables) + attachment_changes["#{name}"] = + if attachables.nil? || Array(attachables).none? + ActiveStorage::Attached::Changes::DeleteMany.new("#{name}", self) + else + ActiveStorage::Attached::Changes::CreateMany.new("#{name}", self, attachables) + end + end + CODE + + has_many :"#{name}_attachments", -> { where(name: name) }, as: :record, class_name: "ActiveStorage::Attachment", inverse_of: :record, dependent: :destroy do + def purge + each(&:purge) + reset + end + + def purge_later + each(&:purge_later) + reset + end + end + has_many :"#{name}_blobs", through: :"#{name}_attachments", class_name: "ActiveStorage::Blob", source: :blob + + scope :"with_attached_#{name}", -> { includes("#{name}_attachments": :blob) } + + after_save { attachment_changes[name.to_s]&.save } + + after_commit(on: %i[ create update ]) { attachment_changes.delete(name.to_s).try(:upload) } + + ActiveRecord::Reflection.add_attachment_reflection( + self, + name, + ActiveRecord::Reflection.create(:has_many_attached, name, nil, { dependent: dependent }, self) + ) + end + end + + def attachment_changes #:nodoc: + @attachment_changes ||= {} + end + + def reload(*) #:nodoc: + super.tap { @attachment_changes = nil } + end + end +end diff --git a/activestorage/lib/active_storage/attached/one.rb b/activestorage/lib/active_storage/attached/one.rb index a582ed0137..c039226fcd 100644 --- a/activestorage/lib/active_storage/attached/one.rb +++ b/activestorage/lib/active_storage/attached/one.rb @@ -10,26 +10,28 @@ module ActiveStorage # You don't have to call this method to access the attachment's methods as # they are all available at the model level. def attachment - record.public_send("#{name}_attachment") + change.present? ? change.attachment : record.public_send("#{name}_attachment") end - # Associates a given attachment with the current record, saving it to the database. + def blank? + !attached? + end + + # Attaches an +attachable+ to the record. + # + # If the record is persisted and unchanged, the attachment is saved to + # the database immediately. Otherwise, it'll be saved to the DB when the + # record is next saved. # # person.avatar.attach(params[:avatar]) # ActionDispatch::Http::UploadedFile object # person.avatar.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload # person.avatar.attach(io: File.open("/path/to/face.jpg"), filename: "face.jpg", content_type: "image/jpg") # person.avatar.attach(avatar_blob) # ActiveStorage::Blob object def attach(attachable) - blob_was = blob if attached? - blob = create_blob_from(attachable) - - unless blob == blob_was - transaction do - detach - write_attachment build_attachment(blob: blob) - end - - blob_was.purge_later if blob_was && dependent == :purge_later + if record.persisted? && !record.changed? + record.update(name => attachable) + else + record.public_send("#{name}=", attachable) end end @@ -47,7 +49,7 @@ module ActiveStorage # Deletes the attachment without purging it, leaving its blob in place. def detach if attached? - attachment.destroy + attachment.delete write_attachment nil end end @@ -65,16 +67,11 @@ module ActiveStorage def purge_later if attached? attachment.purge_later + write_attachment nil end end private - delegate :transaction, to: :record - - def build_attachment(blob:) - ActiveStorage::Attachment.new(record: record, name: name, blob: blob) - end - def write_attachment(attachment) record.public_send("#{name}_attachment=", attachment) end diff --git a/activestorage/lib/active_storage/downloader.rb b/activestorage/lib/active_storage/downloader.rb new file mode 100644 index 0000000000..4d7e832af5 --- /dev/null +++ b/activestorage/lib/active_storage/downloader.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module ActiveStorage + class Downloader #:nodoc: + attr_reader :service + + def initialize(service) + @service = service + end + + def open(key, checksum:, name: "ActiveStorage-", tmpdir: nil) + open_tempfile(name, tmpdir) do |file| + download key, file + verify_integrity_of file, checksum: checksum + yield file + end + end + + private + def open_tempfile(name, tmpdir = nil) + file = Tempfile.open(name, tmpdir) + + begin + yield file + ensure + file.close! + end + end + + def download(key, file) + file.binmode + service.download(key) { |chunk| file.write(chunk) } + file.flush + file.rewind + end + + def verify_integrity_of(file, checksum:) + unless Digest::MD5.file(file).base64digest == checksum + raise ActiveStorage::IntegrityError + end + end + end +end diff --git a/activestorage/lib/active_storage/downloading.rb b/activestorage/lib/active_storage/downloading.rb index f2a1fffdcb..df820bc088 100644 --- a/activestorage/lib/active_storage/downloading.rb +++ b/activestorage/lib/active_storage/downloading.rb @@ -1,9 +1,17 @@ # frozen_string_literal: true require "tmpdir" +require "active_support/core_ext/string/filters" module ActiveStorage module Downloading + def self.included(klass) + ActiveSupport::Deprecation.warn <<~MESSAGE.squish, caller_locations(2) + ActiveStorage::Downloading is deprecated and will be removed in Active Storage 6.1. + Use ActiveStorage::Blob#open instead. + MESSAGE + end + private # Opens a new tempfile in #tempdir and copies blob data into it. Yields the tempfile. def download_blob_to_tempfile #:doc: @@ -27,6 +35,7 @@ module ActiveStorage def download_blob_to(file) #:doc: file.binmode blob.download { |chunk| file.write(chunk) } + file.flush file.rewind end diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb index 1385e2aa84..fc75a8f816 100644 --- a/activestorage/lib/active_storage/engine.rb +++ b/activestorage/lib/active_storage/engine.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true require "rails" +require "action_controller/railtie" +require "active_job/railtie" +require "active_record/railtie" + require "active_storage" require "active_storage/previewer/poppler_pdf_previewer" @@ -10,6 +14,8 @@ require "active_storage/previewer/video_previewer" require "active_storage/analyzer/image_analyzer" require "active_storage/analyzer/video_analyzer" +require "active_storage/reflection" + module ActiveStorage class Engine < Rails::Engine # :nodoc: isolate_namespace ActiveStorage @@ -18,12 +24,15 @@ module ActiveStorage config.active_storage.previewers = [ ActiveStorage::Previewer::PopplerPDFPreviewer, ActiveStorage::Previewer::MuPDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ] config.active_storage.analyzers = [ ActiveStorage::Analyzer::ImageAnalyzer, ActiveStorage::Analyzer::VideoAnalyzer ] config.active_storage.paths = ActiveSupport::OrderedOptions.new + config.active_storage.queues = ActiveSupport::OrderedOptions.new config.active_storage.variable_content_types = %w( image/png image/gif image/jpg image/jpeg + image/pjpeg + image/tiff image/vnd.adobe.photoshop image/vnd.microsoft.icon ) @@ -37,20 +46,37 @@ module ActiveStorage text/xml application/xml application/xhtml+xml + application/mathml+xml + text/cache-manifest + ) + + config.active_storage.content_types_allowed_inline = %w( + image/png + image/gif + image/jpg + image/jpeg + image/tiff + image/vnd.adobe.photoshop + image/vnd.microsoft.icon + application/pdf ) config.eager_load_namespaces << ActiveStorage initializer "active_storage.configs" do config.after_initialize do |app| - ActiveStorage.logger = app.config.active_storage.logger || Rails.logger - ActiveStorage.queue = app.config.active_storage.queue - ActiveStorage.previewers = app.config.active_storage.previewers || [] - ActiveStorage.analyzers = app.config.active_storage.analyzers || [] - ActiveStorage.paths = app.config.active_storage.paths || {} + ActiveStorage.logger = app.config.active_storage.logger || Rails.logger + ActiveStorage.variant_processor = app.config.active_storage.variant_processor || :mini_magick + ActiveStorage.previewers = app.config.active_storage.previewers || [] + ActiveStorage.analyzers = app.config.active_storage.analyzers || [] + ActiveStorage.paths = app.config.active_storage.paths || {} + ActiveStorage.routes_prefix = app.config.active_storage.routes_prefix || "/rails/active_storage" ActiveStorage.variable_content_types = app.config.active_storage.variable_content_types || [] ActiveStorage.content_types_to_serve_as_binary = app.config.active_storage.content_types_to_serve_as_binary || [] + ActiveStorage.service_urls_expire_in = app.config.active_storage.service_urls_expire_in || 5.minutes + ActiveStorage.content_types_allowed_inline = app.config.active_storage.content_types_allowed_inline || [] + ActiveStorage.binary_content_type = app.config.active_storage.binary_content_type || "application/octet-stream" end end @@ -58,7 +84,7 @@ module ActiveStorage require "active_storage/attached" ActiveSupport.on_load(:active_record) do - extend ActiveStorage::Attached::Macros + include ActiveStorage::Attached::Model end end @@ -94,5 +120,26 @@ module ActiveStorage end end end + + initializer "active_storage.queues" do + config.after_initialize do |app| + if queue = app.config.active_storage.queue + ActiveSupport::Deprecation.warn \ + "config.active_storage.queue is deprecated and will be removed in Rails 6.1. " \ + "Set config.active_storage.queues.purge and config.active_storage.queues.analysis instead." + + ActiveStorage.queues = { purge: queue, analysis: queue } + else + ActiveStorage.queues = app.config.active_storage.queues || {} + end + end + end + + initializer "active_storage.reflection" do + ActiveSupport.on_load(:active_record) do + include Reflection::ActiveRecordExtensions + ActiveRecord::Reflection.singleton_class.prepend(Reflection::ReflectionExtension) + end + end end end diff --git a/activestorage/lib/active_storage/errors.rb b/activestorage/lib/active_storage/errors.rb index f099b13f5b..6475c1d076 100644 --- a/activestorage/lib/active_storage/errors.rb +++ b/activestorage/lib/active_storage/errors.rb @@ -1,7 +1,26 @@ # frozen_string_literal: true module ActiveStorage - class InvariableError < StandardError; end - class UnpreviewableError < StandardError; end - class UnrepresentableError < StandardError; end + # Generic base class for all Active Storage exceptions. + class Error < StandardError; end + + # Raised when ActiveStorage::Blob#variant is called on a blob that isn't variable. + # Use ActiveStorage::Blob#variable? to determine whether a blob is variable. + class InvariableError < Error; end + + # Raised when ActiveStorage::Blob#preview is called on a blob that isn't previewable. + # Use ActiveStorage::Blob#previewable? to determine whether a blob is previewable. + class UnpreviewableError < Error; end + + # Raised when ActiveStorage::Blob#representation is called on a blob that isn't representable. + # Use ActiveStorage::Blob#representable? to determine whether a blob is representable. + class UnrepresentableError < Error; end + + # Raised when uploaded or downloaded data does not match a precomputed checksum. + # Indicates that a network error or a software bug caused data corruption. + class IntegrityError < Error; end + + # Raised when ActiveStorage::Blob#download is called on a blob where the + # backing file is no longer present in its service. + class FileNotFoundError < Error; end end diff --git a/activestorage/lib/active_storage/gem_version.rb b/activestorage/lib/active_storage/gem_version.rb index 492620731b..d8f8577cc3 100644 --- a/activestorage/lib/active_storage/gem_version.rb +++ b/activestorage/lib/active_storage/gem_version.rb @@ -10,7 +10,7 @@ module ActiveStorage MAJOR = 6 MINOR = 0 TINY = 0 - PRE = "alpha" + PRE = "beta3" STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".") end diff --git a/activestorage/lib/active_storage/log_subscriber.rb b/activestorage/lib/active_storage/log_subscriber.rb index a4e148c1a5..6c0b4c30e7 100644 --- a/activestorage/lib/active_storage/log_subscriber.rb +++ b/activestorage/lib/active_storage/log_subscriber.rb @@ -14,6 +14,8 @@ module ActiveStorage info event, color("Downloaded file from key: #{key_in(event)}", BLUE) end + alias_method :service_streaming_download, :service_download + def service_delete(event) info event, color("Deleted file from key: #{key_in(event)}", RED) end diff --git a/activestorage/lib/active_storage/previewer.rb b/activestorage/lib/active_storage/previewer.rb index cf19987d72..af6bcadd4c 100644 --- a/activestorage/lib/active_storage/previewer.rb +++ b/activestorage/lib/active_storage/previewer.rb @@ -1,14 +1,10 @@ # frozen_string_literal: true -require "active_storage/downloading" - module ActiveStorage # This is an abstract base class for previewers, which generate images from blobs. See - # ActiveStorage::Previewer::PDFPreviewer and ActiveStorage::Previewer::VideoPreviewer for examples of - # concrete subclasses. + # ActiveStorage::Previewer::MuPDFPreviewer and ActiveStorage::Previewer::VideoPreviewer for + # examples of concrete subclasses. class Previewer - include Downloading - attr_reader :blob # Implement this method in a concrete subclass. Have it return true when given a blob from which @@ -28,9 +24,14 @@ module ActiveStorage end private + # Downloads the blob to a tempfile on disk. Yields the tempfile. + def download_blob_to_tempfile(&block) #:doc: + blob.open tmpdir: tmpdir, &block + end + # Executes a system command, capturing its binary output in a tempfile. Yields the tempfile. # - # Use this method to shell out to a system library (e.g. mupdf or ffmpeg) for preview image + # Use this method to shell out to a system library (e.g. muPDF or FFmpeg) for preview image # generation. The resulting tempfile can be used as the +:io+ value in an attachable Hash: # # def preview @@ -41,18 +42,19 @@ module ActiveStorage # end # end # - # The output tempfile is opened in the directory returned by ActiveStorage::Downloading#tempdir. + # The output tempfile is opened in the directory returned by #tmpdir. def draw(*argv) #:doc: - ActiveSupport::Notifications.instrument("preview.active_storage") do - open_tempfile_for_drawing do |file| + open_tempfile do |file| + instrument :preview, key: blob.key do capture(*argv, to: file) - yield file end + + yield file end end - def open_tempfile_for_drawing - tempfile = Tempfile.open("ActiveStorage", tempdir) + def open_tempfile + tempfile = Tempfile.open("ActiveStorage-", tmpdir) begin yield tempfile @@ -61,6 +63,10 @@ module ActiveStorage end end + def instrument(operation, payload = {}, &block) + ActiveSupport::Notifications.instrument "#{operation}.active_storage", payload, &block + end + def capture(*argv, to:) to.binmode IO.popen(argv, err: File::NULL) { |out| IO.copy_stream(out, to) } @@ -70,5 +76,9 @@ module ActiveStorage def logger #:doc: ActiveStorage.logger end + + def tmpdir #:doc: + Dir.tmpdir + end end end diff --git a/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb b/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb index 2a787362cf..6bf501a607 100644 --- a/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb +++ b/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb @@ -12,7 +12,7 @@ module ActiveStorage end def pdftoppm_exists? - return @pdftoppm_exists unless @pdftoppm_exists.nil? + return @pdftoppm_exists if defined?(@pdftoppm_exists) @pdftoppm_exists = system(pdftoppm_path, "-v", out: File::NULL, err: File::NULL) end @@ -28,7 +28,7 @@ module ActiveStorage private def draw_first_page_from(file, &block) - # use 72 dpi to match thumbnail dimesions of the PDF + # use 72 dpi to match thumbnail dimensions of the PDF draw self.class.pdftoppm_path, "-singlefile", "-r", "72", "-png", file.path, &block end end diff --git a/activestorage/lib/active_storage/previewer/video_previewer.rb b/activestorage/lib/active_storage/previewer/video_previewer.rb index 2f28a3d341..50e13d202a 100644 --- a/activestorage/lib/active_storage/previewer/video_previewer.rb +++ b/activestorage/lib/active_storage/previewer/video_previewer.rb @@ -9,15 +9,14 @@ module ActiveStorage def preview download_blob_to_tempfile do |input| draw_relevant_frame_from input do |output| - yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + yield io: output, filename: "#{blob.filename.base}.jpg", content_type: "image/jpeg" end end end private def draw_relevant_frame_from(file, &block) - draw ffmpeg_path, "-i", file.path, "-y", "-vcodec", "png", - "-vf", "thumbnail", "-vframes", "1", "-f", "image2", "-", &block + draw ffmpeg_path, "-i", file.path, "-y", "-vframes", "1", "-f", "image2", "-", &block end def ffmpeg_path diff --git a/activestorage/lib/active_storage/reflection.rb b/activestorage/lib/active_storage/reflection.rb new file mode 100644 index 0000000000..ce248c88b5 --- /dev/null +++ b/activestorage/lib/active_storage/reflection.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module ActiveStorage + module Reflection + # Holds all the metadata about a has_one_attached attachment as it was + # specified in the Active Record class. + class HasOneAttachedReflection < ActiveRecord::Reflection::MacroReflection #:nodoc: + def macro + :has_one_attached + end + end + + # Holds all the metadata about a has_many_attached attachment as it was + # specified in the Active Record class. + class HasManyAttachedReflection < ActiveRecord::Reflection::MacroReflection #:nodoc: + def macro + :has_many_attached + end + end + + module ReflectionExtension # :nodoc: + def add_attachment_reflection(model, name, reflection) + model.attachment_reflections = model.attachment_reflections.merge(name.to_s => reflection) + end + + private + def reflection_class_for(macro) + case macro + when :has_one_attached + HasOneAttachedReflection + when :has_many_attached + HasManyAttachedReflection + else + super + end + end + end + + module ActiveRecordExtensions + extend ActiveSupport::Concern + + included do + class_attribute :attachment_reflections, instance_writer: false, default: {} + end + + module ClassMethods + # Returns an array of reflection objects for all the attachments in the + # class. + def reflect_on_all_attachments + attachment_reflections.values + end + + # Returns the reflection object for the named +attachment+. + # + # User.reflect_on_attachment(:avatar) + # # => the avatar reflection + # + def reflect_on_attachment(attachment) + attachment_reflections[attachment.to_s] + end + end + end + end +end diff --git a/activestorage/lib/active_storage/service.rb b/activestorage/lib/active_storage/service.rb index 949969fc95..aac1e62e7f 100644 --- a/activestorage/lib/active_storage/service.rb +++ b/activestorage/lib/active_storage/service.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true require "active_storage/log_subscriber" +require "action_dispatch" +require "action_dispatch/http/content_disposition" module ActiveStorage - class IntegrityError < StandardError; end - # Abstract class serving as an interface for concrete services. # # The available services are: @@ -41,8 +41,6 @@ module ActiveStorage extend ActiveSupport::Autoload autoload :Configurator - class_attribute :url_expires_in, default: 5.minutes - class << self # Configure an Active Storage service by name from a set of configurations, # typically loaded from a YAML file. The Active Storage engine uses this @@ -64,10 +62,16 @@ module ActiveStorage # Upload the +io+ to the +key+ specified. If a +checksum+ is provided, the service will # ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError. - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, **options) raise NotImplementedError end + # Update metadata for the file identified by +key+ in the service. + # Override in subclasses only if the service needs to store specific + # metadata that has to be updated upon identification. + def update_metadata(key, **metadata) + end + # Return the content of the file at the +key+. def download(key) raise NotImplementedError @@ -78,6 +82,10 @@ module ActiveStorage raise NotImplementedError end + def open(*args, &block) + ActiveStorage::Downloader.new(self).open(*args, &block) + end + # Delete the file at the +key+. def delete(key) raise NotImplementedError @@ -94,7 +102,7 @@ module ActiveStorage end # Returns a signed, temporary URL for the file at the +key+. The URL will be valid for the amount - # of seconds specified in +expires_in+. You most also provide the +disposition+ (+:inline+ or +:attachment+), + # of seconds specified in +expires_in+. You must also provide the +disposition+ (+:inline+ or +:attachment+), # +filename+, and +content_type+ that you wish the file to be served with on request. def url(key, expires_in:, disposition:, filename:, content_type:) raise NotImplementedError @@ -126,7 +134,8 @@ module ActiveStorage end def content_disposition_with(type: "inline", filename:) - (type.to_s.presence_in(%w( attachment inline )) || "inline") + "; #{filename.parameters}" + disposition = (type.to_s.presence_in(%w( attachment inline )) || "inline") + ActionDispatch::Http::ContentDisposition.format(disposition: disposition, filename: filename.sanitized) end end end diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index 2867a4e441..993cc0e5f7 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -10,19 +10,17 @@ module ActiveStorage class Service::AzureStorageService < Service attr_reader :client, :blobs, :container, :signer - def initialize(storage_account_name:, storage_access_key:, container:) - @client = Azure::Storage::Client.create(storage_account_name: storage_account_name, storage_access_key: storage_access_key) + def initialize(storage_account_name:, storage_access_key:, container:, **options) + @client = Azure::Storage::Client.create(storage_account_name: storage_account_name, storage_access_key: storage_access_key, **options) @signer = Azure::Storage::Core::Auth::SharedAccessSignature.new(storage_account_name, storage_access_key) @blobs = client.blob_client @container = container end - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, **) instrument :upload, key: key, checksum: checksum do - begin - blobs.create_block_blob(container, key, io, content_md5: checksum) - rescue Azure::Core::Http::HTTPError - raise ActiveStorage::IntegrityError + handle_errors do + blobs.create_block_blob(container, key, IO.try_convert(io) || io, content_md5: checksum) end end end @@ -34,26 +32,29 @@ module ActiveStorage end else instrument :download, key: key do - _, io = blobs.get_blob(container, key) - io.force_encoding(Encoding::BINARY) + handle_errors do + _, io = blobs.get_blob(container, key) + io.force_encoding(Encoding::BINARY) + end end end end def download_chunk(key, range) instrument :download_chunk, key: key, range: range do - _, io = blobs.get_blob(container, key, start_range: range.begin, end_range: range.exclude_end? ? range.end - 1 : range.end) - io.force_encoding(Encoding::BINARY) + handle_errors do + _, io = blobs.get_blob(container, key, start_range: range.begin, end_range: range.exclude_end? ? range.end - 1 : range.end) + io.force_encoding(Encoding::BINARY) + end end end def delete(key) instrument :delete, key: key do - begin - blobs.delete_blob(container, key) - rescue Azure::Core::Http::HTTPError - # Ignore files already deleted - end + blobs.delete_blob(container, key) + rescue Azure::Core::Http::HTTPError => e + raise unless e.type == "BlobNotFound" + # Ignore files already deleted end end @@ -139,11 +140,26 @@ module ActiveStorage chunk_size = 5.megabytes offset = 0 + raise ActiveStorage::FileNotFoundError unless blob.present? + while offset < blob.properties[:content_length] _, chunk = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) yield chunk.force_encoding(Encoding::BINARY) offset += chunk_size end end + + def handle_errors + yield + rescue Azure::Core::Http::HTTPError => e + case e.type + when "BlobNotFound" + raise ActiveStorage::FileNotFoundError + when "Md5Mismatch" + raise ActiveStorage::IntegrityError + else + raise + end + end end end diff --git a/activestorage/lib/active_storage/service/configurator.rb b/activestorage/lib/active_storage/service/configurator.rb index 39951fd026..fa80c66c3b 100644 --- a/activestorage/lib/active_storage/service/configurator.rb +++ b/activestorage/lib/active_storage/service/configurator.rb @@ -26,7 +26,9 @@ module ActiveStorage def resolve(class_name) require "active_storage/service/#{class_name.to_s.underscore}_service" - ActiveStorage::Service.const_get(:"#{class_name}Service") + ActiveStorage::Service.const_get(:"#{class_name.camelize}Service") + rescue LoadError + raise "Missing service adapter for #{class_name.inspect}" end end end diff --git a/activestorage/lib/active_storage/service/disk_service.rb b/activestorage/lib/active_storage/service/disk_service.rb index 5b652fe74e..67892d43b2 100644 --- a/activestorage/lib/active_storage/service/disk_service.rb +++ b/activestorage/lib/active_storage/service/disk_service.rb @@ -15,25 +15,23 @@ module ActiveStorage @root = root end - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, **) instrument :upload, key: key, checksum: checksum do IO.copy_stream(io, make_path_for(key)) ensure_integrity_of(key, checksum) if checksum end end - def download(key) + def download(key, &block) if block_given? instrument :streaming_download, key: key do - File.open(path_for(key), "rb") do |file| - while data = file.read(64.kilobytes) - yield data - end - end + stream key, &block end else instrument :download, key: key do File.binread path_for(key) + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError end end end @@ -44,16 +42,16 @@ module ActiveStorage file.seek range.begin file.read range.size end + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError end end def delete(key) instrument :delete, key: key do - begin - File.delete path_for(key) - rescue Errno::ENOENT - # Ignore files already deleted - end + File.delete path_for(key) + rescue Errno::ENOENT + # Ignore files already deleted end end @@ -75,17 +73,23 @@ module ActiveStorage def url(key, expires_in:, filename:, disposition:, content_type:) instrument :url, key: key do |payload| - verified_key_with_expiration = ActiveStorage.verifier.generate(key, expires_in: expires_in, purpose: :blob_key) - - generated_url = - url_helpers.rails_disk_service_url( - verified_key_with_expiration, - host: current_host, - filename: filename, - disposition: content_disposition_with(type: disposition, filename: filename), + content_disposition = content_disposition_with(type: disposition, filename: filename) + verified_key_with_expiration = ActiveStorage.verifier.generate( + { + key: key, + disposition: content_disposition, content_type: content_type - ) + }, + { expires_in: expires_in, + purpose: :blob_key } + ) + generated_url = url_helpers.rails_disk_service_url(verified_key_with_expiration, + host: current_host, + disposition: content_disposition, + content_type: content_type, + filename: filename + ) payload[:url] = generated_url generated_url @@ -117,9 +121,19 @@ module ActiveStorage { "Content-Type" => content_type } end + def path_for(key) #:nodoc: + File.join root, folder_for(key), key + end + private - def path_for(key) - File.join root, folder_for(key), key + def stream(key) + File.open(path_for(key), "rb") do |file| + while data = file.read(5.megabytes) + yield data + end + end + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError end def folder_for(key) diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb index 369c33cbdb..09abc613f3 100644 --- a/activestorage/lib/active_storage/service/gcs_service.rb +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -1,12 +1,7 @@ # frozen_string_literal: true -gem "google-cloud-storage", "~> 1.8" - +gem "google-cloud-storage", "~> 1.11" require "google/cloud/storage" -require "net/http" - -require "active_support/core_ext/object/to_query" -require "active_storage/filename" module ActiveStorage # Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API @@ -16,59 +11,65 @@ module ActiveStorage @config = config end - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, content_type: nil, disposition: nil, filename: nil) instrument :upload, key: key, checksum: checksum do - begin - # The official GCS client library doesn't allow us to create a file with no Content-Type metadata. - # We need the file we create to have no Content-Type so we can control it via the response-content-type - # param in signed URLs. Workaround: let the GCS client create the file with an inferred - # Content-Type (usually "application/octet-stream") then clear it. - bucket.create_file(io, key, md5: checksum).update do |file| - file.content_type = nil - end - rescue Google::Cloud::InvalidArgumentError - raise ActiveStorage::IntegrityError - end + # GCS's signed URLs don't include params such as response-content-type response-content_disposition + # in the signature, which means an attacker can modify them and bypass our effort to force these to + # binary and attachment when the file's content type requires it. The only way to force them is to + # store them as object's metadata. + content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename + bucket.create_file(io, key, md5: checksum, content_type: content_type, content_disposition: content_disposition) + rescue Google::Cloud::InvalidArgumentError + raise ActiveStorage::IntegrityError end end - # FIXME: Download in chunks when given a block. - def download(key) - instrument :download, key: key do - io = file_for(key).download - io.rewind + def download(key, &block) + if block_given? + instrument :streaming_download, key: key do + stream(key, &block) + end + else + instrument :download, key: key do + file_for(key).download.string + rescue Google::Cloud::NotFoundError + raise ActiveStorage::FileNotFoundError + end + end + end - if block_given? - yield io.read - else - io.read + def update_metadata(key, content_type:, disposition: nil, filename: nil) + instrument :update_metadata, key: key, content_type: content_type, disposition: disposition do + file_for(key).update do |file| + file.content_type = content_type + file.content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename end end end def download_chunk(key, range) instrument :download_chunk, key: key, range: range do - uri = URI(url(key, expires_in: 30.seconds, filename: ActiveStorage::Filename.new(""), content_type: "application/octet-stream", disposition: "inline")) - - Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |client| - client.get(uri, "Range" => "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body - end + file_for(key).download(range: range).string + rescue Google::Cloud::NotFoundError + raise ActiveStorage::FileNotFoundError end end def delete(key) instrument :delete, key: key do - begin - file_for(key).delete - rescue Google::Cloud::NotFoundError - # Ignore files already deleted - end + file_for(key).delete + rescue Google::Cloud::NotFoundError + # Ignore files already deleted end end def delete_prefixed(prefix) instrument :delete_prefixed, prefix: prefix do - bucket.files(prefix: prefix).all(&:delete) + bucket.files(prefix: prefix).all do |file| + file.delete + rescue Google::Cloud::NotFoundError + # Ignore concurrently-deleted files + end end end @@ -110,12 +111,27 @@ module ActiveStorage private attr_reader :config - def file_for(key) - bucket.file(key, skip_lookup: true) + def file_for(key, skip_lookup: true) + bucket.file(key, skip_lookup: skip_lookup) + end + + # Reads the file for the given key in chunks, yielding each to the block. + def stream(key) + file = file_for(key, skip_lookup: false) + + chunk_size = 5.megabytes + offset = 0 + + raise ActiveStorage::FileNotFoundError unless file.present? + + while offset < file.size + yield file.download(range: offset..(offset + chunk_size - 1)).string + offset += chunk_size + end end def bucket - @bucket ||= client.bucket(config.fetch(:bucket)) + @bucket ||= client.bucket(config.fetch(:bucket), skip_lookup: true) end def client diff --git a/activestorage/lib/active_storage/service/mirror_service.rb b/activestorage/lib/active_storage/service/mirror_service.rb index 6002ef5a00..aa41df304e 100644 --- a/activestorage/lib/active_storage/service/mirror_service.rb +++ b/activestorage/lib/active_storage/service/mirror_service.rb @@ -9,7 +9,7 @@ module ActiveStorage class Service::MirrorService < Service attr_reader :primary, :mirrors - delegate :download, :download_chunk, :exist?, :url, to: :primary + delegate :download, :download_chunk, :exist?, :url, :path_for, to: :primary # Stitch together from named services. def self.build(primary:, mirrors:, configurator:, **options) #:nodoc: @@ -24,9 +24,9 @@ module ActiveStorage # Upload the +io+ to the +key+ specified to all services. If a +checksum+ is provided, all services will # ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError. - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, **options) each_service.collect do |service| - service.upload key, io.tap(&:rewind), checksum: checksum + service.upload key, io.tap(&:rewind), checksum: checksum, **options end end diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index 5e489f4be1..bf94f3f49e 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -16,13 +16,11 @@ module ActiveStorage @upload_options = upload end - def upload(key, io, checksum: nil) + def upload(key, io, checksum: nil, content_type: nil, **) instrument :upload, key: key, checksum: checksum do - begin - object_for(key).put(upload_options.merge(body: io, content_md5: checksum)) - rescue Aws::S3::Errors::BadDigest - raise ActiveStorage::IntegrityError - end + object_for(key).put(upload_options.merge(body: io, content_md5: checksum, content_type: content_type)) + rescue Aws::S3::Errors::BadDigest + raise ActiveStorage::IntegrityError end end @@ -33,7 +31,9 @@ module ActiveStorage end else instrument :download, key: key do - object_for(key).get.body.read.force_encoding(Encoding::BINARY) + object_for(key).get.body.string.force_encoding(Encoding::BINARY) + rescue Aws::S3::Errors::NoSuchKey + raise ActiveStorage::FileNotFoundError end end end @@ -41,6 +41,8 @@ module ActiveStorage def download_chunk(key, range) instrument :download_chunk, key: key, range: range do object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.read.force_encoding(Encoding::BINARY) + rescue Aws::S3::Errors::NoSuchKey + raise ActiveStorage::FileNotFoundError end end @@ -103,6 +105,8 @@ module ActiveStorage chunk_size = 5.megabytes offset = 0 + raise ActiveStorage::FileNotFoundError unless object.exists? + while offset < object.content_length yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.read.force_encoding(Encoding::BINARY) offset += chunk_size diff --git a/activestorage/lib/active_storage/transformers/image_processing_transformer.rb b/activestorage/lib/active_storage/transformers/image_processing_transformer.rb new file mode 100644 index 0000000000..506150576c --- /dev/null +++ b/activestorage/lib/active_storage/transformers/image_processing_transformer.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require "image_processing" + +module ActiveStorage + module Transformers + class ImageProcessingTransformer < Transformer + private + def process(file, format:) + processor. + source(file). + loader(page: 0). + convert(format). + apply(operations). + call + end + + def processor + ImageProcessing.const_get(ActiveStorage.variant_processor.to_s.camelize) + end + + def operations + transformations.each_with_object([]) do |(name, argument), list| + if name.to_s == "combine_options" + ActiveSupport::Deprecation.warn <<~WARNING.squish + Active Storage's ImageProcessing transformer doesn't support :combine_options, + as it always generates a single ImageMagick command. Passing :combine_options will + not be supported in Rails 6.1. + WARNING + + list.concat argument.keep_if { |key, value| value.present? }.to_a + elsif argument.present? + list << [ name, argument ] + end + end + end + end + end +end diff --git a/activestorage/lib/active_storage/transformers/mini_magick_transformer.rb b/activestorage/lib/active_storage/transformers/mini_magick_transformer.rb new file mode 100644 index 0000000000..e8e99cea9e --- /dev/null +++ b/activestorage/lib/active_storage/transformers/mini_magick_transformer.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require "mini_magick" + +module ActiveStorage + module Transformers + class MiniMagickTransformer < Transformer + private + def process(file, format:) + image = MiniMagick::Image.new(file.path, file) + + transformations.each do |name, argument_or_subtransformations| + image.mogrify do |command| + if name.to_s == "combine_options" + argument_or_subtransformations.each do |subtransformation_name, subtransformation_argument| + pass_transform_argument(command, subtransformation_name, subtransformation_argument) + end + else + pass_transform_argument(command, name, argument_or_subtransformations) + end + end + end + + image.format(format) if format + + image.tempfile.tap(&:open) + end + + def pass_transform_argument(command, method, argument) + if argument == true + command.public_send(method) + elsif argument.present? + command.public_send(method, argument) + end + end + end + end +end diff --git a/activestorage/lib/active_storage/transformers/transformer.rb b/activestorage/lib/active_storage/transformers/transformer.rb new file mode 100644 index 0000000000..2e21201004 --- /dev/null +++ b/activestorage/lib/active_storage/transformers/transformer.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module ActiveStorage + module Transformers + # A Transformer applies a set of transformations to an image. + # + # The following concrete subclasses are included in Active Storage: + # + # * ActiveStorage::Transformers::ImageProcessingTransformer: + # backed by ImageProcessing, a common interface for MiniMagick and ruby-vips + # + # * ActiveStorage::Transformers::MiniMagickTransformer: + # backed by MiniMagick, a wrapper around the ImageMagick CLI + class Transformer + attr_reader :transformations + + def initialize(transformations) + @transformations = transformations + end + + # Applies the transformations to the source image in +file+, producing a target image in the + # specified +format+. Yields an open Tempfile containing the target image. Closes and unlinks + # the output tempfile after yielding to the given block. Returns the result of the block. + def transform(file, format:) + output = process(file, format: format) + + begin + yield output + ensure + output.close! + end + end + + private + # Returns an open Tempfile containing a transformed image in the given +format+. + # All subclasses implement this method. + def process(file, format:) #:doc: + raise NotImplementedError + end + end + end +end |