diff options
Diffstat (limited to 'activestorage/lib')
38 files changed, 2272 insertions, 0 deletions
diff --git a/activestorage/lib/active_storage.rb b/activestorage/lib/active_storage.rb new file mode 100644 index 0000000000..a94ef626f2 --- /dev/null +++ b/activestorage/lib/active_storage.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +#-- +# Copyright (c) 2017-2018 David Heinemeier Hansson, Basecamp +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +#++ + +require "active_record" +require "active_support" +require "active_support/rails" + +require "active_storage/version" +require "active_storage/errors" + +require "marcel" + +module ActiveStorage + extend ActiveSupport::Autoload + + autoload :Attached + autoload :Service + autoload :Previewer + autoload :Analyzer + + mattr_accessor :logger + mattr_accessor :verifier + mattr_accessor :queue + mattr_accessor :previewers, default: [] + mattr_accessor :analyzers, default: [] + mattr_accessor :variant_processor, default: :mini_magick + mattr_accessor :paths, default: {} + mattr_accessor :variable_content_types, default: [] + mattr_accessor :content_types_to_serve_as_binary, default: [] + mattr_accessor :service_urls_expire_in, default: 5.minutes + mattr_accessor :routes_prefix, default: "/rails/active_storage" + + module Transformers + extend ActiveSupport::Autoload + + autoload :Transformer + autoload :ImageProcessingTransformer + autoload :MiniMagickTransformer + end +end diff --git a/activestorage/lib/active_storage/analyzer.rb b/activestorage/lib/active_storage/analyzer.rb new file mode 100644 index 0000000000..caa25418a5 --- /dev/null +++ b/activestorage/lib/active_storage/analyzer.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module ActiveStorage + # This is an abstract base class for analyzers, which extract metadata from blobs. See + # ActiveStorage::Analyzer::ImageAnalyzer for an example of a concrete subclass. + class Analyzer + attr_reader :blob + + # Implement this method in a concrete subclass. Have it return true when given a blob from which + # the analyzer can extract metadata. + def self.accept?(blob) + false + end + + def initialize(blob) + @blob = blob + end + + # Override this method in a concrete subclass. Have it return a Hash of metadata. + def metadata + raise NotImplementedError + end + + private + # Downloads the blob to a tempfile on disk. Yields the tempfile. + def download_blob_to_tempfile(&block) #:doc: + blob.open tempdir: tempdir, &block + end + + def logger #:doc: + ActiveStorage.logger + end + + def tempdir #:doc: + Dir.tmpdir + end + end +end diff --git a/activestorage/lib/active_storage/analyzer/image_analyzer.rb b/activestorage/lib/active_storage/analyzer/image_analyzer.rb new file mode 100644 index 0000000000..3b39de91be --- /dev/null +++ b/activestorage/lib/active_storage/analyzer/image_analyzer.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module ActiveStorage + # Extracts width and height in pixels from an image blob. + # + # If the image contains EXIF data indicating its angle is 90 or 270 degrees, its width and height are swapped for convenience. + # + # Example: + # + # ActiveStorage::Analyzer::ImageAnalyzer.new(blob).metadata + # # => { width: 4104, height: 2736 } + # + # This analyzer relies on the third-party {MiniMagick}[https://github.com/minimagick/minimagick] gem. MiniMagick requires + # the {ImageMagick}[http://www.imagemagick.org] system library. + class Analyzer::ImageAnalyzer < Analyzer + def self.accept?(blob) + blob.image? + end + + def metadata + read_image do |image| + if rotated_image?(image) + { width: image.height, height: image.width } + else + { width: image.width, height: image.height } + end + end + rescue LoadError + logger.info "Skipping image analysis because the mini_magick gem isn't installed" + {} + end + + private + def read_image + download_blob_to_tempfile do |file| + require "mini_magick" + yield MiniMagick::Image.new(file.path) + end + end + + def rotated_image?(image) + %w[ RightTop LeftBottom ].include?(image["%[orientation]"]) + end + end +end diff --git a/activestorage/lib/active_storage/analyzer/null_analyzer.rb b/activestorage/lib/active_storage/analyzer/null_analyzer.rb new file mode 100644 index 0000000000..8ff7ce48e5 --- /dev/null +++ b/activestorage/lib/active_storage/analyzer/null_analyzer.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module ActiveStorage + class Analyzer::NullAnalyzer < Analyzer # :nodoc: + def self.accept?(blob) + true + end + + def metadata + {} + end + end +end diff --git a/activestorage/lib/active_storage/analyzer/video_analyzer.rb b/activestorage/lib/active_storage/analyzer/video_analyzer.rb new file mode 100644 index 0000000000..18d8ff8237 --- /dev/null +++ b/activestorage/lib/active_storage/analyzer/video_analyzer.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module ActiveStorage + # Extracts the following from a video blob: + # + # * Width (pixels) + # * Height (pixels) + # * Duration (seconds) + # * Angle (degrees) + # * Display aspect ratio + # + # Example: + # + # ActiveStorage::VideoAnalyzer.new(blob).metadata + # # => { width: 640.0, height: 480.0, duration: 5.0, angle: 0, display_aspect_ratio: [4, 3] } + # + # When a video's angle is 90 or 270 degrees, its width and height are automatically swapped for convenience. + # + # This analyzer requires the {FFmpeg}[https://www.ffmpeg.org] system library, which is not provided by Rails. + class Analyzer::VideoAnalyzer < Analyzer + def self.accept?(blob) + blob.video? + end + + def metadata + { width: width, height: height, duration: duration, angle: angle, display_aspect_ratio: display_aspect_ratio }.compact + end + + private + def width + if rotated? + computed_height || encoded_height + else + encoded_width + end + end + + def height + if rotated? + encoded_width + else + computed_height || encoded_height + end + end + + def duration + Float(video_stream["duration"]) if video_stream["duration"] + end + + def angle + Integer(tags["rotate"]) if tags["rotate"] + end + + def display_aspect_ratio + if descriptor = video_stream["display_aspect_ratio"] + if terms = descriptor.split(":", 2) + numerator = Integer(terms[0]) + denominator = Integer(terms[1]) + + [numerator, denominator] unless numerator == 0 + end + end + end + + + def rotated? + angle == 90 || angle == 270 + end + + def computed_height + if encoded_width && display_height_scale + encoded_width * display_height_scale + end + end + + def encoded_width + @encoded_width ||= Float(video_stream["width"]) if video_stream["width"] + end + + def encoded_height + @encoded_height ||= Float(video_stream["height"]) if video_stream["height"] + end + + def display_height_scale + @display_height_scale ||= Float(display_aspect_ratio.last) / display_aspect_ratio.first if display_aspect_ratio + end + + + def tags + @tags ||= video_stream["tags"] || {} + end + + def video_stream + @video_stream ||= streams.detect { |stream| stream["codec_type"] == "video" } || {} + end + + def streams + probe["streams"] || [] + end + + def probe + download_blob_to_tempfile { |file| probe_from(file) } + end + + def probe_from(file) + IO.popen([ ffprobe_path, "-print_format", "json", "-show_streams", "-v", "error", file.path ]) do |output| + JSON.parse(output.read) + end + rescue Errno::ENOENT + logger.info "Skipping video analysis because FFmpeg isn't installed" + {} + end + + def ffprobe_path + ActiveStorage.paths[:ffprobe] || "ffprobe" + end + end +end diff --git a/activestorage/lib/active_storage/attached.rb b/activestorage/lib/active_storage/attached.rb new file mode 100644 index 0000000000..b540f85fbe --- /dev/null +++ b/activestorage/lib/active_storage/attached.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require "active_support/core_ext/module/delegation" + +module ActiveStorage + # Abstract base class for the concrete ActiveStorage::Attached::One and ActiveStorage::Attached::Many + # classes that both provide proxy access to the blob association for a record. + class Attached + attr_reader :name, :record + + def initialize(name, record) + @name, @record = name, record + end + + private + def change + record.attachment_changes[name] + end + end +end + +require "active_storage/attached/model" +require "active_storage/attached/one" +require "active_storage/attached/many" +require "active_storage/attached/changes" diff --git a/activestorage/lib/active_storage/attached/changes.rb b/activestorage/lib/active_storage/attached/changes.rb new file mode 100644 index 0000000000..1db3906a63 --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module ActiveStorage + module Attached::Changes #:nodoc: + extend ActiveSupport::Autoload + + eager_autoload do + autoload :CreateOne + autoload :CreateMany + autoload :CreateOneOfMany + + autoload :DeleteOne + autoload :DeleteMany + end + end +end diff --git a/activestorage/lib/active_storage/attached/changes/create_many.rb b/activestorage/lib/active_storage/attached/changes/create_many.rb new file mode 100644 index 0000000000..a7a8553e0f --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes/create_many.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module ActiveStorage + class Attached::Changes::CreateMany #:nodoc: + attr_reader :name, :record, :attachables + + def initialize(name, record, attachables) + @name, @record, @attachables = name, record, Array(attachables) + end + + def attachments + @attachments ||= subchanges.collect(&:attachment) + end + + def blobs + @blobs ||= subchanges.collect(&:blob) + end + + def upload + subchanges.each(&:upload) + end + + def save + assign_associated_attachments + reset_associated_blobs + end + + private + def subchanges + @subchanges ||= attachables.collect { |attachable| build_subchange_from(attachable) } + end + + def build_subchange_from(attachable) + ActiveStorage::Attached::Changes::CreateOneOfMany.new(name, record, attachable) + end + + + def assign_associated_attachments + record.public_send("#{name}_attachments=", attachments) + end + + def reset_associated_blobs + record.public_send("#{name}_blobs").reset + end + end +end diff --git a/activestorage/lib/active_storage/attached/changes/create_one.rb b/activestorage/lib/active_storage/attached/changes/create_one.rb new file mode 100644 index 0000000000..5812fd2b08 --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes/create_one.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require "action_dispatch" +require "action_dispatch/http/upload" + +module ActiveStorage + class Attached::Changes::CreateOne #:nodoc: + attr_reader :name, :record, :attachable + + def initialize(name, record, attachable) + @name, @record, @attachable = name, record, attachable + end + + def attachment + @attachment ||= find_or_build_attachment + end + + def blob + @blob ||= find_or_build_blob + end + + def upload + case attachable + when ActionDispatch::Http::UploadedFile, Rack::Test::UploadedFile + blob.upload_without_unfurling(attachable.open) + when Hash + blob.upload_without_unfurling(attachable.fetch(:io)) + end + end + + def save + record.public_send("#{name}_attachment=", attachment) + end + + private + def find_or_build_attachment + find_attachment || build_attachment + end + + def find_attachment + if record.public_send("#{name}_blob") == blob + record.public_send("#{name}_attachment") + end + end + + def build_attachment + ActiveStorage::Attachment.new(record: record, name: name, blob: blob) + end + + def find_or_build_blob + case attachable + when ActiveStorage::Blob + attachable + when ActionDispatch::Http::UploadedFile, Rack::Test::UploadedFile + ActiveStorage::Blob.build_after_unfurling \ + io: attachable.open, + filename: attachable.original_filename, + content_type: attachable.content_type + when Hash + ActiveStorage::Blob.build_after_unfurling(attachable) + when String + ActiveStorage::Blob.find_signed(attachable) + else + raise ArgumentError, "Could not find or build blob: expected attachable, got #{attachable.inspect}" + end + end + end +end diff --git a/activestorage/lib/active_storage/attached/changes/create_one_of_many.rb b/activestorage/lib/active_storage/attached/changes/create_one_of_many.rb new file mode 100644 index 0000000000..7268e87316 --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes/create_one_of_many.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module ActiveStorage + class Attached::Changes::CreateOneOfMany < Attached::Changes::CreateOne #:nodoc: + private + def find_attachment + record.public_send("#{name}_attachments").detect { |attachment| attachment.blob_id == blob.id } + end + end +end diff --git a/activestorage/lib/active_storage/attached/changes/delete_many.rb b/activestorage/lib/active_storage/attached/changes/delete_many.rb new file mode 100644 index 0000000000..6cbd1158dc --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes/delete_many.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module ActiveStorage + class Attached::Changes::DeleteMany #:nodoc: + attr_reader :name, :record + + def initialize(name, record) + @name, @record = name, record + end + + def attachments + ActiveStorage::Attachment.none + end + + def blobs + ActiveStorage::Blob.none + end + + def save + record.public_send("#{name}_attachments=", []) + end + end +end diff --git a/activestorage/lib/active_storage/attached/changes/delete_one.rb b/activestorage/lib/active_storage/attached/changes/delete_one.rb new file mode 100644 index 0000000000..2f7d356613 --- /dev/null +++ b/activestorage/lib/active_storage/attached/changes/delete_one.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module ActiveStorage + class Attached::Changes::DeleteOne #:nodoc: + attr_reader :name, :record + + def initialize(name, record) + @name, @record = name, record + end + + def attachment + nil + end + + def save + record.public_send("#{name}_attachment=", nil) + end + end +end diff --git a/activestorage/lib/active_storage/attached/many.rb b/activestorage/lib/active_storage/attached/many.rb new file mode 100644 index 0000000000..25f88284df --- /dev/null +++ b/activestorage/lib/active_storage/attached/many.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module ActiveStorage + # Decorated proxy object representing of multiple attachments to a model. + class Attached::Many < Attached + delegate_missing_to :attachments + + # Returns all the associated attachment records. + # + # All methods called on this proxy object that aren't listed here will automatically be delegated to +attachments+. + def attachments + change.present? ? change.attachments : record.public_send("#{name}_attachments") + end + + # Returns all attached blobs. + def blobs + change.present? ? change.blobs : record.public_send("#{name}_blobs") + end + + # Attaches one or more +attachables+ to the record. + # + # If the record is persisted and unchanged, the attachments are saved to + # the database immediately. Otherwise, they'll be saved to the DB when the + # record is next saved. + # + # document.images.attach(params[:images]) # Array of ActionDispatch::Http::UploadedFile objects + # document.images.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload + # document.images.attach(io: File.open("/path/to/racecar.jpg"), filename: "racecar.jpg", content_type: "image/jpg") + # document.images.attach([ first_blob, second_blob ]) + def attach(*attachables) + if record.persisted? && !record.changed? + record.update(name => blobs + attachables.flatten) + else + record.public_send("#{name}=", blobs + attachables.flatten) + end + end + + # Returns true if any attachments has been made. + # + # class Gallery < ActiveRecord::Base + # has_many_attached :photos + # end + # + # Gallery.new.photos.attached? # => false + def attached? + attachments.any? + end + + # Deletes associated attachments without purging them, leaving their respective blobs in place. + def detach + attachments.delete_all if attached? + end + + ## + # :method: purge + # + # Directly purges each associated attachment (i.e. destroys the blobs and + # attachments and deletes the files on the service). + + ## + # :method: purge_later + # + # Purges each associated attachment through the queuing system. + end +end diff --git a/activestorage/lib/active_storage/attached/model.rb b/activestorage/lib/active_storage/attached/model.rb new file mode 100644 index 0000000000..ae7f0685f2 --- /dev/null +++ b/activestorage/lib/active_storage/attached/model.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module ActiveStorage + # Provides the class-level DSL for declaring an Active Record model's attachments. + module Attached::Model + extend ActiveSupport::Concern + + class_methods do + # Specifies the relation between a single attachment and the model. + # + # class User < ActiveRecord::Base + # has_one_attached :avatar + # end + # + # There is no column defined on the model side, Active Storage takes + # care of the mapping between your records and the attachment. + # + # To avoid N+1 queries, you can include the attached blobs in your query like so: + # + # User.with_attached_avatar + # + # Under the covers, this relationship is implemented as a +has_one+ association to a + # ActiveStorage::Attachment record and a +has_one-through+ association to a + # ActiveStorage::Blob record. These associations are available as +avatar_attachment+ + # and +avatar_blob+. But you shouldn't need to work with these associations directly in + # most circumstances. + # + # The system has been designed to having you go through the ActiveStorage::Attached::One + # proxy that provides the dynamic proxy to the associations and factory methods, like +attach+. + # + # If the +:dependent+ option isn't set, the attachment will be purged + # (i.e. destroyed) whenever the record is destroyed. + def has_one_attached(name, dependent: :purge_later) + generated_association_methods.class_eval <<-CODE, __FILE__, __LINE__ + 1 + def #{name} + @active_storage_attached_#{name} ||= ActiveStorage::Attached::One.new("#{name}", self) + end + + def #{name}=(attachable) + attachment_changes["#{name}"] = + if attachable.nil? + ActiveStorage::Attached::Changes::DeleteOne.new("#{name}", self) + else + ActiveStorage::Attached::Changes::CreateOne.new("#{name}", self, attachable) + end + end + CODE + + has_one :"#{name}_attachment", -> { where(name: name) }, class_name: "ActiveStorage::Attachment", as: :record, inverse_of: :record, dependent: :destroy + has_one :"#{name}_blob", through: :"#{name}_attachment", class_name: "ActiveStorage::Blob", source: :blob + + scope :"with_attached_#{name}", -> { includes("#{name}_attachment": :blob) } + + after_save { attachment_changes[name.to_s]&.save } + + after_commit(on: %i[ create update ]) { attachment_changes.delete(name.to_s).try(:upload) } + + ActiveRecord::Reflection.add_attachment_reflection( + self, + name, + ActiveRecord::Reflection.create(:has_one_attached, name, nil, { dependent: dependent }, self) + ) + end + + # Specifies the relation between multiple attachments and the model. + # + # class Gallery < ActiveRecord::Base + # has_many_attached :photos + # end + # + # There are no columns defined on the model side, Active Storage takes + # care of the mapping between your records and the attachments. + # + # To avoid N+1 queries, you can include the attached blobs in your query like so: + # + # Gallery.where(user: Current.user).with_attached_photos + # + # Under the covers, this relationship is implemented as a +has_many+ association to a + # ActiveStorage::Attachment record and a +has_many-through+ association to a + # ActiveStorage::Blob record. These associations are available as +photos_attachments+ + # and +photos_blobs+. But you shouldn't need to work with these associations directly in + # most circumstances. + # + # The system has been designed to having you go through the ActiveStorage::Attached::Many + # proxy that provides the dynamic proxy to the associations and factory methods, like +#attach+. + # + # If the +:dependent+ option isn't set, all the attachments will be purged + # (i.e. destroyed) whenever the record is destroyed. + def has_many_attached(name, dependent: :purge_later) + generated_association_methods.class_eval <<-CODE, __FILE__, __LINE__ + 1 + def #{name} + @active_storage_attached_#{name} ||= ActiveStorage::Attached::Many.new("#{name}", self) + end + + def #{name}=(attachables) + attachment_changes["#{name}"] = + if attachables.nil? || Array(attachables).none? + ActiveStorage::Attached::Changes::DeleteMany.new("#{name}", self) + else + ActiveStorage::Attached::Changes::CreateMany.new("#{name}", self, attachables) + end + end + CODE + + has_many :"#{name}_attachments", -> { where(name: name) }, as: :record, class_name: "ActiveStorage::Attachment", inverse_of: :record, dependent: :destroy do + def purge + each(&:purge) + reset + end + + def purge_later + each(&:purge_later) + reset + end + end + has_many :"#{name}_blobs", through: :"#{name}_attachments", class_name: "ActiveStorage::Blob", source: :blob + + scope :"with_attached_#{name}", -> { includes("#{name}_attachments": :blob) } + + after_save { attachment_changes[name.to_s]&.save } + + after_commit(on: %i[ create update ]) { attachment_changes.delete(name.to_s).try(:upload) } + + ActiveRecord::Reflection.add_attachment_reflection( + self, + name, + ActiveRecord::Reflection.create(:has_many_attached, name, nil, { dependent: dependent }, self) + ) + end + end + + def attachment_changes #:nodoc: + @attachment_changes ||= {} + end + + def reload(*) #:nodoc: + super.tap { @attachment_changes = nil } + end + end +end diff --git a/activestorage/lib/active_storage/attached/one.rb b/activestorage/lib/active_storage/attached/one.rb new file mode 100644 index 0000000000..c039226fcd --- /dev/null +++ b/activestorage/lib/active_storage/attached/one.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module ActiveStorage + # Representation of a single attachment to a model. + class Attached::One < Attached + delegate_missing_to :attachment + + # Returns the associated attachment record. + # + # You don't have to call this method to access the attachment's methods as + # they are all available at the model level. + def attachment + change.present? ? change.attachment : record.public_send("#{name}_attachment") + end + + def blank? + !attached? + end + + # Attaches an +attachable+ to the record. + # + # If the record is persisted and unchanged, the attachment is saved to + # the database immediately. Otherwise, it'll be saved to the DB when the + # record is next saved. + # + # person.avatar.attach(params[:avatar]) # ActionDispatch::Http::UploadedFile object + # person.avatar.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload + # person.avatar.attach(io: File.open("/path/to/face.jpg"), filename: "face.jpg", content_type: "image/jpg") + # person.avatar.attach(avatar_blob) # ActiveStorage::Blob object + def attach(attachable) + if record.persisted? && !record.changed? + record.update(name => attachable) + else + record.public_send("#{name}=", attachable) + end + end + + # Returns +true+ if an attachment has been made. + # + # class User < ActiveRecord::Base + # has_one_attached :avatar + # end + # + # User.new.avatar.attached? # => false + def attached? + attachment.present? + end + + # Deletes the attachment without purging it, leaving its blob in place. + def detach + if attached? + attachment.delete + write_attachment nil + end + end + + # Directly purges the attachment (i.e. destroys the blob and + # attachment and deletes the file on the service). + def purge + if attached? + attachment.purge + write_attachment nil + end + end + + # Purges the attachment through the queuing system. + def purge_later + if attached? + attachment.purge_later + write_attachment nil + end + end + + private + def write_attachment(attachment) + record.public_send("#{name}_attachment=", attachment) + end + end +end diff --git a/activestorage/lib/active_storage/downloader.rb b/activestorage/lib/active_storage/downloader.rb new file mode 100644 index 0000000000..87be6efb05 --- /dev/null +++ b/activestorage/lib/active_storage/downloader.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module ActiveStorage + class Downloader #:nodoc: + def initialize(blob, tempdir: nil) + @blob = blob + @tempdir = tempdir + end + + def download_blob_to_tempfile + open_tempfile do |file| + download_blob_to file + verify_integrity_of file + yield file + end + end + + private + attr_reader :blob, :tempdir + + def open_tempfile + file = Tempfile.open([ "ActiveStorage-#{blob.id}-", blob.filename.extension_with_delimiter ], tempdir) + + begin + yield file + ensure + file.close! + end + end + + def download_blob_to(file) + file.binmode + blob.download { |chunk| file.write(chunk) } + file.flush + file.rewind + end + + def verify_integrity_of(file) + unless Digest::MD5.file(file).base64digest == blob.checksum + raise ActiveStorage::IntegrityError + end + end + end +end diff --git a/activestorage/lib/active_storage/downloading.rb b/activestorage/lib/active_storage/downloading.rb new file mode 100644 index 0000000000..df820bc088 --- /dev/null +++ b/activestorage/lib/active_storage/downloading.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require "tmpdir" +require "active_support/core_ext/string/filters" + +module ActiveStorage + module Downloading + def self.included(klass) + ActiveSupport::Deprecation.warn <<~MESSAGE.squish, caller_locations(2) + ActiveStorage::Downloading is deprecated and will be removed in Active Storage 6.1. + Use ActiveStorage::Blob#open instead. + MESSAGE + end + + private + # Opens a new tempfile in #tempdir and copies blob data into it. Yields the tempfile. + def download_blob_to_tempfile #:doc: + open_tempfile_for_blob do |file| + download_blob_to file + yield file + end + end + + def open_tempfile_for_blob + tempfile = Tempfile.open([ "ActiveStorage", blob.filename.extension_with_delimiter ], tempdir) + + begin + yield tempfile + ensure + tempfile.close! + end + end + + # Efficiently downloads blob data into the given file. + def download_blob_to(file) #:doc: + file.binmode + blob.download { |chunk| file.write(chunk) } + file.flush + file.rewind + end + + # Returns the directory in which tempfiles should be opened. Defaults to +Dir.tmpdir+. + def tempdir #:doc: + Dir.tmpdir + end + end +end diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb new file mode 100644 index 0000000000..7eb93b5e16 --- /dev/null +++ b/activestorage/lib/active_storage/engine.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +require "rails" +require "active_storage" + +require "active_storage/previewer/poppler_pdf_previewer" +require "active_storage/previewer/mupdf_previewer" +require "active_storage/previewer/video_previewer" + +require "active_storage/analyzer/image_analyzer" +require "active_storage/analyzer/video_analyzer" + +require "active_storage/reflection" + +module ActiveStorage + class Engine < Rails::Engine # :nodoc: + isolate_namespace ActiveStorage + + config.active_storage = ActiveSupport::OrderedOptions.new + config.active_storage.previewers = [ ActiveStorage::Previewer::PopplerPDFPreviewer, ActiveStorage::Previewer::MuPDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ] + config.active_storage.analyzers = [ ActiveStorage::Analyzer::ImageAnalyzer, ActiveStorage::Analyzer::VideoAnalyzer ] + config.active_storage.paths = ActiveSupport::OrderedOptions.new + + config.active_storage.variable_content_types = %w( + image/png + image/gif + image/jpg + image/jpeg + image/vnd.adobe.photoshop + image/vnd.microsoft.icon + ) + + config.active_storage.content_types_to_serve_as_binary = %w( + text/html + text/javascript + image/svg+xml + application/postscript + application/x-shockwave-flash + text/xml + application/xml + application/xhtml+xml + ) + + config.eager_load_namespaces << ActiveStorage + + initializer "active_storage.configs" do + config.after_initialize do |app| + ActiveStorage.logger = app.config.active_storage.logger || Rails.logger + ActiveStorage.queue = app.config.active_storage.queue + ActiveStorage.variant_processor = app.config.active_storage.variant_processor || :mini_magick + ActiveStorage.previewers = app.config.active_storage.previewers || [] + ActiveStorage.analyzers = app.config.active_storage.analyzers || [] + ActiveStorage.paths = app.config.active_storage.paths || {} + ActiveStorage.routes_prefix = app.config.active_storage.routes_prefix || "/rails/active_storage" + + ActiveStorage.variable_content_types = app.config.active_storage.variable_content_types || [] + ActiveStorage.content_types_to_serve_as_binary = app.config.active_storage.content_types_to_serve_as_binary || [] + ActiveStorage.service_urls_expire_in = app.config.active_storage.service_urls_expire_in || 5.minutes + end + end + + initializer "active_storage.attached" do + require "active_storage/attached" + + ActiveSupport.on_load(:active_record) do + include ActiveStorage::Attached::Model + end + end + + initializer "active_storage.verifier" do + config.after_initialize do |app| + ActiveStorage.verifier = app.message_verifier("ActiveStorage") + end + end + + initializer "active_storage.services" do + ActiveSupport.on_load(:active_storage_blob) do + if config_choice = Rails.configuration.active_storage.service + configs = Rails.configuration.active_storage.service_configurations ||= begin + config_file = Pathname.new(Rails.root.join("config/storage.yml")) + raise("Couldn't find Active Storage configuration in #{config_file}") unless config_file.exist? + + require "yaml" + require "erb" + + YAML.load(ERB.new(config_file.read).result) || {} + rescue Psych::SyntaxError => e + raise "YAML syntax error occurred while parsing #{config_file}. " \ + "Please note that YAML must be consistently indented using spaces. Tabs are not allowed. " \ + "Error: #{e.message}" + end + + ActiveStorage::Blob.service = + begin + ActiveStorage::Service.configure config_choice, configs + rescue => e + raise e, "Cannot load `Rails.config.active_storage.service`:\n#{e.message}", e.backtrace + end + end + end + end + + initializer "active_storage.reflection" do + ActiveSupport.on_load(:active_record) do + include Reflection::ActiveRecordExtensions + ActiveRecord::Reflection.singleton_class.prepend(Reflection::ReflectionExtension) + end + end + end +end diff --git a/activestorage/lib/active_storage/errors.rb b/activestorage/lib/active_storage/errors.rb new file mode 100644 index 0000000000..6475c1d076 --- /dev/null +++ b/activestorage/lib/active_storage/errors.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module ActiveStorage + # Generic base class for all Active Storage exceptions. + class Error < StandardError; end + + # Raised when ActiveStorage::Blob#variant is called on a blob that isn't variable. + # Use ActiveStorage::Blob#variable? to determine whether a blob is variable. + class InvariableError < Error; end + + # Raised when ActiveStorage::Blob#preview is called on a blob that isn't previewable. + # Use ActiveStorage::Blob#previewable? to determine whether a blob is previewable. + class UnpreviewableError < Error; end + + # Raised when ActiveStorage::Blob#representation is called on a blob that isn't representable. + # Use ActiveStorage::Blob#representable? to determine whether a blob is representable. + class UnrepresentableError < Error; end + + # Raised when uploaded or downloaded data does not match a precomputed checksum. + # Indicates that a network error or a software bug caused data corruption. + class IntegrityError < Error; end + + # Raised when ActiveStorage::Blob#download is called on a blob where the + # backing file is no longer present in its service. + class FileNotFoundError < Error; end +end diff --git a/activestorage/lib/active_storage/gem_version.rb b/activestorage/lib/active_storage/gem_version.rb new file mode 100644 index 0000000000..492620731b --- /dev/null +++ b/activestorage/lib/active_storage/gem_version.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module ActiveStorage + # Returns the version of the currently loaded Active Storage as a <tt>Gem::Version</tt>. + def self.gem_version + Gem::Version.new VERSION::STRING + end + + module VERSION + MAJOR = 6 + MINOR = 0 + TINY = 0 + PRE = "alpha" + + STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".") + end +end diff --git a/activestorage/lib/active_storage/log_subscriber.rb b/activestorage/lib/active_storage/log_subscriber.rb new file mode 100644 index 0000000000..6c0b4c30e7 --- /dev/null +++ b/activestorage/lib/active_storage/log_subscriber.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require "active_support/log_subscriber" + +module ActiveStorage + class LogSubscriber < ActiveSupport::LogSubscriber + def service_upload(event) + message = "Uploaded file to key: #{key_in(event)}" + message += " (checksum: #{event.payload[:checksum]})" if event.payload[:checksum] + info event, color(message, GREEN) + end + + def service_download(event) + info event, color("Downloaded file from key: #{key_in(event)}", BLUE) + end + + alias_method :service_streaming_download, :service_download + + def service_delete(event) + info event, color("Deleted file from key: #{key_in(event)}", RED) + end + + def service_delete_prefixed(event) + info event, color("Deleted files by key prefix: #{event.payload[:prefix]}", RED) + end + + def service_exist(event) + debug event, color("Checked if file exists at key: #{key_in(event)} (#{event.payload[:exist] ? "yes" : "no"})", BLUE) + end + + def service_url(event) + debug event, color("Generated URL for file at key: #{key_in(event)} (#{event.payload[:url]})", BLUE) + end + + def logger + ActiveStorage.logger + end + + private + def info(event, colored_message) + super log_prefix_for_service(event) + colored_message + end + + def debug(event, colored_message) + super log_prefix_for_service(event) + colored_message + end + + def log_prefix_for_service(event) + color " #{event.payload[:service]} Storage (#{event.duration.round(1)}ms) ", CYAN + end + + def key_in(event) + event.payload[:key] + end + end +end + +ActiveStorage::LogSubscriber.attach_to :active_storage diff --git a/activestorage/lib/active_storage/previewer.rb b/activestorage/lib/active_storage/previewer.rb new file mode 100644 index 0000000000..95a041fd16 --- /dev/null +++ b/activestorage/lib/active_storage/previewer.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module ActiveStorage + # This is an abstract base class for previewers, which generate images from blobs. See + # ActiveStorage::Previewer::MuPDFPreviewer and ActiveStorage::Previewer::VideoPreviewer for + # examples of concrete subclasses. + class Previewer + attr_reader :blob + + # Implement this method in a concrete subclass. Have it return true when given a blob from which + # the previewer can generate an image. + def self.accept?(blob) + false + end + + def initialize(blob) + @blob = blob + end + + # Override this method in a concrete subclass. Have it yield an attachable preview image (i.e. + # anything accepted by ActiveStorage::Attached::One#attach). + def preview + raise NotImplementedError + end + + private + # Downloads the blob to a tempfile on disk. Yields the tempfile. + def download_blob_to_tempfile(&block) #:doc: + blob.open tempdir: tempdir, &block + end + + # Executes a system command, capturing its binary output in a tempfile. Yields the tempfile. + # + # Use this method to shell out to a system library (e.g. muPDF or FFmpeg) for preview image + # generation. The resulting tempfile can be used as the +:io+ value in an attachable Hash: + # + # def preview + # download_blob_to_tempfile do |input| + # draw "my-drawing-command", input.path, "--format", "png", "-" do |output| + # yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + # end + # end + # end + # + # The output tempfile is opened in the directory returned by #tempdir. + def draw(*argv) #:doc: + open_tempfile do |file| + instrument :preview, key: blob.key do + capture(*argv, to: file) + end + + yield file + end + end + + def open_tempfile + tempfile = Tempfile.open("ActiveStorage-", tempdir) + + begin + yield tempfile + ensure + tempfile.close! + end + end + + def instrument(operation, payload = {}, &block) + ActiveSupport::Notifications.instrument "#{operation}.active_storage", payload, &block + end + + def capture(*argv, to:) + to.binmode + IO.popen(argv, err: File::NULL) { |out| IO.copy_stream(out, to) } + to.rewind + end + + def logger #:doc: + ActiveStorage.logger + end + + def tempdir #:doc: + Dir.tmpdir + end + end +end diff --git a/activestorage/lib/active_storage/previewer/mupdf_previewer.rb b/activestorage/lib/active_storage/previewer/mupdf_previewer.rb new file mode 100644 index 0000000000..ae02a4889d --- /dev/null +++ b/activestorage/lib/active_storage/previewer/mupdf_previewer.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module ActiveStorage + class Previewer::MuPDFPreviewer < Previewer + class << self + def accept?(blob) + blob.content_type == "application/pdf" && mutool_exists? + end + + def mutool_path + ActiveStorage.paths[:mutool] || "mutool" + end + + def mutool_exists? + return @mutool_exists unless @mutool_exists.nil? + + system mutool_path, out: File::NULL, err: File::NULL + + @mutool_exists = $?.exitstatus == 1 + end + end + + def preview + download_blob_to_tempfile do |input| + draw_first_page_from input do |output| + yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + end + end + end + + private + def draw_first_page_from(file, &block) + draw self.class.mutool_path, "draw", "-F", "png", "-o", "-", file.path, "1", &block + end + end +end diff --git a/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb b/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb new file mode 100644 index 0000000000..69eb617d7b --- /dev/null +++ b/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module ActiveStorage + class Previewer::PopplerPDFPreviewer < Previewer + class << self + def accept?(blob) + blob.content_type == "application/pdf" && pdftoppm_exists? + end + + def pdftoppm_path + ActiveStorage.paths[:pdftoppm] || "pdftoppm" + end + + def pdftoppm_exists? + return @pdftoppm_exists if defined?(@pdftoppm_exists) + + @pdftoppm_exists = system(pdftoppm_path, "-v", out: File::NULL, err: File::NULL) + end + end + + def preview + download_blob_to_tempfile do |input| + draw_first_page_from input do |output| + yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + end + end + end + + private + def draw_first_page_from(file, &block) + # use 72 dpi to match thumbnail dimesions of the PDF + draw self.class.pdftoppm_path, "-singlefile", "-r", "72", "-png", file.path, &block + end + end +end diff --git a/activestorage/lib/active_storage/previewer/video_previewer.rb b/activestorage/lib/active_storage/previewer/video_previewer.rb new file mode 100644 index 0000000000..50e13d202a --- /dev/null +++ b/activestorage/lib/active_storage/previewer/video_previewer.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module ActiveStorage + class Previewer::VideoPreviewer < Previewer + def self.accept?(blob) + blob.video? + end + + def preview + download_blob_to_tempfile do |input| + draw_relevant_frame_from input do |output| + yield io: output, filename: "#{blob.filename.base}.jpg", content_type: "image/jpeg" + end + end + end + + private + def draw_relevant_frame_from(file, &block) + draw ffmpeg_path, "-i", file.path, "-y", "-vframes", "1", "-f", "image2", "-", &block + end + + def ffmpeg_path + ActiveStorage.paths[:ffmpeg] || "ffmpeg" + end + end +end diff --git a/activestorage/lib/active_storage/reflection.rb b/activestorage/lib/active_storage/reflection.rb new file mode 100644 index 0000000000..ce248c88b5 --- /dev/null +++ b/activestorage/lib/active_storage/reflection.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module ActiveStorage + module Reflection + # Holds all the metadata about a has_one_attached attachment as it was + # specified in the Active Record class. + class HasOneAttachedReflection < ActiveRecord::Reflection::MacroReflection #:nodoc: + def macro + :has_one_attached + end + end + + # Holds all the metadata about a has_many_attached attachment as it was + # specified in the Active Record class. + class HasManyAttachedReflection < ActiveRecord::Reflection::MacroReflection #:nodoc: + def macro + :has_many_attached + end + end + + module ReflectionExtension # :nodoc: + def add_attachment_reflection(model, name, reflection) + model.attachment_reflections = model.attachment_reflections.merge(name.to_s => reflection) + end + + private + def reflection_class_for(macro) + case macro + when :has_one_attached + HasOneAttachedReflection + when :has_many_attached + HasManyAttachedReflection + else + super + end + end + end + + module ActiveRecordExtensions + extend ActiveSupport::Concern + + included do + class_attribute :attachment_reflections, instance_writer: false, default: {} + end + + module ClassMethods + # Returns an array of reflection objects for all the attachments in the + # class. + def reflect_on_all_attachments + attachment_reflections.values + end + + # Returns the reflection object for the named +attachment+. + # + # User.reflect_on_attachment(:avatar) + # # => the avatar reflection + # + def reflect_on_attachment(attachment) + attachment_reflections[attachment.to_s] + end + end + end + end +end diff --git a/activestorage/lib/active_storage/service.rb b/activestorage/lib/active_storage/service.rb new file mode 100644 index 0000000000..54ba08fb87 --- /dev/null +++ b/activestorage/lib/active_storage/service.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +require "active_storage/log_subscriber" +require "action_dispatch" +require "action_dispatch/http/content_disposition" + +module ActiveStorage + # Abstract class serving as an interface for concrete services. + # + # The available services are: + # + # * +Disk+, to manage attachments saved directly on the hard drive. + # * +GCS+, to manage attachments through Google Cloud Storage. + # * +S3+, to manage attachments through Amazon S3. + # * +AzureStorage+, to manage attachments through Microsoft Azure Storage. + # * +Mirror+, to be able to use several services to manage attachments. + # + # Inside a Rails application, you can set-up your services through the + # generated <tt>config/storage.yml</tt> file and reference one + # of the aforementioned constant under the +service+ key. For example: + # + # local: + # service: Disk + # root: <%= Rails.root.join("storage") %> + # + # You can checkout the service's constructor to know which keys are required. + # + # Then, in your application's configuration, you can specify the service to + # use like this: + # + # config.active_storage.service = :local + # + # If you are using Active Storage outside of a Ruby on Rails application, you + # can configure the service to use like this: + # + # ActiveStorage::Blob.service = ActiveStorage::Service.configure( + # :Disk, + # root: Pathname("/foo/bar/storage") + # ) + class Service + extend ActiveSupport::Autoload + autoload :Configurator + + class << self + # Configure an Active Storage service by name from a set of configurations, + # typically loaded from a YAML file. The Active Storage engine uses this + # to set the global Active Storage service when the app boots. + def configure(service_name, configurations) + Configurator.build(service_name, configurations) + end + + # Override in subclasses that stitch together multiple services and hence + # need to build additional services using the configurator. + # + # Passes the configurator and all of the service's config as keyword args. + # + # See MirrorService for an example. + def build(configurator:, service: nil, **service_config) #:nodoc: + new(**service_config) + end + end + + # Upload the +io+ to the +key+ specified. If a +checksum+ is provided, the service will + # ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError. + def upload(key, io, checksum: nil) + raise NotImplementedError + end + + # Return the content of the file at the +key+. + def download(key) + raise NotImplementedError + end + + # Return the partial content in the byte +range+ of the file at the +key+. + def download_chunk(key, range) + raise NotImplementedError + end + + # Delete the file at the +key+. + def delete(key) + raise NotImplementedError + end + + # Delete files at keys starting with the +prefix+. + def delete_prefixed(prefix) + raise NotImplementedError + end + + # Return +true+ if a file exists at the +key+. + def exist?(key) + raise NotImplementedError + end + + # Returns a signed, temporary URL for the file at the +key+. The URL will be valid for the amount + # of seconds specified in +expires_in+. You must also provide the +disposition+ (+:inline+ or +:attachment+), + # +filename+, and +content_type+ that you wish the file to be served with on request. + def url(key, expires_in:, disposition:, filename:, content_type:) + raise NotImplementedError + end + + # Returns a signed, temporary URL that a direct upload file can be PUT to on the +key+. + # The URL will be valid for the amount of seconds specified in +expires_in+. + # You must also provide the +content_type+, +content_length+, and +checksum+ of the file + # that will be uploaded. All these attributes will be validated by the service upon upload. + def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) + raise NotImplementedError + end + + # Returns a Hash of headers for +url_for_direct_upload+ requests. + def headers_for_direct_upload(key, filename:, content_type:, content_length:, checksum:) + {} + end + + private + def instrument(operation, payload = {}, &block) + ActiveSupport::Notifications.instrument( + "service_#{operation}.active_storage", + payload.merge(service: service_name), &block) + end + + def service_name + # ActiveStorage::Service::DiskService => Disk + self.class.name.split("::").third.remove("Service") + end + + def content_disposition_with(type: "inline", filename:) + disposition = (type.to_s.presence_in(%w( attachment inline )) || "inline") + ActionDispatch::Http::ContentDisposition.format(disposition: disposition, filename: filename.sanitized) + end + end +end diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb new file mode 100644 index 0000000000..8de3889cb5 --- /dev/null +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -0,0 +1,167 @@ +# frozen_string_literal: true + +require "active_support/core_ext/numeric/bytes" +require "azure/storage" +require "azure/storage/core/auth/shared_access_signature" + +module ActiveStorage + # Wraps the Microsoft Azure Storage Blob Service as an Active Storage service. + # See ActiveStorage::Service for the generic API documentation that applies to all services. + class Service::AzureStorageService < Service + attr_reader :client, :blobs, :container, :signer + + def initialize(storage_account_name:, storage_access_key:, container:) + @client = Azure::Storage::Client.create(storage_account_name: storage_account_name, storage_access_key: storage_access_key) + @signer = Azure::Storage::Core::Auth::SharedAccessSignature.new(storage_account_name, storage_access_key) + @blobs = client.blob_client + @container = container + end + + def upload(key, io, checksum: nil) + instrument :upload, key: key, checksum: checksum do + handle_errors do + blobs.create_block_blob(container, key, IO.try_convert(io) || io, content_md5: checksum) + end + end + end + + def download(key, &block) + if block_given? + instrument :streaming_download, key: key do + stream(key, &block) + end + else + instrument :download, key: key do + handle_errors do + _, io = blobs.get_blob(container, key) + io.force_encoding(Encoding::BINARY) + end + end + end + end + + def download_chunk(key, range) + instrument :download_chunk, key: key, range: range do + handle_errors do + _, io = blobs.get_blob(container, key, start_range: range.begin, end_range: range.exclude_end? ? range.end - 1 : range.end) + io.force_encoding(Encoding::BINARY) + end + end + end + + def delete(key) + instrument :delete, key: key do + begin + blobs.delete_blob(container, key) + rescue Azure::Core::Http::HTTPError => e + raise unless e.type == "BlobNotFound" + # Ignore files already deleted + end + end + end + + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + marker = nil + + loop do + results = blobs.list_blobs(container, prefix: prefix, marker: marker) + + results.each do |blob| + blobs.delete_blob(container, blob.name) + end + + break unless marker = results.continuation_token.presence + end + end + end + + def exist?(key) + instrument :exist, key: key do |payload| + answer = blob_for(key).present? + payload[:exist] = answer + answer + end + end + + def url(key, expires_in:, filename:, disposition:, content_type:) + instrument :url, key: key do |payload| + generated_url = signer.signed_uri( + uri_for(key), false, + service: "b", + permissions: "r", + expiry: format_expiry(expires_in), + content_disposition: content_disposition_with(type: disposition, filename: filename), + content_type: content_type + ).to_s + + payload[:url] = generated_url + + generated_url + end + end + + def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) + instrument :url, key: key do |payload| + generated_url = signer.signed_uri( + uri_for(key), false, + service: "b", + permissions: "rw", + expiry: format_expiry(expires_in) + ).to_s + + payload[:url] = generated_url + + generated_url + end + end + + def headers_for_direct_upload(key, content_type:, checksum:, **) + { "Content-Type" => content_type, "Content-MD5" => checksum, "x-ms-blob-type" => "BlockBlob" } + end + + private + def uri_for(key) + blobs.generate_uri("#{container}/#{key}") + end + + def blob_for(key) + blobs.get_blob_properties(container, key) + rescue Azure::Core::Http::HTTPError + false + end + + def format_expiry(expires_in) + expires_in ? Time.now.utc.advance(seconds: expires_in).iso8601 : nil + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key) + blob = blob_for(key) + + chunk_size = 5.megabytes + offset = 0 + + raise ActiveStorage::FileNotFoundError unless blob.present? + + while offset < blob.properties[:content_length] + _, chunk = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) + yield chunk.force_encoding(Encoding::BINARY) + offset += chunk_size + end + end + + def handle_errors + yield + rescue Azure::Core::Http::HTTPError => e + case e.type + when "BlobNotFound" + raise ActiveStorage::FileNotFoundError + when "Md5Mismatch" + raise ActiveStorage::IntegrityError + else + raise + end + end + end +end diff --git a/activestorage/lib/active_storage/service/configurator.rb b/activestorage/lib/active_storage/service/configurator.rb new file mode 100644 index 0000000000..fa80c66c3b --- /dev/null +++ b/activestorage/lib/active_storage/service/configurator.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module ActiveStorage + class Service::Configurator #:nodoc: + attr_reader :configurations + + def self.build(service_name, configurations) + new(configurations).build(service_name) + end + + def initialize(configurations) + @configurations = configurations.deep_symbolize_keys + end + + def build(service_name) + config = config_for(service_name.to_sym) + resolve(config.fetch(:service)).build(**config, configurator: self) + end + + private + def config_for(name) + configurations.fetch name do + raise "Missing configuration for the #{name.inspect} Active Storage service. Configurations available for #{configurations.keys.inspect}" + end + end + + def resolve(class_name) + require "active_storage/service/#{class_name.to_s.underscore}_service" + ActiveStorage::Service.const_get(:"#{class_name.camelize}Service") + rescue LoadError + raise "Missing service adapter for #{class_name.inspect}" + end + end +end diff --git a/activestorage/lib/active_storage/service/disk_service.rb b/activestorage/lib/active_storage/service/disk_service.rb new file mode 100644 index 0000000000..52f3a3df16 --- /dev/null +++ b/activestorage/lib/active_storage/service/disk_service.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +require "fileutils" +require "pathname" +require "digest/md5" +require "active_support/core_ext/numeric/bytes" + +module ActiveStorage + # Wraps a local disk path as an Active Storage service. See ActiveStorage::Service for the generic API + # documentation that applies to all services. + class Service::DiskService < Service + attr_reader :root + + def initialize(root:) + @root = root + end + + def upload(key, io, checksum: nil) + instrument :upload, key: key, checksum: checksum do + IO.copy_stream(io, make_path_for(key)) + ensure_integrity_of(key, checksum) if checksum + end + end + + def download(key, &block) + if block_given? + instrument :streaming_download, key: key do + stream key, &block + end + else + instrument :download, key: key do + begin + File.binread path_for(key) + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError + end + end + end + end + + def download_chunk(key, range) + instrument :download_chunk, key: key, range: range do + begin + File.open(path_for(key), "rb") do |file| + file.seek range.begin + file.read range.size + end + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError + end + end + end + + def delete(key) + instrument :delete, key: key do + begin + File.delete path_for(key) + rescue Errno::ENOENT + # Ignore files already deleted + end + end + end + + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + Dir.glob(path_for("#{prefix}*")).each do |path| + FileUtils.rm_rf(path) + end + end + end + + def exist?(key) + instrument :exist, key: key do |payload| + answer = File.exist? path_for(key) + payload[:exist] = answer + answer + end + end + + def url(key, expires_in:, filename:, disposition:, content_type:) + instrument :url, key: key do |payload| + verified_key_with_expiration = ActiveStorage.verifier.generate(key, expires_in: expires_in, purpose: :blob_key) + + generated_url = + url_helpers.rails_disk_service_url( + verified_key_with_expiration, + host: current_host, + filename: filename, + disposition: content_disposition_with(type: disposition, filename: filename), + content_type: content_type + ) + + payload[:url] = generated_url + + generated_url + end + end + + def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) + instrument :url, key: key do |payload| + verified_token_with_expiration = ActiveStorage.verifier.generate( + { + key: key, + content_type: content_type, + content_length: content_length, + checksum: checksum + }, + { expires_in: expires_in, + purpose: :blob_token } + ) + + generated_url = url_helpers.update_rails_disk_service_url(verified_token_with_expiration, host: current_host) + + payload[:url] = generated_url + + generated_url + end + end + + def headers_for_direct_upload(key, content_type:, **) + { "Content-Type" => content_type } + end + + def path_for(key) #:nodoc: + File.join root, folder_for(key), key + end + + private + def stream(key) + File.open(path_for(key), "rb") do |file| + while data = file.read(5.megabytes) + yield data + end + end + rescue Errno::ENOENT + raise ActiveStorage::FileNotFoundError + end + + def folder_for(key) + [ key[0..1], key[2..3] ].join("/") + end + + def make_path_for(key) + path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } + end + + def ensure_integrity_of(key, checksum) + unless Digest::MD5.file(path_for(key)).base64digest == checksum + delete key + raise ActiveStorage::IntegrityError + end + end + + def url_helpers + @url_helpers ||= Rails.application.routes.url_helpers + end + + def current_host + ActiveStorage::Current.host + end + end +end diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb new file mode 100644 index 0000000000..18c0f14cfc --- /dev/null +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +gem "google-cloud-storage", "~> 1.11" +require "google/cloud/storage" + +module ActiveStorage + # Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API + # documentation that applies to all services. + class Service::GCSService < Service + def initialize(**config) + @config = config + end + + def upload(key, io, checksum: nil) + instrument :upload, key: key, checksum: checksum do + begin + # The official GCS client library doesn't allow us to create a file with no Content-Type metadata. + # We need the file we create to have no Content-Type so we can control it via the response-content-type + # param in signed URLs. Workaround: let the GCS client create the file with an inferred + # Content-Type (usually "application/octet-stream") then clear it. + bucket.create_file(io, key, md5: checksum).update do |file| + file.content_type = nil + end + rescue Google::Cloud::InvalidArgumentError + raise ActiveStorage::IntegrityError + end + end + end + + def download(key, &block) + if block_given? + instrument :streaming_download, key: key do + stream(key, &block) + end + else + instrument :download, key: key do + begin + file_for(key).download.string + rescue Google::Cloud::NotFoundError + raise ActiveStorage::FileNotFoundError + end + end + end + end + + def download_chunk(key, range) + instrument :download_chunk, key: key, range: range do + begin + file_for(key).download(range: range).string + rescue Google::Cloud::NotFoundError + raise ActiveStorage::FileNotFoundError + end + end + end + + def delete(key) + instrument :delete, key: key do + begin + file_for(key).delete + rescue Google::Cloud::NotFoundError + # Ignore files already deleted + end + end + end + + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + bucket.files(prefix: prefix).all do |file| + begin + file.delete + rescue Google::Cloud::NotFoundError + # Ignore concurrently-deleted files + end + end + end + end + + def exist?(key) + instrument :exist, key: key do |payload| + answer = file_for(key).exists? + payload[:exist] = answer + answer + end + end + + def url(key, expires_in:, filename:, content_type:, disposition:) + instrument :url, key: key do |payload| + generated_url = file_for(key).signed_url expires: expires_in, query: { + "response-content-disposition" => content_disposition_with(type: disposition, filename: filename), + "response-content-type" => content_type + } + + payload[:url] = generated_url + + generated_url + end + end + + def url_for_direct_upload(key, expires_in:, checksum:, **) + instrument :url, key: key do |payload| + generated_url = bucket.signed_url key, method: "PUT", expires: expires_in, content_md5: checksum + + payload[:url] = generated_url + + generated_url + end + end + + def headers_for_direct_upload(key, checksum:, **) + { "Content-MD5" => checksum } + end + + private + attr_reader :config + + def file_for(key, skip_lookup: true) + bucket.file(key, skip_lookup: skip_lookup) + end + + # Reads the file for the given key in chunks, yielding each to the block. + def stream(key) + file = file_for(key, skip_lookup: false) + + chunk_size = 5.megabytes + offset = 0 + + raise ActiveStorage::FileNotFoundError unless file.present? + + while offset < file.size + yield file.download(range: offset..(offset + chunk_size - 1)).string + offset += chunk_size + end + end + + def bucket + @bucket ||= client.bucket(config.fetch(:bucket)) + end + + def client + @client ||= Google::Cloud::Storage.new(config.except(:bucket)) + end + end +end diff --git a/activestorage/lib/active_storage/service/mirror_service.rb b/activestorage/lib/active_storage/service/mirror_service.rb new file mode 100644 index 0000000000..6002ef5a00 --- /dev/null +++ b/activestorage/lib/active_storage/service/mirror_service.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require "active_support/core_ext/module/delegation" + +module ActiveStorage + # Wraps a set of mirror services and provides a single ActiveStorage::Service object that will all + # have the files uploaded to them. A +primary+ service is designated to answer calls to +download+, +exists?+, + # and +url+. + class Service::MirrorService < Service + attr_reader :primary, :mirrors + + delegate :download, :download_chunk, :exist?, :url, to: :primary + + # Stitch together from named services. + def self.build(primary:, mirrors:, configurator:, **options) #:nodoc: + new \ + primary: configurator.build(primary), + mirrors: mirrors.collect { |name| configurator.build name } + end + + def initialize(primary:, mirrors:) + @primary, @mirrors = primary, mirrors + end + + # Upload the +io+ to the +key+ specified to all services. If a +checksum+ is provided, all services will + # ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError. + def upload(key, io, checksum: nil) + each_service.collect do |service| + service.upload key, io.tap(&:rewind), checksum: checksum + end + end + + # Delete the file at the +key+ on all services. + def delete(key) + perform_across_services :delete, key + end + + # Delete files at keys starting with the +prefix+ on all services. + def delete_prefixed(prefix) + perform_across_services :delete_prefixed, prefix + end + + private + def each_service(&block) + [ primary, *mirrors ].each(&block) + end + + def perform_across_services(method, *args) + # FIXME: Convert to be threaded + each_service.collect do |service| + service.public_send method, *args + end + end + end +end diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb new file mode 100644 index 0000000000..89a9e54158 --- /dev/null +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require "aws-sdk-s3" +require "active_support/core_ext/numeric/bytes" + +module ActiveStorage + # Wraps the Amazon Simple Storage Service (S3) as an Active Storage service. + # See ActiveStorage::Service for the generic API documentation that applies to all services. + class Service::S3Service < Service + attr_reader :client, :bucket, :upload_options + + def initialize(bucket:, upload: {}, **options) + @client = Aws::S3::Resource.new(**options) + @bucket = @client.bucket(bucket) + + @upload_options = upload + end + + def upload(key, io, checksum: nil) + instrument :upload, key: key, checksum: checksum do + begin + object_for(key).put(upload_options.merge(body: io, content_md5: checksum)) + rescue Aws::S3::Errors::BadDigest + raise ActiveStorage::IntegrityError + end + end + end + + def download(key, &block) + if block_given? + instrument :streaming_download, key: key do + stream(key, &block) + end + else + instrument :download, key: key do + begin + object_for(key).get.body.string.force_encoding(Encoding::BINARY) + rescue Aws::S3::Errors::NoSuchKey + raise ActiveStorage::FileNotFoundError + end + end + end + end + + def download_chunk(key, range) + instrument :download_chunk, key: key, range: range do + begin + object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.read.force_encoding(Encoding::BINARY) + rescue Aws::S3::Errors::NoSuchKey + raise ActiveStorage::FileNotFoundError + end + end + end + + def delete(key) + instrument :delete, key: key do + object_for(key).delete + end + end + + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + bucket.objects(prefix: prefix).batch_delete! + end + end + + def exist?(key) + instrument :exist, key: key do |payload| + answer = object_for(key).exists? + payload[:exist] = answer + answer + end + end + + def url(key, expires_in:, filename:, disposition:, content_type:) + instrument :url, key: key do |payload| + generated_url = object_for(key).presigned_url :get, expires_in: expires_in.to_i, + response_content_disposition: content_disposition_with(type: disposition, filename: filename), + response_content_type: content_type + + payload[:url] = generated_url + + generated_url + end + end + + def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) + instrument :url, key: key do |payload| + generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i, + content_type: content_type, content_length: content_length, content_md5: checksum + + payload[:url] = generated_url + + generated_url + end + end + + def headers_for_direct_upload(key, content_type:, checksum:, **) + { "Content-Type" => content_type, "Content-MD5" => checksum } + end + + private + def object_for(key) + bucket.object(key) + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key) + object = object_for(key) + + chunk_size = 5.megabytes + offset = 0 + + raise ActiveStorage::FileNotFoundError unless object.exists? + + while offset < object.content_length + yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.read.force_encoding(Encoding::BINARY) + offset += chunk_size + end + end + end +end diff --git a/activestorage/lib/active_storage/transformers/image_processing_transformer.rb b/activestorage/lib/active_storage/transformers/image_processing_transformer.rb new file mode 100644 index 0000000000..7f8685b72d --- /dev/null +++ b/activestorage/lib/active_storage/transformers/image_processing_transformer.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require "image_processing" + +module ActiveStorage + module Transformers + class ImageProcessingTransformer < Transformer + private + def process(file, format:) + processor. + source(file). + loader(page: 0). + convert(format). + apply(operations). + call + end + + def processor + ImageProcessing.const_get(ActiveStorage.variant_processor.to_s.camelize) + end + + def operations + transformations.each_with_object([]) do |(name, argument), list| + if name.to_s == "combine_options" + ActiveSupport::Deprecation.warn <<~WARNING + Active Storage's ImageProcessing transformer doesn't support :combine_options, + as it always generates a single ImageMagick command. Passing :combine_options will + not be supported in Rails 6.1. + WARNING + + list.concat argument.keep_if { |key, value| value.present? }.to_a + elsif argument.present? + list << [ name, argument ] + end + end + end + end + end +end diff --git a/activestorage/lib/active_storage/transformers/mini_magick_transformer.rb b/activestorage/lib/active_storage/transformers/mini_magick_transformer.rb new file mode 100644 index 0000000000..e8e99cea9e --- /dev/null +++ b/activestorage/lib/active_storage/transformers/mini_magick_transformer.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require "mini_magick" + +module ActiveStorage + module Transformers + class MiniMagickTransformer < Transformer + private + def process(file, format:) + image = MiniMagick::Image.new(file.path, file) + + transformations.each do |name, argument_or_subtransformations| + image.mogrify do |command| + if name.to_s == "combine_options" + argument_or_subtransformations.each do |subtransformation_name, subtransformation_argument| + pass_transform_argument(command, subtransformation_name, subtransformation_argument) + end + else + pass_transform_argument(command, name, argument_or_subtransformations) + end + end + end + + image.format(format) if format + + image.tempfile.tap(&:open) + end + + def pass_transform_argument(command, method, argument) + if argument == true + command.public_send(method) + elsif argument.present? + command.public_send(method, argument) + end + end + end + end +end diff --git a/activestorage/lib/active_storage/transformers/transformer.rb b/activestorage/lib/active_storage/transformers/transformer.rb new file mode 100644 index 0000000000..2e21201004 --- /dev/null +++ b/activestorage/lib/active_storage/transformers/transformer.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module ActiveStorage + module Transformers + # A Transformer applies a set of transformations to an image. + # + # The following concrete subclasses are included in Active Storage: + # + # * ActiveStorage::Transformers::ImageProcessingTransformer: + # backed by ImageProcessing, a common interface for MiniMagick and ruby-vips + # + # * ActiveStorage::Transformers::MiniMagickTransformer: + # backed by MiniMagick, a wrapper around the ImageMagick CLI + class Transformer + attr_reader :transformations + + def initialize(transformations) + @transformations = transformations + end + + # Applies the transformations to the source image in +file+, producing a target image in the + # specified +format+. Yields an open Tempfile containing the target image. Closes and unlinks + # the output tempfile after yielding to the given block. Returns the result of the block. + def transform(file, format:) + output = process(file, format: format) + + begin + yield output + ensure + output.close! + end + end + + private + # Returns an open Tempfile containing a transformed image in the given +format+. + # All subclasses implement this method. + def process(file, format:) #:doc: + raise NotImplementedError + end + end + end +end diff --git a/activestorage/lib/active_storage/version.rb b/activestorage/lib/active_storage/version.rb new file mode 100644 index 0000000000..4b6631832b --- /dev/null +++ b/activestorage/lib/active_storage/version.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require_relative "gem_version" + +module ActiveStorage + # Returns the version of the currently loaded ActiveStorage as a <tt>Gem::Version</tt> + def self.version + gem_version + end +end diff --git a/activestorage/lib/tasks/activestorage.rake b/activestorage/lib/tasks/activestorage.rake new file mode 100644 index 0000000000..ac254d717f --- /dev/null +++ b/activestorage/lib/tasks/activestorage.rake @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +namespace :active_storage do + # Prevent migration installation task from showing up twice. + Rake::Task["install:migrations"].clear_comments + + desc "Copy over the migration needed to the application" + task install: :environment do + if Rake::Task.task_defined?("active_storage:install:migrations") + Rake::Task["active_storage:install:migrations"].invoke + else + Rake::Task["app:active_storage:install:migrations"].invoke + end + end +end |