diff options
Diffstat (limited to 'activestorage/lib')
24 files changed, 768 insertions, 125 deletions
diff --git a/activestorage/lib/active_storage.rb b/activestorage/lib/active_storage.rb index ccc1d4a163..e1deee1d82 100644 --- a/activestorage/lib/active_storage.rb +++ b/activestorage/lib/active_storage.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true #-- -# Copyright (c) 2017 David Heinemeier Hansson +# Copyright (c) 2017-2018 David Heinemeier Hansson, Basecamp # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -26,13 +26,27 @@ require "active_record" require "active_support" require "active_support/rails" -require_relative "active_storage/version" + +require "active_storage/version" +require "active_storage/errors" + +require "marcel" module ActiveStorage extend ActiveSupport::Autoload autoload :Attached autoload :Service + autoload :Previewer + autoload :Analyzer + mattr_accessor :logger mattr_accessor :verifier + mattr_accessor :queue + mattr_accessor :previewers, default: [] + mattr_accessor :analyzers, default: [] + mattr_accessor :variant_processor, default: :mini_magick + mattr_accessor :paths, default: {} + mattr_accessor :variable_content_types, default: [] + mattr_accessor :content_types_to_serve_as_binary, default: [] end diff --git a/activestorage/lib/active_storage/analyzer.rb b/activestorage/lib/active_storage/analyzer.rb new file mode 100644 index 0000000000..7c4168c1a0 --- /dev/null +++ b/activestorage/lib/active_storage/analyzer.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require "active_storage/downloading" + +module ActiveStorage + # This is an abstract base class for analyzers, which extract metadata from blobs. See + # ActiveStorage::Analyzer::ImageAnalyzer for an example of a concrete subclass. + class Analyzer + include Downloading + + attr_reader :blob + + # Implement this method in a concrete subclass. Have it return true when given a blob from which + # the analyzer can extract metadata. + def self.accept?(blob) + false + end + + def initialize(blob) + @blob = blob + end + + # Override this method in a concrete subclass. Have it return a Hash of metadata. + def metadata + raise NotImplementedError + end + + private + def logger #:doc: + ActiveStorage.logger + end + end +end diff --git a/activestorage/lib/active_storage/analyzer/image_analyzer.rb b/activestorage/lib/active_storage/analyzer/image_analyzer.rb new file mode 100644 index 0000000000..3b39de91be --- /dev/null +++ b/activestorage/lib/active_storage/analyzer/image_analyzer.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module ActiveStorage + # Extracts width and height in pixels from an image blob. + # + # If the image contains EXIF data indicating its angle is 90 or 270 degrees, its width and height are swapped for convenience. + # + # Example: + # + # ActiveStorage::Analyzer::ImageAnalyzer.new(blob).metadata + # # => { width: 4104, height: 2736 } + # + # This analyzer relies on the third-party {MiniMagick}[https://github.com/minimagick/minimagick] gem. MiniMagick requires + # the {ImageMagick}[http://www.imagemagick.org] system library. + class Analyzer::ImageAnalyzer < Analyzer + def self.accept?(blob) + blob.image? + end + + def metadata + read_image do |image| + if rotated_image?(image) + { width: image.height, height: image.width } + else + { width: image.width, height: image.height } + end + end + rescue LoadError + logger.info "Skipping image analysis because the mini_magick gem isn't installed" + {} + end + + private + def read_image + download_blob_to_tempfile do |file| + require "mini_magick" + yield MiniMagick::Image.new(file.path) + end + end + + def rotated_image?(image) + %w[ RightTop LeftBottom ].include?(image["%[orientation]"]) + end + end +end diff --git a/activestorage/lib/active_storage/analyzer/null_analyzer.rb b/activestorage/lib/active_storage/analyzer/null_analyzer.rb new file mode 100644 index 0000000000..8ff7ce48e5 --- /dev/null +++ b/activestorage/lib/active_storage/analyzer/null_analyzer.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module ActiveStorage + class Analyzer::NullAnalyzer < Analyzer # :nodoc: + def self.accept?(blob) + true + end + + def metadata + {} + end + end +end diff --git a/activestorage/lib/active_storage/analyzer/video_analyzer.rb b/activestorage/lib/active_storage/analyzer/video_analyzer.rb new file mode 100644 index 0000000000..e31bdb0edb --- /dev/null +++ b/activestorage/lib/active_storage/analyzer/video_analyzer.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module ActiveStorage + # Extracts the following from a video blob: + # + # * Width (pixels) + # * Height (pixels) + # * Duration (seconds) + # * Angle (degrees) + # * Display aspect ratio + # + # Example: + # + # ActiveStorage::VideoAnalyzer.new(blob).metadata + # # => { width: 640.0, height: 480.0, duration: 5.0, angle: 0, display_aspect_ratio: [4, 3] } + # + # When a video's angle is 90 or 270 degrees, its width and height are automatically swapped for convenience. + # + # This analyzer requires the {ffmpeg}[https://www.ffmpeg.org] system library, which is not provided by Rails. + class Analyzer::VideoAnalyzer < Analyzer + def self.accept?(blob) + blob.video? + end + + def metadata + { width: width, height: height, duration: duration, angle: angle, display_aspect_ratio: display_aspect_ratio }.compact + end + + private + def width + if rotated? + computed_height || encoded_height + else + encoded_width + end + end + + def height + if rotated? + encoded_width + else + computed_height || encoded_height + end + end + + def duration + Float(video_stream["duration"]) if video_stream["duration"] + end + + def angle + Integer(tags["rotate"]) if tags["rotate"] + end + + def display_aspect_ratio + if descriptor = video_stream["display_aspect_ratio"] + if terms = descriptor.split(":", 2) + numerator = Integer(terms[0]) + denominator = Integer(terms[1]) + + [numerator, denominator] unless numerator == 0 + end + end + end + + + def rotated? + angle == 90 || angle == 270 + end + + def computed_height + if encoded_width && display_height_scale + encoded_width * display_height_scale + end + end + + def encoded_width + @encoded_width ||= Float(video_stream["width"]) if video_stream["width"] + end + + def encoded_height + @encoded_height ||= Float(video_stream["height"]) if video_stream["height"] + end + + def display_height_scale + @display_height_scale ||= Float(display_aspect_ratio.last) / display_aspect_ratio.first if display_aspect_ratio + end + + + def tags + @tags ||= video_stream["tags"] || {} + end + + def video_stream + @video_stream ||= streams.detect { |stream| stream["codec_type"] == "video" } || {} + end + + def streams + probe["streams"] || [] + end + + def probe + download_blob_to_tempfile { |file| probe_from(file) } + end + + def probe_from(file) + IO.popen([ ffprobe_path, "-print_format", "json", "-show_streams", "-v", "error", file.path ]) do |output| + JSON.parse(output.read) + end + rescue Errno::ENOENT + logger.info "Skipping video analysis because ffmpeg isn't installed" + {} + end + + def ffprobe_path + ActiveStorage.paths[:ffprobe] || "ffprobe" + end + end +end diff --git a/activestorage/lib/active_storage/attached/macros.rb b/activestorage/lib/active_storage/attached/macros.rb index 35a081adc4..819f00cc06 100644 --- a/activestorage/lib/active_storage/attached/macros.rb +++ b/activestorage/lib/active_storage/attached/macros.rb @@ -12,6 +12,10 @@ module ActiveStorage # There is no column defined on the model side, Active Storage takes # care of the mapping between your records and the attachment. # + # To avoid N+1 queries, you can include the attached blobs in your query like so: + # + # User.with_attached_avatar + # # Under the covers, this relationship is implemented as a +has_one+ association to a # ActiveStorage::Attachment record and a +has_one-through+ association to a # ActiveStorage::Blob record. These associations are available as +avatar_attachment+ @@ -28,13 +32,21 @@ module ActiveStorage def #{name} @active_storage_attached_#{name} ||= ActiveStorage::Attached::One.new("#{name}", self, dependent: #{dependent == :purge_later ? ":purge_later" : "false"}) end + + def #{name}=(attachable) + #{name}.attach(attachable) + end CODE - has_one :"#{name}_attachment", -> { where(name: name) }, class_name: "ActiveStorage::Attachment", as: :record + has_one :"#{name}_attachment", -> { where(name: name) }, class_name: "ActiveStorage::Attachment", as: :record, inverse_of: :record, dependent: false has_one :"#{name}_blob", through: :"#{name}_attachment", class_name: "ActiveStorage::Blob", source: :blob + scope :"with_attached_#{name}", -> { includes("#{name}_attachment": :blob) } + if dependent == :purge_later - before_destroy { public_send(name).purge_later } + after_destroy_commit { public_send(name).purge_later } + else + before_destroy { public_send(name).detach } end end @@ -67,15 +79,31 @@ module ActiveStorage def #{name} @active_storage_attached_#{name} ||= ActiveStorage::Attached::Many.new("#{name}", self, dependent: #{dependent == :purge_later ? ":purge_later" : "false"}) end + + def #{name}=(attachables) + #{name}.attach(attachables) + end CODE - has_many :"#{name}_attachments", -> { where(name: name) }, as: :record, class_name: "ActiveStorage::Attachment" + has_many :"#{name}_attachments", -> { where(name: name) }, as: :record, class_name: "ActiveStorage::Attachment", inverse_of: :record, dependent: false do + def purge + each(&:purge) + reset + end + + def purge_later + each(&:purge_later) + reset + end + end has_many :"#{name}_blobs", through: :"#{name}_attachments", class_name: "ActiveStorage::Blob", source: :blob scope :"with_attached_#{name}", -> { includes("#{name}_attachments": :blob) } if dependent == :purge_later - before_destroy { public_send(name).purge_later } + after_destroy_commit { public_send(name).purge_later } + else + before_destroy { public_send(name).detach } end end end diff --git a/activestorage/lib/active_storage/attached/many.rb b/activestorage/lib/active_storage/attached/many.rb index 1e0657c33c..d61acb6fad 100644 --- a/activestorage/lib/active_storage/attached/many.rb +++ b/activestorage/lib/active_storage/attached/many.rb @@ -13,7 +13,6 @@ module ActiveStorage end # Associates one or several attachments with the current record, saving them to the database. - # Examples: # # document.images.attach(params[:images]) # Array of ActionDispatch::Http::UploadedFile objects # document.images.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload @@ -21,7 +20,11 @@ module ActiveStorage # document.images.attach([ first_blob, second_blob ]) def attach(*attachables) attachables.flatten.collect do |attachable| - attachments.create!(name: name, blob: create_blob_from(attachable)) + if record.new_record? + attachments.build(record: record, blob: create_blob_from(attachable)) + else + attachments.create!(record: record, blob: create_blob_from(attachable)) + end end end @@ -36,20 +39,21 @@ module ActiveStorage attachments.any? end + # Deletes associated attachments without purging them, leaving their respective blobs in place. + def detach + attachments.destroy_all if attached? + end + + ## + # :method: purge + # # Directly purges each associated attachment (i.e. destroys the blobs and # attachments and deletes the files on the service). - def purge - if attached? - attachments.each(&:purge) - attachments.reload - end - end + + ## + # :method: purge_later + # # Purges each associated attachment through the queuing system. - def purge_later - if attached? - attachments.each(&:purge_later) - end - end end end diff --git a/activestorage/lib/active_storage/attached/one.rb b/activestorage/lib/active_storage/attached/one.rb index c66be08f58..f992cb5f84 100644 --- a/activestorage/lib/active_storage/attached/one.rb +++ b/activestorage/lib/active_storage/attached/one.rb @@ -13,18 +13,27 @@ module ActiveStorage record.public_send("#{name}_attachment") end + def blank? + attachment.blank? + end + # Associates a given attachment with the current record, saving it to the database. - # Examples: # # person.avatar.attach(params[:avatar]) # ActionDispatch::Http::UploadedFile object # person.avatar.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload # person.avatar.attach(io: File.open("/path/to/face.jpg"), filename: "face.jpg", content_type: "image/jpg") # person.avatar.attach(avatar_blob) # ActiveStorage::Blob object def attach(attachable) - if attached? && dependent == :purge_later - replace attachable - else - write_attachment create_attachment_from(attachable) + blob_was = blob if attached? + blob = create_blob_from(attachable) + + unless blob == blob_was + transaction do + detach + write_attachment build_attachment(blob: blob) + end + + blob_was.purge_later if blob_was && dependent == :purge_later end end @@ -39,6 +48,14 @@ module ActiveStorage attachment.present? end + # Deletes the attachment without purging it, leaving its blob in place. + def detach + if attached? + attachment.destroy + write_attachment nil + end + end + # Directly purges the attachment (i.e. destroys the blob and # attachment and deletes the file on the service). def purge @@ -56,17 +73,10 @@ module ActiveStorage end private - def replace(attachable) - blob.tap do - transaction do - destroy - write_attachment create_attachment_from(attachable) - end - end.purge_later - end + delegate :transaction, to: :record - def create_attachment_from(attachable) - ActiveStorage::Attachment.create!(record: record, name: name, blob: create_blob_from(attachable)) + def build_attachment(blob:) + ActiveStorage::Attachment.new(record: record, name: name, blob: blob) end def write_attachment(attachment) diff --git a/activestorage/lib/active_storage/downloading.rb b/activestorage/lib/active_storage/downloading.rb new file mode 100644 index 0000000000..7c3d20ade0 --- /dev/null +++ b/activestorage/lib/active_storage/downloading.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require "tmpdir" + +module ActiveStorage + module Downloading + private + # Opens a new tempfile in #tempdir and copies blob data into it. Yields the tempfile. + def download_blob_to_tempfile #:doc: + open_tempfile_for_blob do |file| + download_blob_to file + yield file + end + end + + def open_tempfile_for_blob + tempfile = Tempfile.open([ "ActiveStorage", blob.filename.extension_with_delimiter ], tempdir) + + begin + yield tempfile + ensure + tempfile.close! + end + end + + # Efficiently downloads blob data into the given file. + def download_blob_to(file) #:doc: + file.binmode + blob.download { |chunk| file.write(chunk) } + file.flush + file.rewind + end + + # Returns the directory in which tempfiles should be opened. Defaults to +Dir.tmpdir+. + def tempdir #:doc: + Dir.tmpdir + end + end +end diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb index 590a36a30a..99588cdd4b 100644 --- a/activestorage/lib/active_storage/engine.rb +++ b/activestorage/lib/active_storage/engine.rb @@ -3,19 +3,55 @@ require "rails" require "active_storage" +require "active_storage/previewer/poppler_pdf_previewer" +require "active_storage/previewer/mupdf_previewer" +require "active_storage/previewer/video_previewer" + +require "active_storage/analyzer/image_analyzer" +require "active_storage/analyzer/video_analyzer" + module ActiveStorage class Engine < Rails::Engine # :nodoc: isolate_namespace ActiveStorage config.active_storage = ActiveSupport::OrderedOptions.new + config.active_storage.previewers = [ ActiveStorage::Previewer::PopplerPDFPreviewer, ActiveStorage::Previewer::MuPDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ] + config.active_storage.analyzers = [ ActiveStorage::Analyzer::ImageAnalyzer, ActiveStorage::Analyzer::VideoAnalyzer ] + config.active_storage.paths = ActiveSupport::OrderedOptions.new - config.eager_load_namespaces << ActiveStorage + config.active_storage.variable_content_types = %w( + image/png + image/gif + image/jpg + image/jpeg + image/vnd.adobe.photoshop + image/vnd.microsoft.icon + ) - initializer "active_storage.logger" do - require "active_storage/service" + config.active_storage.content_types_to_serve_as_binary = %w( + text/html + text/javascript + image/svg+xml + application/postscript + application/x-shockwave-flash + text/xml + application/xml + application/xhtml+xml + ) + config.eager_load_namespaces << ActiveStorage + + initializer "active_storage.configs" do config.after_initialize do |app| - ActiveStorage::Service.logger = app.config.active_storage.logger || Rails.logger + ActiveStorage.logger = app.config.active_storage.logger || Rails.logger + ActiveStorage.queue = app.config.active_storage.queue + ActiveStorage.variant_processor = app.config.active_storage.variant_processor || :mini_magick + ActiveStorage.previewers = app.config.active_storage.previewers || [] + ActiveStorage.analyzers = app.config.active_storage.analyzers || [] + ActiveStorage.paths = app.config.active_storage.paths || {} + + ActiveStorage.variable_content_types = app.config.active_storage.variable_content_types || [] + ActiveStorage.content_types_to_serve_as_binary = app.config.active_storage.content_types_to_serve_as_binary || [] end end @@ -34,7 +70,7 @@ module ActiveStorage end initializer "active_storage.services" do - config.to_prepare do + ActiveSupport.on_load(:active_storage_blob) do if config_choice = Rails.configuration.active_storage.service configs = Rails.configuration.active_storage.service_configurations ||= begin config_file = Pathname.new(Rails.root.join("config/storage.yml")) diff --git a/activestorage/lib/active_storage/errors.rb b/activestorage/lib/active_storage/errors.rb new file mode 100644 index 0000000000..f099b13f5b --- /dev/null +++ b/activestorage/lib/active_storage/errors.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module ActiveStorage + class InvariableError < StandardError; end + class UnpreviewableError < StandardError; end + class UnrepresentableError < StandardError; end +end diff --git a/activestorage/lib/active_storage/gem_version.rb b/activestorage/lib/active_storage/gem_version.rb index e1d7b3493a..492620731b 100644 --- a/activestorage/lib/active_storage/gem_version.rb +++ b/activestorage/lib/active_storage/gem_version.rb @@ -7,8 +7,8 @@ module ActiveStorage end module VERSION - MAJOR = 5 - MINOR = 2 + MAJOR = 6 + MINOR = 0 TINY = 0 PRE = "alpha" diff --git a/activestorage/lib/active_storage/log_subscriber.rb b/activestorage/lib/active_storage/log_subscriber.rb index 0d00a75c0e..a4e148c1a5 100644 --- a/activestorage/lib/active_storage/log_subscriber.rb +++ b/activestorage/lib/active_storage/log_subscriber.rb @@ -18,6 +18,10 @@ module ActiveStorage info event, color("Deleted file from key: #{key_in(event)}", RED) end + def service_delete_prefixed(event) + info event, color("Deleted files by key prefix: #{event.payload[:prefix]}", RED) + end + def service_exist(event) debug event, color("Checked if file exists at key: #{key_in(event)} (#{event.payload[:exist] ? "yes" : "no"})", BLUE) end @@ -27,7 +31,7 @@ module ActiveStorage end def logger - ActiveStorage::Service.logger + ActiveStorage.logger end private diff --git a/activestorage/lib/active_storage/previewer.rb b/activestorage/lib/active_storage/previewer.rb new file mode 100644 index 0000000000..cf19987d72 --- /dev/null +++ b/activestorage/lib/active_storage/previewer.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require "active_storage/downloading" + +module ActiveStorage + # This is an abstract base class for previewers, which generate images from blobs. See + # ActiveStorage::Previewer::PDFPreviewer and ActiveStorage::Previewer::VideoPreviewer for examples of + # concrete subclasses. + class Previewer + include Downloading + + attr_reader :blob + + # Implement this method in a concrete subclass. Have it return true when given a blob from which + # the previewer can generate an image. + def self.accept?(blob) + false + end + + def initialize(blob) + @blob = blob + end + + # Override this method in a concrete subclass. Have it yield an attachable preview image (i.e. + # anything accepted by ActiveStorage::Attached::One#attach). + def preview + raise NotImplementedError + end + + private + # Executes a system command, capturing its binary output in a tempfile. Yields the tempfile. + # + # Use this method to shell out to a system library (e.g. mupdf or ffmpeg) for preview image + # generation. The resulting tempfile can be used as the +:io+ value in an attachable Hash: + # + # def preview + # download_blob_to_tempfile do |input| + # draw "my-drawing-command", input.path, "--format", "png", "-" do |output| + # yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + # end + # end + # end + # + # The output tempfile is opened in the directory returned by ActiveStorage::Downloading#tempdir. + def draw(*argv) #:doc: + ActiveSupport::Notifications.instrument("preview.active_storage") do + open_tempfile_for_drawing do |file| + capture(*argv, to: file) + yield file + end + end + end + + def open_tempfile_for_drawing + tempfile = Tempfile.open("ActiveStorage", tempdir) + + begin + yield tempfile + ensure + tempfile.close! + end + end + + def capture(*argv, to:) + to.binmode + IO.popen(argv, err: File::NULL) { |out| IO.copy_stream(out, to) } + to.rewind + end + + def logger #:doc: + ActiveStorage.logger + end + end +end diff --git a/activestorage/lib/active_storage/previewer/mupdf_previewer.rb b/activestorage/lib/active_storage/previewer/mupdf_previewer.rb new file mode 100644 index 0000000000..ae02a4889d --- /dev/null +++ b/activestorage/lib/active_storage/previewer/mupdf_previewer.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module ActiveStorage + class Previewer::MuPDFPreviewer < Previewer + class << self + def accept?(blob) + blob.content_type == "application/pdf" && mutool_exists? + end + + def mutool_path + ActiveStorage.paths[:mutool] || "mutool" + end + + def mutool_exists? + return @mutool_exists unless @mutool_exists.nil? + + system mutool_path, out: File::NULL, err: File::NULL + + @mutool_exists = $?.exitstatus == 1 + end + end + + def preview + download_blob_to_tempfile do |input| + draw_first_page_from input do |output| + yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + end + end + end + + private + def draw_first_page_from(file, &block) + draw self.class.mutool_path, "draw", "-F", "png", "-o", "-", file.path, "1", &block + end + end +end diff --git a/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb b/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb new file mode 100644 index 0000000000..2a787362cf --- /dev/null +++ b/activestorage/lib/active_storage/previewer/poppler_pdf_previewer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module ActiveStorage + class Previewer::PopplerPDFPreviewer < Previewer + class << self + def accept?(blob) + blob.content_type == "application/pdf" && pdftoppm_exists? + end + + def pdftoppm_path + ActiveStorage.paths[:pdftoppm] || "pdftoppm" + end + + def pdftoppm_exists? + return @pdftoppm_exists unless @pdftoppm_exists.nil? + + @pdftoppm_exists = system(pdftoppm_path, "-v", out: File::NULL, err: File::NULL) + end + end + + def preview + download_blob_to_tempfile do |input| + draw_first_page_from input do |output| + yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + end + end + end + + private + def draw_first_page_from(file, &block) + # use 72 dpi to match thumbnail dimesions of the PDF + draw self.class.pdftoppm_path, "-singlefile", "-r", "72", "-png", file.path, &block + end + end +end diff --git a/activestorage/lib/active_storage/previewer/video_previewer.rb b/activestorage/lib/active_storage/previewer/video_previewer.rb new file mode 100644 index 0000000000..2f28a3d341 --- /dev/null +++ b/activestorage/lib/active_storage/previewer/video_previewer.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module ActiveStorage + class Previewer::VideoPreviewer < Previewer + def self.accept?(blob) + blob.video? + end + + def preview + download_blob_to_tempfile do |input| + draw_relevant_frame_from input do |output| + yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + end + end + end + + private + def draw_relevant_frame_from(file, &block) + draw ffmpeg_path, "-i", file.path, "-y", "-vcodec", "png", + "-vf", "thumbnail", "-vframes", "1", "-f", "image2", "-", &block + end + + def ffmpeg_path + ActiveStorage.paths[:ffmpeg] || "ffmpeg" + end + end +end diff --git a/activestorage/lib/active_storage/service.rb b/activestorage/lib/active_storage/service.rb index b80fdea1ab..949969fc95 100644 --- a/activestorage/lib/active_storage/service.rb +++ b/activestorage/lib/active_storage/service.rb @@ -4,6 +4,7 @@ require "active_storage/log_subscriber" module ActiveStorage class IntegrityError < StandardError; end + # Abstract class serving as an interface for concrete services. # # The available services are: @@ -40,7 +41,7 @@ module ActiveStorage extend ActiveSupport::Autoload autoload :Configurator - class_attribute :logger + class_attribute :url_expires_in, default: 5.minutes class << self # Configure an Active Storage service by name from a set of configurations, @@ -72,11 +73,21 @@ module ActiveStorage raise NotImplementedError end + # Return the partial content in the byte +range+ of the file at the +key+. + def download_chunk(key, range) + raise NotImplementedError + end + # Delete the file at the +key+. def delete(key) raise NotImplementedError end + # Delete files at keys starting with the +prefix+. + def delete_prefixed(prefix) + raise NotImplementedError + end + # Return +true+ if a file exists at the +key+. def exist?(key) raise NotImplementedError @@ -91,7 +102,7 @@ module ActiveStorage # Returns a signed, temporary URL that a direct upload file can be PUT to on the +key+. # The URL will be valid for the amount of seconds specified in +expires_in+. - # You most also provide the +content_type+, +content_length+, and +checksum+ of the file + # You must also provide the +content_type+, +content_length+, and +checksum+ of the file # that will be uploaded. All these attributes will be validated by the service upon upload. def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) raise NotImplementedError @@ -103,15 +114,19 @@ module ActiveStorage end private - def instrument(operation, key, payload = {}, &block) + def instrument(operation, payload = {}, &block) ActiveSupport::Notifications.instrument( "service_#{operation}.active_storage", - payload.merge(key: key, service: service_name), &block) + payload.merge(service: service_name), &block) end def service_name # ActiveStorage::Service::DiskService => Disk self.class.name.split("::").third.remove("Service") end + + def content_disposition_with(type: "inline", filename:) + (type.to_s.presence_in(%w( attachment inline )) || "inline") + "; #{filename.parameters}" + end end end diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index 895cc9c2f1..2867a4e441 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -8,18 +8,17 @@ module ActiveStorage # Wraps the Microsoft Azure Storage Blob Service as an Active Storage service. # See ActiveStorage::Service for the generic API documentation that applies to all services. class Service::AzureStorageService < Service - attr_reader :client, :path, :blobs, :container, :signer + attr_reader :client, :blobs, :container, :signer - def initialize(path:, storage_account_name:, storage_access_key:, container:) + def initialize(storage_account_name:, storage_access_key:, container:) @client = Azure::Storage::Client.create(storage_account_name: storage_account_name, storage_access_key: storage_access_key) @signer = Azure::Storage::Core::Auth::SharedAccessSignature.new(storage_account_name, storage_access_key) @blobs = client.blob_client @container = container - @path = path end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do begin blobs.create_block_blob(container, key, io, content_md5: checksum) rescue Azure::Core::Http::HTTPError @@ -28,31 +27,54 @@ module ActiveStorage end end - def download(key) + def download(key, &block) if block_given? - instrument :streaming_download, key do + instrument :streaming_download, key: key do stream(key, &block) end else - instrument :download, key do + instrument :download, key: key do _, io = blobs.get_blob(container, key) io.force_encoding(Encoding::BINARY) end end end + def download_chunk(key, range) + instrument :download_chunk, key: key, range: range do + _, io = blobs.get_blob(container, key, start_range: range.begin, end_range: range.exclude_end? ? range.end - 1 : range.end) + io.force_encoding(Encoding::BINARY) + end + end + def delete(key) - instrument :delete, key do + instrument :delete, key: key do begin blobs.delete_blob(container, key) rescue Azure::Core::Http::HTTPError - false + # Ignore files already deleted + end + end + end + + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + marker = nil + + loop do + results = blobs.list_blobs(container, prefix: prefix, marker: marker) + + results.each do |blob| + blobs.delete_blob(container, blob.name) + end + + break unless marker = results.continuation_token.presence end end end def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = blob_for(key).present? payload[:exist] = answer answer @@ -60,13 +82,13 @@ module ActiveStorage end def url(key, expires_in:, filename:, disposition:, content_type:) - instrument :url, key do |payload| - base_url = url_for(key) + instrument :url, key: key do |payload| generated_url = signer.signed_uri( - URI(base_url), false, + uri_for(key), false, + service: "b", permissions: "r", expiry: format_expiry(expires_in), - content_disposition: disposition, + content_disposition: content_disposition_with(type: disposition, filename: filename), content_type: content_type ).to_s @@ -77,10 +99,13 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| - base_url = url_for(key) - generated_url = signer.signed_uri(URI(base_url), false, permissions: "rw", - expiry: format_expiry(expires_in)).to_s + instrument :url, key: key do |payload| + generated_url = signer.signed_uri( + uri_for(key), false, + service: "b", + permissions: "rw", + expiry: format_expiry(expires_in) + ).to_s payload[:url] = generated_url @@ -93,8 +118,8 @@ module ActiveStorage end private - def url_for(key) - "#{path}/#{container}/#{key}" + def uri_for(key) + blobs.generate_uri("#{container}/#{key}") end def blob_for(key) @@ -108,15 +133,15 @@ module ActiveStorage end # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, options = {}, &block) + def stream(key) blob = blob_for(key) chunk_size = 5.megabytes offset = 0 while offset < blob.properties[:content_length] - _, io = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) - yield io + _, chunk = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) + yield chunk.force_encoding(Encoding::BINARY) offset += chunk_size end end diff --git a/activestorage/lib/active_storage/service/disk_service.rb b/activestorage/lib/active_storage/service/disk_service.rb index f600753a08..5b652fe74e 100644 --- a/activestorage/lib/active_storage/service/disk_service.rb +++ b/activestorage/lib/active_storage/service/disk_service.rb @@ -16,7 +16,7 @@ module ActiveStorage end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do IO.copy_stream(io, make_path_for(key)) ensure_integrity_of(key, checksum) if checksum end @@ -24,7 +24,7 @@ module ActiveStorage def download(key) if block_given? - instrument :streaming_download, key do + instrument :streaming_download, key: key do File.open(path_for(key), "rb") do |file| while data = file.read(64.kilobytes) yield data @@ -32,14 +32,23 @@ module ActiveStorage end end else - instrument :download, key do + instrument :download, key: key do File.binread path_for(key) end end end + def download_chunk(key, range) + instrument :download_chunk, key: key, range: range do + File.open(path_for(key), "rb") do |file| + file.seek range.begin + file.read range.size + end + end + end + def delete(key) - instrument :delete, key do + instrument :delete, key: key do begin File.delete path_for(key) rescue Errno::ENOENT @@ -48,8 +57,16 @@ module ActiveStorage end end + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + Dir.glob(path_for("#{prefix}*")).each do |path| + FileUtils.rm_rf(path) + end + end + end + def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = File.exist? path_for(key) payload[:exist] = answer answer @@ -57,17 +74,17 @@ module ActiveStorage end def url(key, expires_in:, filename:, disposition:, content_type:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| verified_key_with_expiration = ActiveStorage.verifier.generate(key, expires_in: expires_in, purpose: :blob_key) generated_url = - if defined?(Rails.application) - Rails.application.routes.url_helpers.rails_disk_service_path \ - verified_key_with_expiration, - filename: filename, disposition: disposition, content_type: content_type - else - "/rails/active_storage/disk/#{verified_key_with_expiration}/#{filename}?content_type=#{content_type}&disposition=#{disposition}" - end + url_helpers.rails_disk_service_url( + verified_key_with_expiration, + host: current_host, + filename: filename, + disposition: content_disposition_with(type: disposition, filename: filename), + content_type: content_type + ) payload[:url] = generated_url @@ -76,7 +93,7 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| verified_token_with_expiration = ActiveStorage.verifier.generate( { key: key, @@ -88,12 +105,7 @@ module ActiveStorage purpose: :blob_token } ) - generated_url = - if defined?(Rails.application) - Rails.application.routes.url_helpers.update_rails_disk_service_path verified_token_with_expiration - else - "/rails/active_storage/disk/#{verified_token_with_expiration}" - end + generated_url = url_helpers.update_rails_disk_service_url(verified_token_with_expiration, host: current_host) payload[:url] = generated_url @@ -124,5 +136,13 @@ module ActiveStorage raise ActiveStorage::IntegrityError end end + + def url_helpers + @url_helpers ||= Rails.application.routes.url_helpers + end + + def current_host + ActiveStorage::Current.host + end end end diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb index 685dd61a0a..16a0765fc5 100644 --- a/activestorage/lib/active_storage/service/gcs_service.rb +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -1,40 +1,64 @@ # frozen_string_literal: true +gem "google-cloud-storage", "~> 1.8" + require "google/cloud/storage" +require "net/http" + require "active_support/core_ext/object/to_query" +require "active_storage/filename" module ActiveStorage # Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API # documentation that applies to all services. class Service::GCSService < Service - attr_reader :client, :bucket - - def initialize(project:, keyfile:, bucket:, **options) - @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile, **options) - @bucket = @client.bucket(bucket) + def initialize(**config) + @config = config end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do begin - bucket.create_file(io, key, md5: checksum) + # The official GCS client library doesn't allow us to create a file with no Content-Type metadata. + # We need the file we create to have no Content-Type so we can control it via the response-content-type + # param in signed URLs. Workaround: let the GCS client create the file with an inferred + # Content-Type (usually "application/octet-stream") then clear it. + bucket.create_file(io, key, md5: checksum).update do |file| + file.content_type = nil + end rescue Google::Cloud::InvalidArgumentError raise ActiveStorage::IntegrityError end end end - # FIXME: Add streaming when given a block + # FIXME: Download in chunks when given a block. def download(key) - instrument :download, key do + instrument :download, key: key do io = file_for(key).download io.rewind - io.read + + if block_given? + yield io.read + else + io.read + end + end + end + + def download_chunk(key, range) + instrument :download_chunk, key: key, range: range do + file = file_for(key) + uri = URI(file.signed_url(expires: 30.seconds)) + + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |client| + client.get(uri, "Range" => "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body + end end end def delete(key) - instrument :delete, key do + instrument :delete, key: key do begin file_for(key).delete rescue Google::Cloud::NotFoundError @@ -43,8 +67,14 @@ module ActiveStorage end end + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + bucket.files(prefix: prefix).all(&:delete) + end + end + def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = file_for(key).exists? payload[:exist] = answer answer @@ -52,9 +82,9 @@ module ActiveStorage end def url(key, expires_in:, filename:, content_type:, disposition:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = file_for(key).signed_url expires: expires_in, query: { - "response-content-disposition" => disposition, + "response-content-disposition" => content_disposition_with(type: disposition, filename: filename), "response-content-type" => content_type } @@ -64,10 +94,9 @@ module ActiveStorage end end - def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| - generated_url = bucket.signed_url key, method: "PUT", expires: expires_in, - content_type: content_type, content_md5: checksum + def url_for_direct_upload(key, expires_in:, checksum:, **) + instrument :url, key: key do |payload| + generated_url = bucket.signed_url key, method: "PUT", expires: expires_in, content_md5: checksum payload[:url] = generated_url @@ -75,13 +104,23 @@ module ActiveStorage end end - def headers_for_direct_upload(key, content_type:, checksum:, **) - { "Content-Type" => content_type, "Content-MD5" => checksum } + def headers_for_direct_upload(key, checksum:, **) + { "Content-MD5" => checksum } end private + attr_reader :config + def file_for(key) bucket.file(key, skip_lookup: true) end + + def bucket + @bucket ||= client.bucket(config.fetch(:bucket)) + end + + def client + @client ||= Google::Cloud::Storage.new(config.except(:bucket)) + end end end diff --git a/activestorage/lib/active_storage/service/mirror_service.rb b/activestorage/lib/active_storage/service/mirror_service.rb index 39e922f7ab..6002ef5a00 100644 --- a/activestorage/lib/active_storage/service/mirror_service.rb +++ b/activestorage/lib/active_storage/service/mirror_service.rb @@ -9,7 +9,7 @@ module ActiveStorage class Service::MirrorService < Service attr_reader :primary, :mirrors - delegate :download, :exist?, :url, to: :primary + delegate :download, :download_chunk, :exist?, :url, to: :primary # Stitch together from named services. def self.build(primary:, mirrors:, configurator:, **options) #:nodoc: @@ -35,6 +35,11 @@ module ActiveStorage perform_across_services :delete, key end + # Delete files at keys starting with the +prefix+ on all services. + def delete_prefixed(prefix) + perform_across_services :delete_prefixed, prefix + end + private def each_service(&block) [ primary, *mirrors ].each(&block) diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index e074269353..0286e7ff21 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -9,15 +9,15 @@ module ActiveStorage class Service::S3Service < Service attr_reader :client, :bucket, :upload_options - def initialize(access_key_id:, secret_access_key:, region:, bucket:, upload: {}, **options) - @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region, **options) + def initialize(bucket:, upload: {}, **options) + @client = Aws::S3::Resource.new(**options) @bucket = @client.bucket(bucket) @upload_options = upload end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do begin object_for(key).put(upload_options.merge(body: io, content_md5: checksum)) rescue Aws::S3::Errors::BadDigest @@ -26,26 +26,38 @@ module ActiveStorage end end - def download(key) + def download(key, &block) if block_given? - instrument :streaming_download, key do + instrument :streaming_download, key: key do stream(key, &block) end else - instrument :download, key do - object_for(key).get.body.read.force_encoding(Encoding::BINARY) + instrument :download, key: key do + object_for(key).get.body.string.force_encoding(Encoding::BINARY) end end end + def download_chunk(key, range) + instrument :download_chunk, key: key, range: range do + object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.read.force_encoding(Encoding::BINARY) + end + end + def delete(key) - instrument :delete, key do + instrument :delete, key: key do object_for(key).delete end end + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + bucket.objects(prefix: prefix).batch_delete! + end + end + def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = object_for(key).exists? payload[:exist] = answer answer @@ -53,9 +65,9 @@ module ActiveStorage end def url(key, expires_in:, filename:, disposition:, content_type:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = object_for(key).presigned_url :get, expires_in: expires_in.to_i, - response_content_disposition: disposition, + response_content_disposition: content_disposition_with(type: disposition, filename: filename), response_content_type: content_type payload[:url] = generated_url @@ -65,7 +77,7 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i, content_type: content_type, content_length: content_length, content_md5: checksum @@ -85,14 +97,14 @@ module ActiveStorage end # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, options = {}, &block) + def stream(key) object = object_for(key) chunk_size = 5.megabytes offset = 0 while offset < object.content_length - yield object.read(options.merge(range: "bytes=#{offset}-#{offset + chunk_size - 1}")) + yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.read.force_encoding(Encoding::BINARY) offset += chunk_size end end diff --git a/activestorage/lib/tasks/activestorage.rake b/activestorage/lib/tasks/activestorage.rake index ef923e5926..296e91afa1 100644 --- a/activestorage/lib/tasks/activestorage.rake +++ b/activestorage/lib/tasks/activestorage.rake @@ -3,6 +3,10 @@ namespace :active_storage do desc "Copy over the migration needed to the application" task install: :environment do - Rake::Task["active_storage:install:migrations"].invoke + if Rake::Task.task_defined?("active_storage:install:migrations") + Rake::Task["active_storage:install:migrations"].invoke + else + Rake::Task["app:active_storage:install:migrations"].invoke + end end end |