diff options
Diffstat (limited to 'activestorage/lib')
22 files changed, 304 insertions, 100 deletions
diff --git a/activestorage/lib/active_storage.rb b/activestorage/lib/active_storage.rb index d1ff6b7032..e1bd974853 100644 --- a/activestorage/lib/active_storage.rb +++ b/activestorage/lib/active_storage.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true #-- -# Copyright (c) 2017 David Heinemeier Hansson +# Copyright (c) 2017-2018 David Heinemeier Hansson, Basecamp # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -26,7 +26,11 @@ require "active_record" require "active_support" require "active_support/rails" + require "active_storage/version" +require "active_storage/errors" + +require "marcel" module ActiveStorage extend ActiveSupport::Autoload @@ -41,4 +45,7 @@ module ActiveStorage mattr_accessor :queue mattr_accessor :previewers, default: [] mattr_accessor :analyzers, default: [] + mattr_accessor :paths, default: {} + mattr_accessor :variable_content_types, default: [] + mattr_accessor :content_types_to_serve_as_binary, default: [] end diff --git a/activestorage/lib/active_storage/analyzer.rb b/activestorage/lib/active_storage/analyzer.rb index 837785a12b..7c4168c1a0 100644 --- a/activestorage/lib/active_storage/analyzer.rb +++ b/activestorage/lib/active_storage/analyzer.rb @@ -26,7 +26,7 @@ module ActiveStorage end private - def logger + def logger #:doc: ActiveStorage.logger end end diff --git a/activestorage/lib/active_storage/analyzer/image_analyzer.rb b/activestorage/lib/active_storage/analyzer/image_analyzer.rb index 25e0251e6e..3b39de91be 100644 --- a/activestorage/lib/active_storage/analyzer/image_analyzer.rb +++ b/activestorage/lib/active_storage/analyzer/image_analyzer.rb @@ -3,14 +3,15 @@ module ActiveStorage # Extracts width and height in pixels from an image blob. # + # If the image contains EXIF data indicating its angle is 90 or 270 degrees, its width and height are swapped for convenience. + # # Example: # # ActiveStorage::Analyzer::ImageAnalyzer.new(blob).metadata # # => { width: 4104, height: 2736 } # # This analyzer relies on the third-party {MiniMagick}[https://github.com/minimagick/minimagick] gem. MiniMagick requires - # the {ImageMagick}[http://www.imagemagick.org] system library. These libraries are not provided by Rails; you must - # install them yourself to use this analyzer. + # the {ImageMagick}[http://www.imagemagick.org] system library. class Analyzer::ImageAnalyzer < Analyzer def self.accept?(blob) blob.image? @@ -18,7 +19,11 @@ module ActiveStorage def metadata read_image do |image| - { width: image.width, height: image.height } + if rotated_image?(image) + { width: image.height, height: image.width } + else + { width: image.width, height: image.height } + end end rescue LoadError logger.info "Skipping image analysis because the mini_magick gem isn't installed" @@ -32,5 +37,9 @@ module ActiveStorage yield MiniMagick::Image.new(file.path) end end + + def rotated_image?(image) + %w[ RightTop LeftBottom ].include?(image["%[orientation]"]) + end end end diff --git a/activestorage/lib/active_storage/analyzer/video_analyzer.rb b/activestorage/lib/active_storage/analyzer/video_analyzer.rb index 408b5e58e9..656e362187 100644 --- a/activestorage/lib/active_storage/analyzer/video_analyzer.rb +++ b/activestorage/lib/active_storage/analyzer/video_analyzer.rb @@ -9,31 +9,40 @@ module ActiveStorage # * Height (pixels) # * Duration (seconds) # * Angle (degrees) - # * Aspect ratio + # * Display aspect ratio # # Example: # # ActiveStorage::VideoAnalyzer.new(blob).metadata - # # => { width: 640, height: 480, duration: 5.0, angle: 0, aspect_ratio: [4, 3] } + # # => { width: 640.0, height: 480.0, duration: 5.0, angle: 0, display_aspect_ratio: [4, 3] } # - # This analyzer requires the {ffmpeg}[https://www.ffmpeg.org] system library, which is not provided by Rails. You must - # install ffmpeg yourself to use this analyzer. + # When a video's angle is 90 or 270 degrees, its width and height are automatically swapped for convenience. + # + # This analyzer requires the {ffmpeg}[https://www.ffmpeg.org] system library, which is not provided by Rails. class Analyzer::VideoAnalyzer < Analyzer def self.accept?(blob) blob.video? end def metadata - { width: width, height: height, duration: duration, angle: angle, aspect_ratio: aspect_ratio }.compact + { width: width, height: height, duration: duration, angle: angle, display_aspect_ratio: display_aspect_ratio }.compact end private def width - Integer(video_stream["width"]) if video_stream["width"] + if rotated? + computed_height || encoded_height + else + encoded_width + end end def height - Integer(video_stream["height"]) if video_stream["height"] + if rotated? + encoded_width + else + computed_height || encoded_height + end end def duration @@ -44,12 +53,40 @@ module ActiveStorage Integer(tags["rotate"]) if tags["rotate"] end - def aspect_ratio + def display_aspect_ratio if descriptor = video_stream["display_aspect_ratio"] - descriptor.split(":", 2).collect(&:to_i) + if terms = descriptor.split(":", 2) + numerator = Integer(terms[0]) + denominator = Integer(terms[1]) + + [numerator, denominator] unless numerator == 0 + end + end + end + + + def rotated? + angle == 90 || angle == 270 + end + + def computed_height + if encoded_width && display_height_scale + encoded_width * display_height_scale end end + def encoded_width + @encoded_width ||= Float(video_stream["width"]) if video_stream["width"] + end + + def encoded_height + @encoded_height ||= Float(video_stream["height"]) if video_stream["height"] + end + + def display_height_scale + @display_height_scale ||= Float(display_aspect_ratio.last) / display_aspect_ratio.first if display_aspect_ratio + end + def tags @tags ||= video_stream["tags"] || {} @@ -68,12 +105,16 @@ module ActiveStorage end def probe_from(file) - IO.popen([ "ffprobe", "-print_format", "json", "-show_streams", "-v", "error", file.path ]) do |output| + IO.popen([ ffprobe_path, "-print_format", "json", "-show_streams", "-v", "error", file.path ]) do |output| JSON.parse(output.read) end rescue Errno::ENOENT logger.info "Skipping video analysis because ffmpeg isn't installed" {} end + + def ffprobe_path + ActiveStorage.paths[:ffprobe] || "ffprobe" + end end end diff --git a/activestorage/lib/active_storage/attached/macros.rb b/activestorage/lib/active_storage/attached/macros.rb index f0256718ac..c51efa9d6b 100644 --- a/activestorage/lib/active_storage/attached/macros.rb +++ b/activestorage/lib/active_storage/attached/macros.rb @@ -32,15 +32,19 @@ module ActiveStorage def #{name} @active_storage_attached_#{name} ||= ActiveStorage::Attached::One.new("#{name}", self, dependent: #{dependent == :purge_later ? ":purge_later" : "false"}) end + + def #{name}=(attachable) + #{name}.attach(attachable) + end CODE - has_one :"#{name}_attachment", -> { where(name: name) }, class_name: "ActiveStorage::Attachment", as: :record + has_one :"#{name}_attachment", -> { where(name: name) }, class_name: "ActiveStorage::Attachment", as: :record, inverse_of: :record has_one :"#{name}_blob", through: :"#{name}_attachment", class_name: "ActiveStorage::Blob", source: :blob scope :"with_attached_#{name}", -> { includes("#{name}_attachment": :blob) } if dependent == :purge_later - before_destroy { public_send(name).purge_later } + after_destroy_commit { public_send(name).purge_later } end end @@ -73,15 +77,19 @@ module ActiveStorage def #{name} @active_storage_attached_#{name} ||= ActiveStorage::Attached::Many.new("#{name}", self, dependent: #{dependent == :purge_later ? ":purge_later" : "false"}) end + + def #{name}=(attachables) + #{name}.attach(attachables) + end CODE - has_many :"#{name}_attachments", -> { where(name: name) }, as: :record, class_name: "ActiveStorage::Attachment" + has_many :"#{name}_attachments", -> { where(name: name) }, as: :record, class_name: "ActiveStorage::Attachment", inverse_of: :record has_many :"#{name}_blobs", through: :"#{name}_attachments", class_name: "ActiveStorage::Blob", source: :blob scope :"with_attached_#{name}", -> { includes("#{name}_attachments": :blob) } if dependent == :purge_later - before_destroy { public_send(name).purge_later } + after_destroy_commit { public_send(name).purge_later } end end end diff --git a/activestorage/lib/active_storage/attached/many.rb b/activestorage/lib/active_storage/attached/many.rb index 1e0657c33c..6eace65b79 100644 --- a/activestorage/lib/active_storage/attached/many.rb +++ b/activestorage/lib/active_storage/attached/many.rb @@ -13,7 +13,6 @@ module ActiveStorage end # Associates one or several attachments with the current record, saving them to the database. - # Examples: # # document.images.attach(params[:images]) # Array of ActionDispatch::Http::UploadedFile objects # document.images.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload @@ -21,7 +20,11 @@ module ActiveStorage # document.images.attach([ first_blob, second_blob ]) def attach(*attachables) attachables.flatten.collect do |attachable| - attachments.create!(name: name, blob: create_blob_from(attachable)) + if record.new_record? + attachments.build(record: record, blob: create_blob_from(attachable)) + else + attachments.create!(record: record, blob: create_blob_from(attachable)) + end end end @@ -36,6 +39,11 @@ module ActiveStorage attachments.any? end + # Deletes associated attachments without purging them, leaving their respective blobs in place. + def detach + attachments.destroy_all if attached? + end + # Directly purges each associated attachment (i.e. destroys the blobs and # attachments and deletes the files on the service). def purge diff --git a/activestorage/lib/active_storage/attached/one.rb b/activestorage/lib/active_storage/attached/one.rb index dc19512484..0244232b2c 100644 --- a/activestorage/lib/active_storage/attached/one.rb +++ b/activestorage/lib/active_storage/attached/one.rb @@ -14,7 +14,6 @@ module ActiveStorage end # Associates a given attachment with the current record, saving it to the database. - # Examples: # # person.avatar.attach(params[:avatar]) # ActionDispatch::Http::UploadedFile object # person.avatar.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload @@ -24,7 +23,7 @@ module ActiveStorage if attached? && dependent == :purge_later replace attachable else - write_attachment create_attachment_from(attachable) + write_attachment build_attachment_from(attachable) end end @@ -39,6 +38,14 @@ module ActiveStorage attachment.present? end + # Deletes the attachment without purging it, leaving its blob in place. + def detach + if attached? + attachment.destroy + write_attachment nil + end + end + # Directly purges the attachment (i.e. destroys the blob and # attachment and deletes the file on the service). def purge @@ -59,18 +66,14 @@ module ActiveStorage def replace(attachable) blob.tap do transaction do - destroy_attachment - write_attachment create_attachment_from(attachable) + detach + write_attachment build_attachment_from(attachable) end end.purge_later end - def destroy_attachment - attachment.destroy - end - - def create_attachment_from(attachable) - ActiveStorage::Attachment.create!(record: record, name: name, blob: create_blob_from(attachable)) + def build_attachment_from(attachable) + ActiveStorage::Attachment.new(record: record, name: name, blob: create_blob_from(attachable)) end def write_attachment(attachment) diff --git a/activestorage/lib/active_storage/downloading.rb b/activestorage/lib/active_storage/downloading.rb index 3dac6b116a..f2a1fffdcb 100644 --- a/activestorage/lib/active_storage/downloading.rb +++ b/activestorage/lib/active_storage/downloading.rb @@ -1,25 +1,37 @@ # frozen_string_literal: true +require "tmpdir" + module ActiveStorage module Downloading private # Opens a new tempfile in #tempdir and copies blob data into it. Yields the tempfile. - def download_blob_to_tempfile # :doc: - Tempfile.open("ActiveStorage", tempdir) do |file| + def download_blob_to_tempfile #:doc: + open_tempfile_for_blob do |file| download_blob_to file yield file end end + def open_tempfile_for_blob + tempfile = Tempfile.open([ "ActiveStorage", blob.filename.extension_with_delimiter ], tempdir) + + begin + yield tempfile + ensure + tempfile.close! + end + end + # Efficiently downloads blob data into the given file. - def download_blob_to(file) # :doc: + def download_blob_to(file) #:doc: file.binmode blob.download { |chunk| file.write(chunk) } file.rewind end # Returns the directory in which tempfiles should be opened. Defaults to +Dir.tmpdir+. - def tempdir # :doc: + def tempdir #:doc: Dir.tmpdir end end diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb index 6cf6635c4f..8ba32490b1 100644 --- a/activestorage/lib/active_storage/engine.rb +++ b/activestorage/lib/active_storage/engine.rb @@ -15,7 +15,20 @@ module ActiveStorage config.active_storage = ActiveSupport::OrderedOptions.new config.active_storage.previewers = [ ActiveStorage::Previewer::PDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ] - config.active_storage.analyzers = [ ActiveStorage::Analyzer::ImageAnalyzer, ActiveStorage::Analyzer::VideoAnalyzer ] + config.active_storage.analyzers = [ ActiveStorage::Analyzer::ImageAnalyzer, ActiveStorage::Analyzer::VideoAnalyzer ] + config.active_storage.paths = ActiveSupport::OrderedOptions.new + + config.active_storage.variable_content_types = %w( image/png image/gif image/jpg image/jpeg image/vnd.adobe.photoshop ) + config.active_storage.content_types_to_serve_as_binary = %w( + text/html + text/javascript + image/svg+xml + application/postscript + application/x-shockwave-flash + text/xml + application/xml + application/xhtml+xml + ) config.eager_load_namespaces << ActiveStorage @@ -25,6 +38,10 @@ module ActiveStorage ActiveStorage.queue = app.config.active_storage.queue ActiveStorage.previewers = app.config.active_storage.previewers || [] ActiveStorage.analyzers = app.config.active_storage.analyzers || [] + ActiveStorage.paths = app.config.active_storage.paths || {} + + ActiveStorage.variable_content_types = app.config.active_storage.variable_content_types || [] + ActiveStorage.content_types_to_serve_as_binary = app.config.active_storage.content_types_to_serve_as_binary || [] end end diff --git a/activestorage/lib/active_storage/errors.rb b/activestorage/lib/active_storage/errors.rb new file mode 100644 index 0000000000..f099b13f5b --- /dev/null +++ b/activestorage/lib/active_storage/errors.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module ActiveStorage + class InvariableError < StandardError; end + class UnpreviewableError < StandardError; end + class UnrepresentableError < StandardError; end +end diff --git a/activestorage/lib/active_storage/gem_version.rb b/activestorage/lib/active_storage/gem_version.rb index e1d7b3493a..492620731b 100644 --- a/activestorage/lib/active_storage/gem_version.rb +++ b/activestorage/lib/active_storage/gem_version.rb @@ -7,8 +7,8 @@ module ActiveStorage end module VERSION - MAJOR = 5 - MINOR = 2 + MAJOR = 6 + MINOR = 0 TINY = 0 PRE = "alpha" diff --git a/activestorage/lib/active_storage/log_subscriber.rb b/activestorage/lib/active_storage/log_subscriber.rb index 5cbf4bd1a5..a4e148c1a5 100644 --- a/activestorage/lib/active_storage/log_subscriber.rb +++ b/activestorage/lib/active_storage/log_subscriber.rb @@ -18,6 +18,10 @@ module ActiveStorage info event, color("Deleted file from key: #{key_in(event)}", RED) end + def service_delete_prefixed(event) + info event, color("Deleted files by key prefix: #{event.payload[:prefix]}", RED) + end + def service_exist(event) debug event, color("Checked if file exists at key: #{key_in(event)} (#{event.payload[:exist] ? "yes" : "no"})", BLUE) end diff --git a/activestorage/lib/active_storage/previewer.rb b/activestorage/lib/active_storage/previewer.rb index ed75bae3b5..cf19987d72 100644 --- a/activestorage/lib/active_storage/previewer.rb +++ b/activestorage/lib/active_storage/previewer.rb @@ -42,17 +42,33 @@ module ActiveStorage # end # # The output tempfile is opened in the directory returned by ActiveStorage::Downloading#tempdir. - def draw(*argv) # :doc: - Tempfile.open("ActiveStorage", tempdir) do |file| - capture(*argv, to: file) - yield file + def draw(*argv) #:doc: + ActiveSupport::Notifications.instrument("preview.active_storage") do + open_tempfile_for_drawing do |file| + capture(*argv, to: file) + yield file + end + end + end + + def open_tempfile_for_drawing + tempfile = Tempfile.open("ActiveStorage", tempdir) + + begin + yield tempfile + ensure + tempfile.close! end end def capture(*argv, to:) to.binmode - IO.popen(argv) { |out| IO.copy_stream(out, to) } + IO.popen(argv, err: File::NULL) { |out| IO.copy_stream(out, to) } to.rewind end + + def logger #:doc: + ActiveStorage.logger + end end end diff --git a/activestorage/lib/active_storage/previewer/pdf_previewer.rb b/activestorage/lib/active_storage/previewer/pdf_previewer.rb index a2f05c74a6..426ff51eb1 100644 --- a/activestorage/lib/active_storage/previewer/pdf_previewer.rb +++ b/activestorage/lib/active_storage/previewer/pdf_previewer.rb @@ -8,10 +8,19 @@ module ActiveStorage def preview download_blob_to_tempfile do |input| - draw "mutool", "draw", "-F", "png", "-o", "-", input.path, "1" do |output| + draw_first_page_from input do |output| yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" end end end + + private + def draw_first_page_from(file, &block) + draw mutool_path, "draw", "-F", "png", "-o", "-", file.path, "1", &block + end + + def mutool_path + ActiveStorage.paths[:mutool] || "mutool" + end end end diff --git a/activestorage/lib/active_storage/previewer/video_previewer.rb b/activestorage/lib/active_storage/previewer/video_previewer.rb index 49f128d142..2f28a3d341 100644 --- a/activestorage/lib/active_storage/previewer/video_previewer.rb +++ b/activestorage/lib/active_storage/previewer/video_previewer.rb @@ -16,8 +16,12 @@ module ActiveStorage private def draw_relevant_frame_from(file, &block) - draw "ffmpeg", "-i", file.path, "-y", "-vcodec", "png", + draw ffmpeg_path, "-i", file.path, "-y", "-vcodec", "png", "-vf", "thumbnail", "-vframes", "1", "-f", "image2", "-", &block end + + def ffmpeg_path + ActiveStorage.paths[:ffmpeg] || "ffmpeg" + end end end diff --git a/activestorage/lib/active_storage/service.rb b/activestorage/lib/active_storage/service.rb index aa150e4d8a..f2e1269f27 100644 --- a/activestorage/lib/active_storage/service.rb +++ b/activestorage/lib/active_storage/service.rb @@ -78,6 +78,11 @@ module ActiveStorage raise NotImplementedError end + # Delete files at keys starting with the +prefix+. + def delete_prefixed(prefix) + raise NotImplementedError + end + # Return +true+ if a file exists at the +key+. def exist?(key) raise NotImplementedError @@ -92,7 +97,7 @@ module ActiveStorage # Returns a signed, temporary URL that a direct upload file can be PUT to on the +key+. # The URL will be valid for the amount of seconds specified in +expires_in+. - # You most also provide the +content_type+, +content_length+, and +checksum+ of the file + # You must also provide the +content_type+, +content_length+, and +checksum+ of the file # that will be uploaded. All these attributes will be validated by the service upon upload. def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) raise NotImplementedError @@ -104,10 +109,10 @@ module ActiveStorage end private - def instrument(operation, key, payload = {}, &block) + def instrument(operation, payload = {}, &block) ActiveSupport::Notifications.instrument( "service_#{operation}.active_storage", - payload.merge(key: key, service: service_name), &block) + payload.merge(service: service_name), &block) end def service_name diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index f3877ad9c9..0a9eb7f23d 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -19,7 +19,7 @@ module ActiveStorage end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do begin blobs.create_block_blob(container, key, io, content_md5: checksum) rescue Azure::Core::Http::HTTPError @@ -30,11 +30,11 @@ module ActiveStorage def download(key, &block) if block_given? - instrument :streaming_download, key do + instrument :streaming_download, key: key do stream(key, &block) end else - instrument :download, key do + instrument :download, key: key do _, io = blobs.get_blob(container, key) io.force_encoding(Encoding::BINARY) end @@ -42,17 +42,33 @@ module ActiveStorage end def delete(key) - instrument :delete, key do + instrument :delete, key: key do begin blobs.delete_blob(container, key) rescue Azure::Core::Http::HTTPError - false + # Ignore files already deleted + end + end + end + + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + marker = nil + + loop do + results = blobs.list_blobs(container, prefix: prefix, marker: marker) + + results.each do |blob| + blobs.delete_blob(container, blob.name) + end + + break unless marker = results.continuation_token.presence end end end def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = blob_for(key).present? payload[:exist] = answer answer @@ -60,7 +76,7 @@ module ActiveStorage end def url(key, expires_in:, filename:, disposition:, content_type:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| base_url = url_for(key) generated_url = signer.signed_uri( URI(base_url), false, @@ -77,7 +93,7 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| base_url = url_for(key) generated_url = signer.signed_uri(URI(base_url), false, permissions: "rw", expiry: format_expiry(expires_in)).to_s diff --git a/activestorage/lib/active_storage/service/disk_service.rb b/activestorage/lib/active_storage/service/disk_service.rb index 52eaba4e7b..d17eea9046 100644 --- a/activestorage/lib/active_storage/service/disk_service.rb +++ b/activestorage/lib/active_storage/service/disk_service.rb @@ -9,14 +9,14 @@ module ActiveStorage # Wraps a local disk path as an Active Storage service. See ActiveStorage::Service for the generic API # documentation that applies to all services. class Service::DiskService < Service - attr_reader :root + attr_reader :root, :host - def initialize(root:) - @root = root + def initialize(root:, host: "http://localhost:3000") + @root, @host = root, host end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do IO.copy_stream(io, make_path_for(key)) ensure_integrity_of(key, checksum) if checksum end @@ -24,7 +24,7 @@ module ActiveStorage def download(key) if block_given? - instrument :streaming_download, key do + instrument :streaming_download, key: key do File.open(path_for(key), "rb") do |file| while data = file.read(64.kilobytes) yield data @@ -32,14 +32,14 @@ module ActiveStorage end end else - instrument :download, key do + instrument :download, key: key do File.binread path_for(key) end end end def delete(key) - instrument :delete, key do + instrument :delete, key: key do begin File.delete path_for(key) rescue Errno::ENOENT @@ -48,8 +48,16 @@ module ActiveStorage end end + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + Dir.glob(path_for("#{prefix}*")).each do |path| + FileUtils.rm_rf(path) + end + end + end + def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = File.exist? path_for(key) payload[:exist] = answer answer @@ -57,18 +65,17 @@ module ActiveStorage end def url(key, expires_in:, filename:, disposition:, content_type:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| verified_key_with_expiration = ActiveStorage.verifier.generate(key, expires_in: expires_in, purpose: :blob_key) generated_url = - if defined?(Rails.application) - Rails.application.routes.url_helpers.rails_disk_service_path \ - verified_key_with_expiration, - filename: filename, disposition: content_disposition_with(type: disposition, filename: filename), content_type: content_type - else - "/rails/active_storage/disk/#{verified_key_with_expiration}/#{filename}?content_type=#{content_type}" \ - "&disposition=#{content_disposition_with(type: disposition, filename: filename)}" - end + Rails.application.routes.url_helpers.rails_disk_service_url( + verified_key_with_expiration, + filename: filename, + disposition: content_disposition_with(type: disposition, filename: filename), + content_type: content_type, + host: host + ) payload[:url] = generated_url @@ -77,7 +84,7 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| verified_token_with_expiration = ActiveStorage.verifier.generate( { key: key, @@ -89,12 +96,7 @@ module ActiveStorage purpose: :blob_token } ) - generated_url = - if defined?(Rails.application) - Rails.application.routes.url_helpers.update_rails_disk_service_path verified_token_with_expiration - else - "/rails/active_storage/disk/#{verified_token_with_expiration}" - end + generated_url = Rails.application.routes.url_helpers.update_rails_disk_service_url(verified_token_with_expiration, host: host) payload[:url] = generated_url diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb index b4ffeeeb8a..6f6f4105fe 100644 --- a/activestorage/lib/active_storage/service/gcs_service.rb +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +gem "google-cloud-storage", "~> 1.8" + require "google/cloud/storage" require "active_support/core_ext/object/to_query" @@ -7,17 +9,20 @@ module ActiveStorage # Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API # documentation that applies to all services. class Service::GCSService < Service - attr_reader :client, :bucket - - def initialize(project:, keyfile:, bucket:, **options) - @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile, **options) - @bucket = @client.bucket(bucket) + def initialize(**config) + @config = config end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do begin - bucket.create_file(io, key, md5: checksum) + # The official GCS client library doesn't allow us to create a file with no Content-Type metadata. + # We need the file we create to have no Content-Type so we can control it via the response-content-type + # param in signed URLs. Workaround: let the GCS client create the file with an inferred + # Content-Type (usually "application/octet-stream") then clear it. + bucket.create_file(io, key, md5: checksum).update do |file| + file.content_type = nil + end rescue Google::Cloud::InvalidArgumentError raise ActiveStorage::IntegrityError end @@ -26,7 +31,7 @@ module ActiveStorage # FIXME: Download in chunks when given a block. def download(key) - instrument :download, key do + instrument :download, key: key do io = file_for(key).download io.rewind @@ -39,7 +44,7 @@ module ActiveStorage end def delete(key) - instrument :delete, key do + instrument :delete, key: key do begin file_for(key).delete rescue Google::Cloud::NotFoundError @@ -48,8 +53,14 @@ module ActiveStorage end end + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + bucket.files(prefix: prefix).all(&:delete) + end + end + def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = file_for(key).exists? payload[:exist] = answer answer @@ -57,7 +68,7 @@ module ActiveStorage end def url(key, expires_in:, filename:, content_type:, disposition:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = file_for(key).signed_url expires: expires_in, query: { "response-content-disposition" => content_disposition_with(type: disposition, filename: filename), "response-content-type" => content_type @@ -70,7 +81,7 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = bucket.signed_url key, method: "PUT", expires: expires_in, content_type: content_type, content_md5: checksum @@ -85,8 +96,18 @@ module ActiveStorage end private + attr_reader :config + def file_for(key) bucket.file(key, skip_lookup: true) end + + def bucket + @bucket ||= client.bucket(config.fetch(:bucket)) + end + + def client + @client ||= Google::Cloud::Storage.new(config.except(:bucket)) + end end end diff --git a/activestorage/lib/active_storage/service/mirror_service.rb b/activestorage/lib/active_storage/service/mirror_service.rb index 39e922f7ab..7eca8ce7f4 100644 --- a/activestorage/lib/active_storage/service/mirror_service.rb +++ b/activestorage/lib/active_storage/service/mirror_service.rb @@ -35,6 +35,11 @@ module ActiveStorage perform_across_services :delete, key end + # Delete files at keys starting with the +prefix+ on all services. + def delete_prefixed(prefix) + perform_across_services :delete_prefixed, prefix + end + private def each_service(&block) [ primary, *mirrors ].each(&block) diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index 6957119780..c95672f338 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -17,7 +17,7 @@ module ActiveStorage end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do begin object_for(key).put(upload_options.merge(body: io, content_md5: checksum)) rescue Aws::S3::Errors::BadDigest @@ -28,24 +28,30 @@ module ActiveStorage def download(key, &block) if block_given? - instrument :streaming_download, key do + instrument :streaming_download, key: key do stream(key, &block) end else - instrument :download, key do + instrument :download, key: key do object_for(key).get.body.read.force_encoding(Encoding::BINARY) end end end def delete(key) - instrument :delete, key do + instrument :delete, key: key do object_for(key).delete end end + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + bucket.objects(prefix: prefix).batch_delete! + end + end + def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = object_for(key).exists? payload[:exist] = answer answer @@ -53,7 +59,7 @@ module ActiveStorage end def url(key, expires_in:, filename:, disposition:, content_type:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = object_for(key).presigned_url :get, expires_in: expires_in.to_i, response_content_disposition: content_disposition_with(type: disposition, filename: filename), response_content_type: content_type @@ -65,7 +71,7 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i, content_type: content_type, content_length: content_length, content_md5: checksum diff --git a/activestorage/lib/tasks/activestorage.rake b/activestorage/lib/tasks/activestorage.rake index ef923e5926..296e91afa1 100644 --- a/activestorage/lib/tasks/activestorage.rake +++ b/activestorage/lib/tasks/activestorage.rake @@ -3,6 +3,10 @@ namespace :active_storage do desc "Copy over the migration needed to the application" task install: :environment do - Rake::Task["active_storage:install:migrations"].invoke + if Rake::Task.task_defined?("active_storage:install:migrations") + Rake::Task["active_storage:install:migrations"].invoke + else + Rake::Task["app:active_storage:install:migrations"].invoke + end end end |