diff options
Diffstat (limited to 'activestorage/lib/active_storage')
9 files changed, 87 insertions, 19 deletions
diff --git a/activestorage/lib/active_storage/analyzer/image_analyzer.rb b/activestorage/lib/active_storage/analyzer/image_analyzer.rb index c8bc8fe953..bd1bef3076 100644 --- a/activestorage/lib/active_storage/analyzer/image_analyzer.rb +++ b/activestorage/lib/active_storage/analyzer/image_analyzer.rb @@ -43,6 +43,9 @@ module ActiveStorage rescue LoadError logger.info "Skipping image analysis because the mini_magick gem isn't installed" {} + rescue MiniMagick::Error => error + logger.error "Skipping image analysis due to an ImageMagick error: #{error.message}" + {} end def rotated_image?(image) diff --git a/activestorage/lib/active_storage/attached/model.rb b/activestorage/lib/active_storage/attached/model.rb index ae7f0685f2..06864a846f 100644 --- a/activestorage/lib/active_storage/attached/model.rb +++ b/activestorage/lib/active_storage/attached/model.rb @@ -93,12 +93,19 @@ module ActiveStorage end def #{name}=(attachables) - attachment_changes["#{name}"] = - if attachables.nil? || Array(attachables).none? - ActiveStorage::Attached::Changes::DeleteMany.new("#{name}", self) - else - ActiveStorage::Attached::Changes::CreateMany.new("#{name}", self, attachables) + if ActiveStorage.replace_on_assign_to_many + attachment_changes["#{name}"] = + if attachables.nil? || Array(attachables).none? + ActiveStorage::Attached::Changes::DeleteMany.new("#{name}", self) + else + ActiveStorage::Attached::Changes::CreateMany.new("#{name}", self, attachables) + end + else + if !attachables.nil? || Array(attachables).any? + attachment_changes["#{name}"] = + ActiveStorage::Attached::Changes::CreateMany.new("#{name}", self, #{name}.blobs + attachables) end + end end CODE diff --git a/activestorage/lib/active_storage/attached/one.rb b/activestorage/lib/active_storage/attached/one.rb index c039226fcd..003be1cb43 100644 --- a/activestorage/lib/active_storage/attached/one.rb +++ b/activestorage/lib/active_storage/attached/one.rb @@ -3,7 +3,7 @@ module ActiveStorage # Representation of a single attachment to a model. class Attached::One < Attached - delegate_missing_to :attachment + delegate_missing_to :attachment, allow_nil: true # Returns the associated attachment record. # diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb index cbb205627e..9d9cd02d12 100644 --- a/activestorage/lib/active_storage/engine.rb +++ b/activestorage/lib/active_storage/engine.rb @@ -24,7 +24,7 @@ module ActiveStorage config.active_storage.previewers = [ ActiveStorage::Previewer::PopplerPDFPreviewer, ActiveStorage::Previewer::MuPDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ] config.active_storage.analyzers = [ ActiveStorage::Analyzer::ImageAnalyzer, ActiveStorage::Analyzer::VideoAnalyzer ] config.active_storage.paths = ActiveSupport::OrderedOptions.new - config.active_storage.queues = ActiveSupport::OrderedOptions.new + config.active_storage.queues = ActiveSupport::InheritableOptions.new(mirror: :active_storage_mirror) config.active_storage.variable_content_types = %w( image/png @@ -73,12 +73,15 @@ module ActiveStorage ActiveStorage.analyzers = app.config.active_storage.analyzers || [] ActiveStorage.paths = app.config.active_storage.paths || {} ActiveStorage.routes_prefix = app.config.active_storage.routes_prefix || "/rails/active_storage" + ActiveStorage.draw_routes = app.config.active_storage.draw_routes != false ActiveStorage.variable_content_types = app.config.active_storage.variable_content_types || [] ActiveStorage.content_types_to_serve_as_binary = app.config.active_storage.content_types_to_serve_as_binary || [] ActiveStorage.service_urls_expire_in = app.config.active_storage.service_urls_expire_in || 5.minutes ActiveStorage.content_types_allowed_inline = app.config.active_storage.content_types_allowed_inline || [] ActiveStorage.binary_content_type = app.config.active_storage.binary_content_type || "application/octet-stream" + + ActiveStorage.replace_on_assign_to_many = app.config.active_storage.replace_on_assign_to_many || false end end @@ -130,7 +133,7 @@ module ActiveStorage "config.active_storage.queue is deprecated and will be removed in Rails 6.1. " \ "Set config.active_storage.queues.purge and config.active_storage.queues.analysis instead." - ActiveStorage.queues = { purge: queue, analysis: queue } + ActiveStorage.queues = { purge: queue, analysis: queue, mirror: queue } else ActiveStorage.queues = app.config.active_storage.queues || {} end diff --git a/activestorage/lib/active_storage/log_subscriber.rb b/activestorage/lib/active_storage/log_subscriber.rb index 6c0b4c30e7..f823f31093 100644 --- a/activestorage/lib/active_storage/log_subscriber.rb +++ b/activestorage/lib/active_storage/log_subscriber.rb @@ -32,6 +32,12 @@ module ActiveStorage debug event, color("Generated URL for file at key: #{key_in(event)} (#{event.payload[:url]})", BLUE) end + def service_mirror(event) + message = "Mirrored file at key: #{key_in(event)}" + message += " (checksum: #{event.payload[:checksum]})" if event.payload[:checksum] + debug event, color(message, GREEN) + end + def logger ActiveStorage.logger end diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index 993cc0e5f7..8d77e9b20f 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -17,10 +17,10 @@ module ActiveStorage @container = container end - def upload(key, io, checksum: nil, **) + def upload(key, io, checksum: nil, content_type: nil, **) instrument :upload, key: key, checksum: checksum do handle_errors do - blobs.create_block_blob(container, key, IO.try_convert(io) || io, content_md5: checksum) + blobs.create_block_blob(container, key, IO.try_convert(io) || io, content_md5: checksum, content_type: content_type) end end end diff --git a/activestorage/lib/active_storage/service/disk_service.rb b/activestorage/lib/active_storage/service/disk_service.rb index 67892d43b2..764a447c69 100644 --- a/activestorage/lib/active_storage/service/disk_service.rb +++ b/activestorage/lib/active_storage/service/disk_service.rb @@ -84,8 +84,12 @@ module ActiveStorage purpose: :blob_key } ) + current_uri = URI.parse(current_host) + generated_url = url_helpers.rails_disk_service_url(verified_key_with_expiration, - host: current_host, + protocol: current_uri.scheme, + host: current_uri.host, + port: current_uri.port, disposition: content_disposition, content_type: content_type, filename: filename diff --git a/activestorage/lib/active_storage/service/mirror_service.rb b/activestorage/lib/active_storage/service/mirror_service.rb index aa41df304e..c44bd1f360 100644 --- a/activestorage/lib/active_storage/service/mirror_service.rb +++ b/activestorage/lib/active_storage/service/mirror_service.rb @@ -4,12 +4,17 @@ require "active_support/core_ext/module/delegation" module ActiveStorage # Wraps a set of mirror services and provides a single ActiveStorage::Service object that will all - # have the files uploaded to them. A +primary+ service is designated to answer calls to +download+, +exists?+, - # and +url+. + # have the files uploaded to them. A +primary+ service is designated to answer calls to: + # * +download+ + # * +exists?+ + # * +url+ + # * +url_for_direct_upload+ + # * +headers_for_direct_upload+ class Service::MirrorService < Service attr_reader :primary, :mirrors - delegate :download, :download_chunk, :exist?, :url, :path_for, to: :primary + delegate :download, :download_chunk, :exist?, :url, + :url_for_direct_upload, :headers_for_direct_upload, :path_for, to: :primary # Stitch together from named services. def self.build(primary:, mirrors:, configurator:, **options) #:nodoc: @@ -26,7 +31,8 @@ module ActiveStorage # ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError. def upload(key, io, checksum: nil, **options) each_service.collect do |service| - service.upload key, io.tap(&:rewind), checksum: checksum, **options + io.rewind + service.upload key, io, checksum: checksum, **options end end @@ -40,6 +46,21 @@ module ActiveStorage perform_across_services :delete_prefixed, prefix end + + # Copy the file at the +key+ from the primary service to each of the mirrors where it doesn't already exist. + def mirror(key, checksum:) + instrument :mirror, key: key, checksum: checksum do + if (mirrors_in_need_of_mirroring = mirrors.select { |service| !service.exist?(key) }).any? + primary.open(key, checksum: checksum) do |io| + mirrors_in_need_of_mirroring.each do |service| + io.rewind + service.upload key, io, checksum: checksum + end + end + end + end + end + private def each_service(&block) [ primary, *mirrors ].each(&block) diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index c7e4ec96a2..e4bd57048a 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +gem "aws-sdk-s3", "~> 1.14" + require "aws-sdk-s3" require "active_support/core_ext/numeric/bytes" @@ -7,20 +9,24 @@ module ActiveStorage # Wraps the Amazon Simple Storage Service (S3) as an Active Storage service. # See ActiveStorage::Service for the generic API documentation that applies to all services. class Service::S3Service < Service - attr_reader :client, :bucket, :upload_options + attr_reader :client, :bucket + attr_reader :multipart_upload_threshold, :upload_options def initialize(bucket:, upload: {}, **options) @client = Aws::S3::Resource.new(**options) @bucket = @client.bucket(bucket) + @multipart_upload_threshold = upload.fetch(:multipart_threshold, 100.megabytes) @upload_options = upload end def upload(key, io, checksum: nil, content_type: nil, **) instrument :upload, key: key, checksum: checksum do - object_for(key).put(upload_options.merge(body: io, content_md5: checksum, content_type: content_type)) - rescue Aws::S3::Errors::BadDigest - raise ActiveStorage::IntegrityError + if io.size < multipart_upload_threshold + upload_with_single_part key, io, checksum: checksum, content_type: content_type + else + upload_with_multipart key, io, content_type: content_type + end end end @@ -94,6 +100,24 @@ module ActiveStorage end private + MAXIMUM_UPLOAD_PARTS_COUNT = 10000 + MINIMUM_UPLOAD_PART_SIZE = 5.megabytes + + def upload_with_single_part(key, io, checksum: nil, content_type: nil) + object_for(key).put(body: io, content_md5: checksum, content_type: content_type, **upload_options) + rescue Aws::S3::Errors::BadDigest + raise ActiveStorage::IntegrityError + end + + def upload_with_multipart(key, io, content_type: nil) + part_size = [ io.size.fdiv(MAXIMUM_UPLOAD_PARTS_COUNT).ceil, MINIMUM_UPLOAD_PART_SIZE ].max + + object_for(key).upload_stream(content_type: content_type, part_size: part_size, **upload_options) do |out| + IO.copy_stream(io, out) + end + end + + def object_for(key) bucket.object(key) end |