From 9ec67362054e874ed905310a79b670941fa397af Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Fri, 3 Nov 2017 11:29:21 -0400 Subject: Permit configuring Active Storage's job queue --- activestorage/app/jobs/active_storage/analyze_job.rb | 2 +- activestorage/app/jobs/active_storage/base_job.rb | 5 +++++ activestorage/app/jobs/active_storage/purge_job.rb | 2 +- activestorage/lib/active_storage.rb | 1 + activestorage/lib/active_storage/engine.rb | 19 +++++-------------- 5 files changed, 13 insertions(+), 16 deletions(-) create mode 100644 activestorage/app/jobs/active_storage/base_job.rb (limited to 'activestorage') diff --git a/activestorage/app/jobs/active_storage/analyze_job.rb b/activestorage/app/jobs/active_storage/analyze_job.rb index a11a73d030..2a952f9f74 100644 --- a/activestorage/app/jobs/active_storage/analyze_job.rb +++ b/activestorage/app/jobs/active_storage/analyze_job.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Provides asynchronous analysis of ActiveStorage::Blob records via ActiveStorage::Blob#analyze_later. -class ActiveStorage::AnalyzeJob < ActiveJob::Base +class ActiveStorage::AnalyzeJob < ActiveStorage::BaseJob def perform(blob) blob.analyze end diff --git a/activestorage/app/jobs/active_storage/base_job.rb b/activestorage/app/jobs/active_storage/base_job.rb new file mode 100644 index 0000000000..6caab42a2d --- /dev/null +++ b/activestorage/app/jobs/active_storage/base_job.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class ActiveStorage::BaseJob < ActiveJob::Base + queue_as { ActiveStorage.queue } +end diff --git a/activestorage/app/jobs/active_storage/purge_job.rb b/activestorage/app/jobs/active_storage/purge_job.rb index 188840f702..98874d2250 100644 --- a/activestorage/app/jobs/active_storage/purge_job.rb +++ b/activestorage/app/jobs/active_storage/purge_job.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Provides asynchronous purging of ActiveStorage::Blob records via ActiveStorage::Blob#purge_later. -class ActiveStorage::PurgeJob < ActiveJob::Base +class ActiveStorage::PurgeJob < ActiveStorage::BaseJob # FIXME: Limit this to a custom ActiveStorage error retry_on StandardError diff --git a/activestorage/lib/active_storage.rb b/activestorage/lib/active_storage.rb index cfdb2a8acd..d1ff6b7032 100644 --- a/activestorage/lib/active_storage.rb +++ b/activestorage/lib/active_storage.rb @@ -38,6 +38,7 @@ module ActiveStorage mattr_accessor :logger mattr_accessor :verifier + mattr_accessor :queue mattr_accessor :previewers, default: [] mattr_accessor :analyzers, default: [] end diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb index a01a14cd83..6cf6635c4f 100644 --- a/activestorage/lib/active_storage/engine.rb +++ b/activestorage/lib/active_storage/engine.rb @@ -19,9 +19,12 @@ module ActiveStorage config.eager_load_namespaces << ActiveStorage - initializer "active_storage.logger" do + initializer "active_storage.configs" do config.after_initialize do |app| - ActiveStorage.logger = app.config.active_storage.logger || Rails.logger + ActiveStorage.logger = app.config.active_storage.logger || Rails.logger + ActiveStorage.queue = app.config.active_storage.queue + ActiveStorage.previewers = app.config.active_storage.previewers || [] + ActiveStorage.analyzers = app.config.active_storage.analyzers || [] end end @@ -65,17 +68,5 @@ module ActiveStorage end end end - - initializer "active_storage.previewers" do - config.after_initialize do |app| - ActiveStorage.previewers = app.config.active_storage.previewers || [] - end - end - - initializer "active_storage.analyzers" do - config.after_initialize do |app| - ActiveStorage.analyzers = app.config.active_storage.analyzers || [] - end - end end end -- cgit v1.2.3 From 86938c495e282e6a61c16a9e1d77582e22c0a4fc Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Mon, 6 Nov 2017 21:29:37 -0500 Subject: Fix streaming downloads from S3/Azure Storage Closes #31073. --- .../lib/active_storage/service/azure_storage_service.rb | 8 ++++---- activestorage/lib/active_storage/service/s3_service.rb | 6 +++--- activestorage/test/service/shared_service_tests.rb | 10 ++++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) (limited to 'activestorage') diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index 27dd192ce6..f3877ad9c9 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -28,7 +28,7 @@ module ActiveStorage end end - def download(key) + def download(key, &block) if block_given? instrument :streaming_download, key do stream(key, &block) @@ -108,15 +108,15 @@ module ActiveStorage end # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, options = {}, &block) + def stream(key) blob = blob_for(key) chunk_size = 5.megabytes offset = 0 while offset < blob.properties[:content_length] - _, io = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) - yield io + _, chunk = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) + yield chunk.force_encoding(Encoding::BINARY) offset += chunk_size end end diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index 3e93cdd072..958b172efb 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -26,7 +26,7 @@ module ActiveStorage end end - def download(key) + def download(key, &block) if block_given? instrument :streaming_download, key do stream(key, &block) @@ -85,14 +85,14 @@ module ActiveStorage end # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, options = {}, &block) + def stream(key, &block) object = object_for(key) chunk_size = 5.megabytes offset = 0 while offset < object.content_length - yield object.read(options.merge(range: "bytes=#{offset}-#{offset + chunk_size - 1}")) + yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.read.force_encoding(Encoding::BINARY) offset += chunk_size end end diff --git a/activestorage/test/service/shared_service_tests.rb b/activestorage/test/service/shared_service_tests.rb index a9e1cb6ce9..ade91ab89a 100644 --- a/activestorage/test/service/shared_service_tests.rb +++ b/activestorage/test/service/shared_service_tests.rb @@ -50,6 +50,16 @@ module ActiveStorage::Service::SharedServiceTests assert_equal FIXTURE_DATA, @service.download(FIXTURE_KEY) end + test "downloading in chunks" do + chunks = [] + + @service.download(FIXTURE_KEY) do |chunk| + chunks << chunk + end + + assert_equal [ FIXTURE_DATA ], chunks + end + test "existing" do assert @service.exist?(FIXTURE_KEY) assert_not @service.exist?(FIXTURE_KEY + "nonsense") -- cgit v1.2.3 From daf77db65d9d5e295ee3ba86605988875cb834e4 Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Tue, 7 Nov 2017 09:06:23 -0500 Subject: Remove needless block parameter --- activestorage/lib/active_storage/service/s3_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'activestorage') diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index 958b172efb..6957119780 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -85,7 +85,7 @@ module ActiveStorage end # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, &block) + def stream(key) object = object_for(key) chunk_size = 5.megabytes -- cgit v1.2.3