From 7d14bda3a7c74ec0262fa021e641a4235f5405d5 Mon Sep 17 00:00:00 2001 From: Yoshiyuki Hirano Date: Sun, 17 Sep 2017 10:06:47 +0900 Subject: Fix file missing in activestorage's example code [ci skip] * File.open("~/face.jpg") raise error: `Errno::ENOENT: No such file or directory @ rb_sysopen - ~/face.jpg` --- activestorage/lib/active_storage/attached/many.rb | 2 +- activestorage/lib/active_storage/attached/one.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/attached/many.rb b/activestorage/lib/active_storage/attached/many.rb index 59b7d7d559..1e0657c33c 100644 --- a/activestorage/lib/active_storage/attached/many.rb +++ b/activestorage/lib/active_storage/attached/many.rb @@ -17,7 +17,7 @@ module ActiveStorage # # document.images.attach(params[:images]) # Array of ActionDispatch::Http::UploadedFile objects # document.images.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload - # document.images.attach(io: File.open("~/racecar.jpg"), filename: "racecar.jpg", content_type: "image/jpg") + # document.images.attach(io: File.open("/path/to/racecar.jpg"), filename: "racecar.jpg", content_type: "image/jpg") # document.images.attach([ first_blob, second_blob ]) def attach(*attachables) attachables.flatten.collect do |attachable| diff --git a/activestorage/lib/active_storage/attached/one.rb b/activestorage/lib/active_storage/attached/one.rb index ac90f32d95..c66be08f58 100644 --- a/activestorage/lib/active_storage/attached/one.rb +++ b/activestorage/lib/active_storage/attached/one.rb @@ -18,7 +18,7 @@ module ActiveStorage # # person.avatar.attach(params[:avatar]) # ActionDispatch::Http::UploadedFile object # person.avatar.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload - # person.avatar.attach(io: File.open("~/face.jpg"), filename: "face.jpg", content_type: "image/jpg") + # person.avatar.attach(io: File.open("/path/to/face.jpg"), filename: "face.jpg", content_type: "image/jpg") # person.avatar.attach(avatar_blob) # ActiveStorage::Blob object def attach(attachable) if attached? && dependent == :purge_later -- cgit v1.2.3 From 704bf9b9c64a2f92c6e72d74350bd8f00aaca5c5 Mon Sep 17 00:00:00 2001 From: Yoshiyuki Hirano Date: Sun, 17 Sep 2017 18:24:16 +0900 Subject: Add `with_attached_*` scope to `has_one_attached` macro * For avoiding N+1 problem, added `with_attached_*` scope to `has_one_attached` macro. --- activestorage/lib/active_storage/attached/macros.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/attached/macros.rb b/activestorage/lib/active_storage/attached/macros.rb index 35a081adc4..f0256718ac 100644 --- a/activestorage/lib/active_storage/attached/macros.rb +++ b/activestorage/lib/active_storage/attached/macros.rb @@ -12,6 +12,10 @@ module ActiveStorage # There is no column defined on the model side, Active Storage takes # care of the mapping between your records and the attachment. # + # To avoid N+1 queries, you can include the attached blobs in your query like so: + # + # User.with_attached_avatar + # # Under the covers, this relationship is implemented as a +has_one+ association to a # ActiveStorage::Attachment record and a +has_one-through+ association to a # ActiveStorage::Blob record. These associations are available as +avatar_attachment+ @@ -33,6 +37,8 @@ module ActiveStorage has_one :"#{name}_attachment", -> { where(name: name) }, class_name: "ActiveStorage::Attachment", as: :record has_one :"#{name}_blob", through: :"#{name}_attachment", class_name: "ActiveStorage::Blob", source: :blob + scope :"with_attached_#{name}", -> { includes("#{name}_attachment": :blob) } + if dependent == :purge_later before_destroy { public_send(name).purge_later } end -- cgit v1.2.3 From d30586211b41e018869a1a3f4e3af778a31591db Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Thu, 28 Sep 2017 16:43:37 -0400 Subject: Preview PDFs and videos --- activestorage/lib/active_storage/engine.rb | 10 ++++ activestorage/lib/active_storage/previewer.rb | 69 ++++++++++++++++++++++ .../lib/active_storage/previewer/pdf_previewer.rb | 17 ++++++ .../active_storage/previewer/video_previewer.rb | 17 ++++++ activestorage/lib/active_storage/service.rb | 7 +++ .../service/azure_storage_service.rb | 2 +- .../lib/active_storage/service/disk_service.rb | 5 +- .../lib/active_storage/service/gcs_service.rb | 11 +++- .../lib/active_storage/service/s3_service.rb | 2 +- 9 files changed, 133 insertions(+), 7 deletions(-) create mode 100644 activestorage/lib/active_storage/previewer.rb create mode 100644 activestorage/lib/active_storage/previewer/pdf_previewer.rb create mode 100644 activestorage/lib/active_storage/previewer/video_previewer.rb (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb index 590a36a30a..335eae8dd8 100644 --- a/activestorage/lib/active_storage/engine.rb +++ b/activestorage/lib/active_storage/engine.rb @@ -3,11 +3,15 @@ require "rails" require "active_storage" +require "active_storage/previewer/pdf_previewer" +require "active_storage/previewer/video_previewer" + module ActiveStorage class Engine < Rails::Engine # :nodoc: isolate_namespace ActiveStorage config.active_storage = ActiveSupport::OrderedOptions.new + config.active_storage.previewers = [ ActiveStorage::Previewer::PDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ] config.eager_load_namespaces << ActiveStorage @@ -59,5 +63,11 @@ module ActiveStorage end end end + + initializer "active_storage.previewers" do + config.after_initialize do |app| + ActiveStorage.previewers = app.config.active_storage.previewers || [] + end + end end end diff --git a/activestorage/lib/active_storage/previewer.rb b/activestorage/lib/active_storage/previewer.rb new file mode 100644 index 0000000000..c91f64ac65 --- /dev/null +++ b/activestorage/lib/active_storage/previewer.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module ActiveStorage + # This is an abstract base class for previewers, which generate images from blobs. See + # ActiveStorage::Previewer::PDFPreviewer and ActiveStorage::Previewer::VideoPreviewer for examples of + # concrete subclasses. + class Previewer + attr_reader :blob + + # Implement this method in a concrete subclass. Have it return true when given a blob from which + # the previewer can generate an image. + def self.accept?(blob) + false + end + + def initialize(blob) + @blob = blob + end + + # Override this method in a concrete subclass. Have it yield an attachable preview image (i.e. + # anything accepted by ActiveStorage::Attached::One#attach). + def preview + raise NotImplementedError + end + + private + # Downloads the blob to a new tempfile. Yields the tempfile. + # + # Use this method to get a tempfile that you can provide to a drawing command. + def open # :doc: + Tempfile.open("input") do |file| + download_blob_to file + yield file + end + end + + def download_blob_to(file) + file.binmode + blob.download { |chunk| file.write(chunk) } + file.rewind + end + + + # Executes a system command, capturing its binary output in a tempfile. Yields the tempfile. + # + # Use this method to shell out to system libraries (e.g. mupdf or ffmpeg) for preview image + # generation. The resulting tempfile can be used as the +:io+ value in an attachable Hash: + # + # def preview + # open do |input| + # draw "my-drawing-command", input.path, "--format", "png", "-" do |output| + # yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + # end + # end + # end + def draw(*argv) # :doc: + Tempfile.open("output") do |file| + capture *argv, to: file + yield file + end + end + + def capture(*argv, to:) + to.binmode + IO.popen(argv) { |out| IO.copy_stream(out, to) } + to.rewind + end + end +end diff --git a/activestorage/lib/active_storage/previewer/pdf_previewer.rb b/activestorage/lib/active_storage/previewer/pdf_previewer.rb new file mode 100644 index 0000000000..31a2a8f120 --- /dev/null +++ b/activestorage/lib/active_storage/previewer/pdf_previewer.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module ActiveStorage + class Previewer::PDFPreviewer < Previewer + def self.accept?(blob) + blob.content_type == "application/pdf" + end + + def preview + open do |input| + draw "mutool", "draw", "-F", "png", "-o", "-", input.path, "1" do |output| + yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + end + end + end + end +end diff --git a/activestorage/lib/active_storage/previewer/video_previewer.rb b/activestorage/lib/active_storage/previewer/video_previewer.rb new file mode 100644 index 0000000000..840d87f100 --- /dev/null +++ b/activestorage/lib/active_storage/previewer/video_previewer.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module ActiveStorage + class Previewer::VideoPreviewer < Previewer + def self.accept?(blob) + blob.video? + end + + def preview + open do |input| + draw "ffmpeg", "-i", input.path, "-y", "-vcodec", "png", "-vf", "thumbnail", "-vframes", "1", "-f", "image2", "-" do |output| + yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" + end + end + end + end +end diff --git a/activestorage/lib/active_storage/service.rb b/activestorage/lib/active_storage/service.rb index b80fdea1ab..1f012da1e7 100644 --- a/activestorage/lib/active_storage/service.rb +++ b/activestorage/lib/active_storage/service.rb @@ -4,6 +4,7 @@ require "active_storage/log_subscriber" module ActiveStorage class IntegrityError < StandardError; end + # Abstract class serving as an interface for concrete services. # # The available services are: @@ -42,6 +43,8 @@ module ActiveStorage class_attribute :logger + class_attribute :url_expires_in, default: 5.minutes + class << self # Configure an Active Storage service by name from a set of configurations, # typically loaded from a YAML file. The Active Storage engine uses this @@ -113,5 +116,9 @@ module ActiveStorage # ActiveStorage::Service::DiskService => Disk self.class.name.split("::").third.remove("Service") end + + def content_disposition_with(type: "inline", filename:) + (type.to_s.presence_in(%w( attachment inline )) || "inline") + "; #{filename.parameters}" + end end end diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index 895cc9c2f1..27dd192ce6 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -66,7 +66,7 @@ module ActiveStorage URI(base_url), false, permissions: "r", expiry: format_expiry(expires_in), - content_disposition: disposition, + content_disposition: content_disposition_with(type: disposition, filename: filename), content_type: content_type ).to_s diff --git a/activestorage/lib/active_storage/service/disk_service.rb b/activestorage/lib/active_storage/service/disk_service.rb index f600753a08..52eaba4e7b 100644 --- a/activestorage/lib/active_storage/service/disk_service.rb +++ b/activestorage/lib/active_storage/service/disk_service.rb @@ -64,9 +64,10 @@ module ActiveStorage if defined?(Rails.application) Rails.application.routes.url_helpers.rails_disk_service_path \ verified_key_with_expiration, - filename: filename, disposition: disposition, content_type: content_type + filename: filename, disposition: content_disposition_with(type: disposition, filename: filename), content_type: content_type else - "/rails/active_storage/disk/#{verified_key_with_expiration}/#{filename}?content_type=#{content_type}&disposition=#{disposition}" + "/rails/active_storage/disk/#{verified_key_with_expiration}/#{filename}?content_type=#{content_type}" \ + "&disposition=#{content_disposition_with(type: disposition, filename: filename)}" end payload[:url] = generated_url diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb index 685dd61a0a..b4ffeeeb8a 100644 --- a/activestorage/lib/active_storage/service/gcs_service.rb +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -24,12 +24,17 @@ module ActiveStorage end end - # FIXME: Add streaming when given a block + # FIXME: Download in chunks when given a block. def download(key) instrument :download, key do io = file_for(key).download io.rewind - io.read + + if block_given? + yield io.read + else + io.read + end end end @@ -54,7 +59,7 @@ module ActiveStorage def url(key, expires_in:, filename:, content_type:, disposition:) instrument :url, key do |payload| generated_url = file_for(key).signed_url expires: expires_in, query: { - "response-content-disposition" => disposition, + "response-content-disposition" => content_disposition_with(type: disposition, filename: filename), "response-content-type" => content_type } diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index e074269353..3e93cdd072 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -55,7 +55,7 @@ module ActiveStorage def url(key, expires_in:, filename:, disposition:, content_type:) instrument :url, key do |payload| generated_url = object_for(key).presigned_url :get, expires_in: expires_in.to_i, - response_content_disposition: disposition, + response_content_disposition: content_disposition_with(type: disposition, filename: filename), response_content_type: content_type payload[:url] = generated_url -- cgit v1.2.3 From 83b7cb3a4711794ca1716d1b593643d2309f0019 Mon Sep 17 00:00:00 2001 From: Ryuta Kamizono Date: Sat, 30 Sep 2017 05:39:27 +0900 Subject: Fix "warning: `*' interpreted as argument prefix" --- activestorage/lib/active_storage/previewer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/previewer.rb b/activestorage/lib/active_storage/previewer.rb index c91f64ac65..930b376067 100644 --- a/activestorage/lib/active_storage/previewer.rb +++ b/activestorage/lib/active_storage/previewer.rb @@ -55,7 +55,7 @@ module ActiveStorage # end def draw(*argv) # :doc: Tempfile.open("output") do |file| - capture *argv, to: file + capture(*argv, to: file) yield file end end -- cgit v1.2.3 From 605484079d297d1ba6835628465be81f03c052ee Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Sun, 22 Oct 2017 13:16:59 -0400 Subject: Extract metadata from images and videos --- activestorage/lib/active_storage/analyzer.rb | 33 +++++++++ .../lib/active_storage/analyzer/image_analyzer.rb | 36 ++++++++++ .../lib/active_storage/analyzer/null_analyzer.rb | 13 ++++ .../lib/active_storage/analyzer/video_analyzer.rb | 79 ++++++++++++++++++++++ activestorage/lib/active_storage/attached/one.rb | 6 +- activestorage/lib/active_storage/downloading.rb | 21 ++++++ activestorage/lib/active_storage/engine.rb | 14 +++- activestorage/lib/active_storage/log_subscriber.rb | 2 +- activestorage/lib/active_storage/previewer.rb | 27 ++------ .../lib/active_storage/previewer/pdf_previewer.rb | 2 +- .../active_storage/previewer/video_previewer.rb | 10 ++- activestorage/lib/active_storage/service.rb | 2 - 12 files changed, 215 insertions(+), 30 deletions(-) create mode 100644 activestorage/lib/active_storage/analyzer.rb create mode 100644 activestorage/lib/active_storage/analyzer/image_analyzer.rb create mode 100644 activestorage/lib/active_storage/analyzer/null_analyzer.rb create mode 100644 activestorage/lib/active_storage/analyzer/video_analyzer.rb create mode 100644 activestorage/lib/active_storage/downloading.rb (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/analyzer.rb b/activestorage/lib/active_storage/analyzer.rb new file mode 100644 index 0000000000..837785a12b --- /dev/null +++ b/activestorage/lib/active_storage/analyzer.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require "active_storage/downloading" + +module ActiveStorage + # This is an abstract base class for analyzers, which extract metadata from blobs. See + # ActiveStorage::Analyzer::ImageAnalyzer for an example of a concrete subclass. + class Analyzer + include Downloading + + attr_reader :blob + + # Implement this method in a concrete subclass. Have it return true when given a blob from which + # the analyzer can extract metadata. + def self.accept?(blob) + false + end + + def initialize(blob) + @blob = blob + end + + # Override this method in a concrete subclass. Have it return a Hash of metadata. + def metadata + raise NotImplementedError + end + + private + def logger + ActiveStorage.logger + end + end +end diff --git a/activestorage/lib/active_storage/analyzer/image_analyzer.rb b/activestorage/lib/active_storage/analyzer/image_analyzer.rb new file mode 100644 index 0000000000..b25d2092b9 --- /dev/null +++ b/activestorage/lib/active_storage/analyzer/image_analyzer.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module ActiveStorage + # Extracts width and height in pixels from an image blob. + # + # Example: + # + # ActiveStorage::Analyzer::ImageAnalyzer.new(blob).metadata + # # => { width: 4104, height: 2736 } + # + # This analyzer relies on the third-party [MiniMagick]{https://github.com/minimagick/minimagick} gem. MiniMagick requires + # the [ImageMagick]{http://www.imagemagick.org} system library. These libraries are not provided by Rails; you must + # install them yourself to use this analyzer. + class Analyzer::ImageAnalyzer < Analyzer + def self.accept?(blob) + blob.image? + end + + def metadata + read_image do |image| + { width: image.width, height: image.height } + end + rescue LoadError + logger.info "Skipping image analysis because the mini_magick gem isn't installed" + {} + end + + private + def read_image + download_blob_to_tempfile do |file| + require "mini_magick" + yield MiniMagick::Image.new(file.path) + end + end + end +end diff --git a/activestorage/lib/active_storage/analyzer/null_analyzer.rb b/activestorage/lib/active_storage/analyzer/null_analyzer.rb new file mode 100644 index 0000000000..8ff7ce48e5 --- /dev/null +++ b/activestorage/lib/active_storage/analyzer/null_analyzer.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module ActiveStorage + class Analyzer::NullAnalyzer < Analyzer # :nodoc: + def self.accept?(blob) + true + end + + def metadata + {} + end + end +end diff --git a/activestorage/lib/active_storage/analyzer/video_analyzer.rb b/activestorage/lib/active_storage/analyzer/video_analyzer.rb new file mode 100644 index 0000000000..408b5e58e9 --- /dev/null +++ b/activestorage/lib/active_storage/analyzer/video_analyzer.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require "active_support/core_ext/hash/compact" + +module ActiveStorage + # Extracts the following from a video blob: + # + # * Width (pixels) + # * Height (pixels) + # * Duration (seconds) + # * Angle (degrees) + # * Aspect ratio + # + # Example: + # + # ActiveStorage::VideoAnalyzer.new(blob).metadata + # # => { width: 640, height: 480, duration: 5.0, angle: 0, aspect_ratio: [4, 3] } + # + # This analyzer requires the {ffmpeg}[https://www.ffmpeg.org] system library, which is not provided by Rails. You must + # install ffmpeg yourself to use this analyzer. + class Analyzer::VideoAnalyzer < Analyzer + def self.accept?(blob) + blob.video? + end + + def metadata + { width: width, height: height, duration: duration, angle: angle, aspect_ratio: aspect_ratio }.compact + end + + private + def width + Integer(video_stream["width"]) if video_stream["width"] + end + + def height + Integer(video_stream["height"]) if video_stream["height"] + end + + def duration + Float(video_stream["duration"]) if video_stream["duration"] + end + + def angle + Integer(tags["rotate"]) if tags["rotate"] + end + + def aspect_ratio + if descriptor = video_stream["display_aspect_ratio"] + descriptor.split(":", 2).collect(&:to_i) + end + end + + + def tags + @tags ||= video_stream["tags"] || {} + end + + def video_stream + @video_stream ||= streams.detect { |stream| stream["codec_type"] == "video" } || {} + end + + def streams + probe["streams"] || [] + end + + def probe + download_blob_to_tempfile { |file| probe_from(file) } + end + + def probe_from(file) + IO.popen([ "ffprobe", "-print_format", "json", "-show_streams", "-v", "error", file.path ]) do |output| + JSON.parse(output.read) + end + rescue Errno::ENOENT + logger.info "Skipping video analysis because ffmpeg isn't installed" + {} + end + end +end diff --git a/activestorage/lib/active_storage/attached/one.rb b/activestorage/lib/active_storage/attached/one.rb index c66be08f58..dc19512484 100644 --- a/activestorage/lib/active_storage/attached/one.rb +++ b/activestorage/lib/active_storage/attached/one.rb @@ -59,12 +59,16 @@ module ActiveStorage def replace(attachable) blob.tap do transaction do - destroy + destroy_attachment write_attachment create_attachment_from(attachable) end end.purge_later end + def destroy_attachment + attachment.destroy + end + def create_attachment_from(attachable) ActiveStorage::Attachment.create!(record: record, name: name, blob: create_blob_from(attachable)) end diff --git a/activestorage/lib/active_storage/downloading.rb b/activestorage/lib/active_storage/downloading.rb new file mode 100644 index 0000000000..bcf42f610e --- /dev/null +++ b/activestorage/lib/active_storage/downloading.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module ActiveStorage + module Downloading + private + # Opens a new tempfile and copies blob data into it. Yields the tempfile. + def download_blob_to_tempfile # :doc: + Tempfile.open("ActiveStorage") do |file| + download_blob_to file + yield file + end + end + + # Efficiently download blob data into the given file. + def download_blob_to(file) # :doc: + file.binmode + blob.download { |chunk| file.write(chunk) } + file.rewind + end + end +end diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb index 335eae8dd8..a01a14cd83 100644 --- a/activestorage/lib/active_storage/engine.rb +++ b/activestorage/lib/active_storage/engine.rb @@ -6,20 +6,22 @@ require "active_storage" require "active_storage/previewer/pdf_previewer" require "active_storage/previewer/video_previewer" +require "active_storage/analyzer/image_analyzer" +require "active_storage/analyzer/video_analyzer" + module ActiveStorage class Engine < Rails::Engine # :nodoc: isolate_namespace ActiveStorage config.active_storage = ActiveSupport::OrderedOptions.new config.active_storage.previewers = [ ActiveStorage::Previewer::PDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ] + config.active_storage.analyzers = [ ActiveStorage::Analyzer::ImageAnalyzer, ActiveStorage::Analyzer::VideoAnalyzer ] config.eager_load_namespaces << ActiveStorage initializer "active_storage.logger" do - require "active_storage/service" - config.after_initialize do |app| - ActiveStorage::Service.logger = app.config.active_storage.logger || Rails.logger + ActiveStorage.logger = app.config.active_storage.logger || Rails.logger end end @@ -69,5 +71,11 @@ module ActiveStorage ActiveStorage.previewers = app.config.active_storage.previewers || [] end end + + initializer "active_storage.analyzers" do + config.after_initialize do |app| + ActiveStorage.analyzers = app.config.active_storage.analyzers || [] + end + end end end diff --git a/activestorage/lib/active_storage/log_subscriber.rb b/activestorage/lib/active_storage/log_subscriber.rb index 0d00a75c0e..5cbf4bd1a5 100644 --- a/activestorage/lib/active_storage/log_subscriber.rb +++ b/activestorage/lib/active_storage/log_subscriber.rb @@ -27,7 +27,7 @@ module ActiveStorage end def logger - ActiveStorage::Service.logger + ActiveStorage.logger end private diff --git a/activestorage/lib/active_storage/previewer.rb b/activestorage/lib/active_storage/previewer.rb index 930b376067..460f6d5678 100644 --- a/activestorage/lib/active_storage/previewer.rb +++ b/activestorage/lib/active_storage/previewer.rb @@ -1,10 +1,14 @@ # frozen_string_literal: true +require "active_storage/downloading" + module ActiveStorage # This is an abstract base class for previewers, which generate images from blobs. See # ActiveStorage::Previewer::PDFPreviewer and ActiveStorage::Previewer::VideoPreviewer for examples of # concrete subclasses. class Previewer + include Downloading + attr_reader :blob # Implement this method in a concrete subclass. Have it return true when given a blob from which @@ -24,37 +28,20 @@ module ActiveStorage end private - # Downloads the blob to a new tempfile. Yields the tempfile. - # - # Use this method to get a tempfile that you can provide to a drawing command. - def open # :doc: - Tempfile.open("input") do |file| - download_blob_to file - yield file - end - end - - def download_blob_to(file) - file.binmode - blob.download { |chunk| file.write(chunk) } - file.rewind - end - - # Executes a system command, capturing its binary output in a tempfile. Yields the tempfile. # - # Use this method to shell out to system libraries (e.g. mupdf or ffmpeg) for preview image + # Use this method to shell out to a system library (e.g. mupdf or ffmpeg) for preview image # generation. The resulting tempfile can be used as the +:io+ value in an attachable Hash: # # def preview - # open do |input| + # download_blob_to_tempfile do |input| # draw "my-drawing-command", input.path, "--format", "png", "-" do |output| # yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" # end # end # end def draw(*argv) # :doc: - Tempfile.open("output") do |file| + Tempfile.open("ActiveStorage") do |file| capture(*argv, to: file) yield file end diff --git a/activestorage/lib/active_storage/previewer/pdf_previewer.rb b/activestorage/lib/active_storage/previewer/pdf_previewer.rb index 31a2a8f120..a2f05c74a6 100644 --- a/activestorage/lib/active_storage/previewer/pdf_previewer.rb +++ b/activestorage/lib/active_storage/previewer/pdf_previewer.rb @@ -7,7 +7,7 @@ module ActiveStorage end def preview - open do |input| + download_blob_to_tempfile do |input| draw "mutool", "draw", "-F", "png", "-o", "-", input.path, "1" do |output| yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" end diff --git a/activestorage/lib/active_storage/previewer/video_previewer.rb b/activestorage/lib/active_storage/previewer/video_previewer.rb index 840d87f100..49f128d142 100644 --- a/activestorage/lib/active_storage/previewer/video_previewer.rb +++ b/activestorage/lib/active_storage/previewer/video_previewer.rb @@ -7,11 +7,17 @@ module ActiveStorage end def preview - open do |input| - draw "ffmpeg", "-i", input.path, "-y", "-vcodec", "png", "-vf", "thumbnail", "-vframes", "1", "-f", "image2", "-" do |output| + download_blob_to_tempfile do |input| + draw_relevant_frame_from input do |output| yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" end end end + + private + def draw_relevant_frame_from(file, &block) + draw "ffmpeg", "-i", file.path, "-y", "-vcodec", "png", + "-vf", "thumbnail", "-vframes", "1", "-f", "image2", "-", &block + end end end diff --git a/activestorage/lib/active_storage/service.rb b/activestorage/lib/active_storage/service.rb index 1f012da1e7..aa150e4d8a 100644 --- a/activestorage/lib/active_storage/service.rb +++ b/activestorage/lib/active_storage/service.rb @@ -41,8 +41,6 @@ module ActiveStorage extend ActiveSupport::Autoload autoload :Configurator - class_attribute :logger - class_attribute :url_expires_in, default: 5.minutes class << self -- cgit v1.2.3 From 398e4fecde6f8b3263cc7c04face664c248d3966 Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Sun, 22 Oct 2017 13:36:37 -0400 Subject: Fix links [ci skip] --- activestorage/lib/active_storage/analyzer/image_analyzer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/analyzer/image_analyzer.rb b/activestorage/lib/active_storage/analyzer/image_analyzer.rb index b25d2092b9..25e0251e6e 100644 --- a/activestorage/lib/active_storage/analyzer/image_analyzer.rb +++ b/activestorage/lib/active_storage/analyzer/image_analyzer.rb @@ -8,8 +8,8 @@ module ActiveStorage # ActiveStorage::Analyzer::ImageAnalyzer.new(blob).metadata # # => { width: 4104, height: 2736 } # - # This analyzer relies on the third-party [MiniMagick]{https://github.com/minimagick/minimagick} gem. MiniMagick requires - # the [ImageMagick]{http://www.imagemagick.org} system library. These libraries are not provided by Rails; you must + # This analyzer relies on the third-party {MiniMagick}[https://github.com/minimagick/minimagick] gem. MiniMagick requires + # the {ImageMagick}[http://www.imagemagick.org] system library. These libraries are not provided by Rails; you must # install them yourself to use this analyzer. class Analyzer::ImageAnalyzer < Analyzer def self.accept?(blob) -- cgit v1.2.3 From f4d1aa5310284ebceb51970140fa08f6c8ea19b6 Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Sun, 22 Oct 2017 23:14:44 -0400 Subject: Use the indicative mood consistently [ci skip] --- activestorage/lib/active_storage/downloading.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/downloading.rb b/activestorage/lib/active_storage/downloading.rb index bcf42f610e..ceb7cce0c7 100644 --- a/activestorage/lib/active_storage/downloading.rb +++ b/activestorage/lib/active_storage/downloading.rb @@ -11,7 +11,7 @@ module ActiveStorage end end - # Efficiently download blob data into the given file. + # Efficiently downloads blob data into the given file. def download_blob_to(file) # :doc: file.binmode blob.download { |chunk| file.write(chunk) } -- cgit v1.2.3 From aa6bcbbac8517d5b077f21073e9902637d7c7157 Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Thu, 2 Nov 2017 15:07:04 -0400 Subject: Allow third-party previewers/analyzers to customize their tempdirs --- activestorage/lib/active_storage/downloading.rb | 9 +++++++-- activestorage/lib/active_storage/previewer.rb | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/downloading.rb b/activestorage/lib/active_storage/downloading.rb index ceb7cce0c7..3dac6b116a 100644 --- a/activestorage/lib/active_storage/downloading.rb +++ b/activestorage/lib/active_storage/downloading.rb @@ -3,9 +3,9 @@ module ActiveStorage module Downloading private - # Opens a new tempfile and copies blob data into it. Yields the tempfile. + # Opens a new tempfile in #tempdir and copies blob data into it. Yields the tempfile. def download_blob_to_tempfile # :doc: - Tempfile.open("ActiveStorage") do |file| + Tempfile.open("ActiveStorage", tempdir) do |file| download_blob_to file yield file end @@ -17,5 +17,10 @@ module ActiveStorage blob.download { |chunk| file.write(chunk) } file.rewind end + + # Returns the directory in which tempfiles should be opened. Defaults to +Dir.tmpdir+. + def tempdir # :doc: + Dir.tmpdir + end end end diff --git a/activestorage/lib/active_storage/previewer.rb b/activestorage/lib/active_storage/previewer.rb index 460f6d5678..ed75bae3b5 100644 --- a/activestorage/lib/active_storage/previewer.rb +++ b/activestorage/lib/active_storage/previewer.rb @@ -40,8 +40,10 @@ module ActiveStorage # end # end # end + # + # The output tempfile is opened in the directory returned by ActiveStorage::Downloading#tempdir. def draw(*argv) # :doc: - Tempfile.open("ActiveStorage") do |file| + Tempfile.open("ActiveStorage", tempdir) do |file| capture(*argv, to: file) yield file end -- cgit v1.2.3 From 9ec67362054e874ed905310a79b670941fa397af Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Fri, 3 Nov 2017 11:29:21 -0400 Subject: Permit configuring Active Storage's job queue --- activestorage/lib/active_storage/engine.rb | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb index a01a14cd83..6cf6635c4f 100644 --- a/activestorage/lib/active_storage/engine.rb +++ b/activestorage/lib/active_storage/engine.rb @@ -19,9 +19,12 @@ module ActiveStorage config.eager_load_namespaces << ActiveStorage - initializer "active_storage.logger" do + initializer "active_storage.configs" do config.after_initialize do |app| - ActiveStorage.logger = app.config.active_storage.logger || Rails.logger + ActiveStorage.logger = app.config.active_storage.logger || Rails.logger + ActiveStorage.queue = app.config.active_storage.queue + ActiveStorage.previewers = app.config.active_storage.previewers || [] + ActiveStorage.analyzers = app.config.active_storage.analyzers || [] end end @@ -65,17 +68,5 @@ module ActiveStorage end end end - - initializer "active_storage.previewers" do - config.after_initialize do |app| - ActiveStorage.previewers = app.config.active_storage.previewers || [] - end - end - - initializer "active_storage.analyzers" do - config.after_initialize do |app| - ActiveStorage.analyzers = app.config.active_storage.analyzers || [] - end - end end end -- cgit v1.2.3 From 86938c495e282e6a61c16a9e1d77582e22c0a4fc Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Mon, 6 Nov 2017 21:29:37 -0500 Subject: Fix streaming downloads from S3/Azure Storage Closes #31073. --- activestorage/lib/active_storage/service/azure_storage_service.rb | 8 ++++---- activestorage/lib/active_storage/service/s3_service.rb | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index 27dd192ce6..f3877ad9c9 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -28,7 +28,7 @@ module ActiveStorage end end - def download(key) + def download(key, &block) if block_given? instrument :streaming_download, key do stream(key, &block) @@ -108,15 +108,15 @@ module ActiveStorage end # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, options = {}, &block) + def stream(key) blob = blob_for(key) chunk_size = 5.megabytes offset = 0 while offset < blob.properties[:content_length] - _, io = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) - yield io + _, chunk = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) + yield chunk.force_encoding(Encoding::BINARY) offset += chunk_size end end diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index 3e93cdd072..958b172efb 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -26,7 +26,7 @@ module ActiveStorage end end - def download(key) + def download(key, &block) if block_given? instrument :streaming_download, key do stream(key, &block) @@ -85,14 +85,14 @@ module ActiveStorage end # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, options = {}, &block) + def stream(key, &block) object = object_for(key) chunk_size = 5.megabytes offset = 0 while offset < object.content_length - yield object.read(options.merge(range: "bytes=#{offset}-#{offset + chunk_size - 1}")) + yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.read.force_encoding(Encoding::BINARY) offset += chunk_size end end -- cgit v1.2.3 From daf77db65d9d5e295ee3ba86605988875cb834e4 Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Tue, 7 Nov 2017 09:06:23 -0500 Subject: Remove needless block parameter --- activestorage/lib/active_storage/service/s3_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index 958b172efb..6957119780 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -85,7 +85,7 @@ module ActiveStorage end # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, &block) + def stream(key) object = object_for(key) chunk_size = 5.megabytes -- cgit v1.2.3 From 499a4164ce9816c4913bf3db14787ea99ef2c266 Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Tue, 14 Nov 2017 10:42:10 -0500 Subject: Introduce ActiveStorage::Attached::{One,Many}#detach --- activestorage/lib/active_storage/attached/many.rb | 6 +++++- activestorage/lib/active_storage/attached/one.rb | 15 +++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/attached/many.rb b/activestorage/lib/active_storage/attached/many.rb index 1e0657c33c..0b3400bccf 100644 --- a/activestorage/lib/active_storage/attached/many.rb +++ b/activestorage/lib/active_storage/attached/many.rb @@ -13,7 +13,6 @@ module ActiveStorage end # Associates one or several attachments with the current record, saving them to the database. - # Examples: # # document.images.attach(params[:images]) # Array of ActionDispatch::Http::UploadedFile objects # document.images.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload @@ -36,6 +35,11 @@ module ActiveStorage attachments.any? end + # Deletes associated attachments without purging them, leaving their respective blobs in place. + def detach + attachments.destroy_all if attached? + end + # Directly purges each associated attachment (i.e. destroys the blobs and # attachments and deletes the files on the service). def purge diff --git a/activestorage/lib/active_storage/attached/one.rb b/activestorage/lib/active_storage/attached/one.rb index dc19512484..7092f6b109 100644 --- a/activestorage/lib/active_storage/attached/one.rb +++ b/activestorage/lib/active_storage/attached/one.rb @@ -14,7 +14,6 @@ module ActiveStorage end # Associates a given attachment with the current record, saving it to the database. - # Examples: # # person.avatar.attach(params[:avatar]) # ActionDispatch::Http::UploadedFile object # person.avatar.attach(params[:signed_blob_id]) # Signed reference to blob from direct upload @@ -39,6 +38,14 @@ module ActiveStorage attachment.present? end + # Deletes the attachment without purging it, leaving its blob in place. + def detach + if attached? + attachment.destroy + write_attachment nil + end + end + # Directly purges the attachment (i.e. destroys the blob and # attachment and deletes the file on the service). def purge @@ -59,16 +66,12 @@ module ActiveStorage def replace(attachable) blob.tap do transaction do - destroy_attachment + detach write_attachment create_attachment_from(attachable) end end.purge_later end - def destroy_attachment - attachment.destroy - end - def create_attachment_from(attachable) ActiveStorage::Attachment.create!(record: record, name: name, blob: create_blob_from(attachable)) end -- cgit v1.2.3 From e05e2ae44f1ecf8e9bb5949f531305c15bc3c665 Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Sun, 19 Nov 2017 17:34:07 -0500 Subject: Permit attaching files to new records Closes #31164. --- activestorage/lib/active_storage/attached/many.rb | 6 +++++- activestorage/lib/active_storage/attached/one.rb | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/attached/many.rb b/activestorage/lib/active_storage/attached/many.rb index 0b3400bccf..6eace65b79 100644 --- a/activestorage/lib/active_storage/attached/many.rb +++ b/activestorage/lib/active_storage/attached/many.rb @@ -20,7 +20,11 @@ module ActiveStorage # document.images.attach([ first_blob, second_blob ]) def attach(*attachables) attachables.flatten.collect do |attachable| - attachments.create!(name: name, blob: create_blob_from(attachable)) + if record.new_record? + attachments.build(record: record, blob: create_blob_from(attachable)) + else + attachments.create!(record: record, blob: create_blob_from(attachable)) + end end end diff --git a/activestorage/lib/active_storage/attached/one.rb b/activestorage/lib/active_storage/attached/one.rb index 7092f6b109..0244232b2c 100644 --- a/activestorage/lib/active_storage/attached/one.rb +++ b/activestorage/lib/active_storage/attached/one.rb @@ -23,7 +23,7 @@ module ActiveStorage if attached? && dependent == :purge_later replace attachable else - write_attachment create_attachment_from(attachable) + write_attachment build_attachment_from(attachable) end end @@ -67,13 +67,13 @@ module ActiveStorage blob.tap do transaction do detach - write_attachment create_attachment_from(attachable) + write_attachment build_attachment_from(attachable) end end.purge_later end - def create_attachment_from(attachable) - ActiveStorage::Attachment.create!(record: record, name: name, blob: create_blob_from(attachable)) + def build_attachment_from(attachable) + ActiveStorage::Attachment.new(record: record, name: name, blob: create_blob_from(attachable)) end def write_attachment(attachment) -- cgit v1.2.3 From 1d24e47140356f136471d15e3ce3fa427f4430c2 Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Mon, 20 Nov 2017 18:06:06 -0500 Subject: Provide attachment writers Permit creating a record and attaching files in a single step. # Before: User.create!(user_params.except(:avatar)).tap do |user| user.avatar.attach(user_params[:avatar]) end # After: User.create!(user_params) [Yoshiyuki Hirano & George Claghorn] --- activestorage/lib/active_storage/attached/macros.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/attached/macros.rb b/activestorage/lib/active_storage/attached/macros.rb index f0256718ac..2b38a9b887 100644 --- a/activestorage/lib/active_storage/attached/macros.rb +++ b/activestorage/lib/active_storage/attached/macros.rb @@ -32,6 +32,10 @@ module ActiveStorage def #{name} @active_storage_attached_#{name} ||= ActiveStorage::Attached::One.new("#{name}", self, dependent: #{dependent == :purge_later ? ":purge_later" : "false"}) end + + def #{name}=(attachable) + #{name}.attach(attachable) + end CODE has_one :"#{name}_attachment", -> { where(name: name) }, class_name: "ActiveStorage::Attachment", as: :record @@ -73,6 +77,10 @@ module ActiveStorage def #{name} @active_storage_attached_#{name} ||= ActiveStorage::Attached::Many.new("#{name}", self, dependent: #{dependent == :purge_later ? ":purge_later" : "false"}) end + + def #{name}=(attachables) + #{name}.attach(attachables) + end CODE has_many :"#{name}_attachments", -> { where(name: name) }, as: :record, class_name: "ActiveStorage::Attachment" -- cgit v1.2.3 From fbb12910bdf256bbe2a95909fff114243d75424d Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Thu, 23 Nov 2017 19:48:25 -0500 Subject: Avoid connecting to GCS during app boot --- activestorage/lib/active_storage/service/gcs_service.rb | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb index b4ffeeeb8a..be6ddf32a0 100644 --- a/activestorage/lib/active_storage/service/gcs_service.rb +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -7,11 +7,8 @@ module ActiveStorage # Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API # documentation that applies to all services. class Service::GCSService < Service - attr_reader :client, :bucket - - def initialize(project:, keyfile:, bucket:, **options) - @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile, **options) - @bucket = @client.bucket(bucket) + def initialize(**config) + @config = config end def upload(key, io, checksum: nil) @@ -85,8 +82,18 @@ module ActiveStorage end private + attr_reader :config + def file_for(key) bucket.file(key, skip_lookup: true) end + + def bucket + @bucket ||= client.bucket(config.fetch(:bucket)) + end + + def client + @client ||= Google::Cloud::Storage.new(config.except(:bucket)) + end end end -- cgit v1.2.3 From cceeeb6e57f1cf8b24d507e2da9ed85d374c8bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Mon, 27 Nov 2017 13:01:15 -0500 Subject: Preparing for 5.2.0.beta1 release --- activestorage/lib/active_storage/gem_version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/gem_version.rb b/activestorage/lib/active_storage/gem_version.rb index e1d7b3493a..b5e783793b 100644 --- a/activestorage/lib/active_storage/gem_version.rb +++ b/activestorage/lib/active_storage/gem_version.rb @@ -10,7 +10,7 @@ module ActiveStorage MAJOR = 5 MINOR = 2 TINY = 0 - PRE = "alpha" + PRE = "beta1" STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".") end -- cgit v1.2.3 From 2837d0f3347e747a8c12bd3c097bc7282072d42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Tue, 28 Nov 2017 00:01:45 -0500 Subject: Preparing for 5.2.0.beta2 release --- activestorage/lib/active_storage/gem_version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/gem_version.rb b/activestorage/lib/active_storage/gem_version.rb index b5e783793b..f048bb0b77 100644 --- a/activestorage/lib/active_storage/gem_version.rb +++ b/activestorage/lib/active_storage/gem_version.rb @@ -10,7 +10,7 @@ module ActiveStorage MAJOR = 5 MINOR = 2 TINY = 0 - PRE = "beta1" + PRE = "beta2" STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".") end -- cgit v1.2.3 From 9d65ac30fde3977d4773ff6052f31b99a5084f0f Mon Sep 17 00:00:00 2001 From: "yuuji.yaginuma" Date: Wed, 29 Nov 2017 13:08:33 +0900 Subject: Use `credentials` instead of `keyfile` in GCS sevice The `keyfile` was renamed to `credentials` in `google-cloud-storage` 1.8.0. https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/master/google-cloud-storage/CHANGELOG.md#180--2017-11-14 Although `keyfile` can still be used, but it looks like deprecate. https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/ddf7b2a856d676316525eb581c1a4cc83ca6097b/google-cloud-storage/lib/google/cloud/storage.rb#L589...L590 Therefore, I think that should use `credentials` in newly generated applications. Ref: https://github.com/GoogleCloudPlatform/google-cloud-ruby/issues/1802 --- activestorage/lib/active_storage/service/gcs_service.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb index be6ddf32a0..fd9916634a 100644 --- a/activestorage/lib/active_storage/service/gcs_service.rb +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +gem "google-cloud-storage", "~> 1.8" + require "google/cloud/storage" require "active_support/core_ext/object/to_query" -- cgit v1.2.3 From d041a1dcbaced9f44ed976a48312c9f492fffe5e Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Thu, 30 Nov 2017 23:54:03 -0500 Subject: Add ActiveStorage::Previewer#logger to match ActiveStorage::Analyzer#logger --- activestorage/lib/active_storage/previewer.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/previewer.rb b/activestorage/lib/active_storage/previewer.rb index ed75bae3b5..3d485988e9 100644 --- a/activestorage/lib/active_storage/previewer.rb +++ b/activestorage/lib/active_storage/previewer.rb @@ -54,5 +54,9 @@ module ActiveStorage IO.popen(argv) { |out| IO.copy_stream(out, to) } to.rewind end + + def logger + ActiveStorage.logger + end end end -- cgit v1.2.3 From b852ef2660dac36e348865b455fab7fbcc0d2a7f Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Fri, 1 Dec 2017 11:07:30 -0500 Subject: Make ASt previewer/analyzer binary paths configurable --- .../lib/active_storage/analyzer/video_analyzer.rb | 4 +++- activestorage/lib/active_storage/engine.rb | 19 ++++++++++++++++++- .../lib/active_storage/previewer/pdf_previewer.rb | 9 ++++++++- .../lib/active_storage/previewer/video_previewer.rb | 4 +++- 4 files changed, 32 insertions(+), 4 deletions(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/analyzer/video_analyzer.rb b/activestorage/lib/active_storage/analyzer/video_analyzer.rb index 408b5e58e9..b6fc54d917 100644 --- a/activestorage/lib/active_storage/analyzer/video_analyzer.rb +++ b/activestorage/lib/active_storage/analyzer/video_analyzer.rb @@ -19,6 +19,8 @@ module ActiveStorage # This analyzer requires the {ffmpeg}[https://www.ffmpeg.org] system library, which is not provided by Rails. You must # install ffmpeg yourself to use this analyzer. class Analyzer::VideoAnalyzer < Analyzer + class_attribute :ffprobe_path, default: "ffprobe" + def self.accept?(blob) blob.video? end @@ -68,7 +70,7 @@ module ActiveStorage end def probe_from(file) - IO.popen([ "ffprobe", "-print_format", "json", "-show_streams", "-v", "error", file.path ]) do |output| + IO.popen([ ffprobe_path, "-print_format", "json", "-show_streams", "-v", "error", file.path ]) do |output| JSON.parse(output.read) end rescue Errno::ENOENT diff --git a/activestorage/lib/active_storage/engine.rb b/activestorage/lib/active_storage/engine.rb index 6cf6635c4f..b870e6d4d6 100644 --- a/activestorage/lib/active_storage/engine.rb +++ b/activestorage/lib/active_storage/engine.rb @@ -15,7 +15,8 @@ module ActiveStorage config.active_storage = ActiveSupport::OrderedOptions.new config.active_storage.previewers = [ ActiveStorage::Previewer::PDFPreviewer, ActiveStorage::Previewer::VideoPreviewer ] - config.active_storage.analyzers = [ ActiveStorage::Analyzer::ImageAnalyzer, ActiveStorage::Analyzer::VideoAnalyzer ] + config.active_storage.analyzers = [ ActiveStorage::Analyzer::ImageAnalyzer, ActiveStorage::Analyzer::VideoAnalyzer ] + config.active_storage.paths = ActiveSupport::OrderedOptions.new config.eager_load_namespaces << ActiveStorage @@ -68,5 +69,21 @@ module ActiveStorage end end end + + initializer "active_storage.paths" do + config.after_initialize do |app| + if ffprobe_path = app.config.active_storage.paths.ffprobe + ActiveStorage::Analyzer::VideoAnalyzer.ffprobe_path = ffprobe_path + end + + if ffmpeg_path = app.config.active_storage.paths.ffmpeg + ActiveStorage::Previewer::VideoPreviewer.ffmpeg_path = ffmpeg_path + end + + if mutool_path = app.config.active_storage.paths.mutool + ActiveStorage::Previewer::PDFPreviewer.mutool_path = mutool_path + end + end + end end end diff --git a/activestorage/lib/active_storage/previewer/pdf_previewer.rb b/activestorage/lib/active_storage/previewer/pdf_previewer.rb index a2f05c74a6..b84aefcc9c 100644 --- a/activestorage/lib/active_storage/previewer/pdf_previewer.rb +++ b/activestorage/lib/active_storage/previewer/pdf_previewer.rb @@ -2,16 +2,23 @@ module ActiveStorage class Previewer::PDFPreviewer < Previewer + class_attribute :mutool_path, default: "mutool" + def self.accept?(blob) blob.content_type == "application/pdf" end def preview download_blob_to_tempfile do |input| - draw "mutool", "draw", "-F", "png", "-o", "-", input.path, "1" do |output| + draw_first_page_from input do |output| yield io: output, filename: "#{blob.filename.base}.png", content_type: "image/png" end end end + + private + def draw_first_page_from(file, &block) + draw mutool_path, "draw", "-F", "png", "-o", "-", file.path, "1", &block + end end end diff --git a/activestorage/lib/active_storage/previewer/video_previewer.rb b/activestorage/lib/active_storage/previewer/video_previewer.rb index 49f128d142..5d06e33f44 100644 --- a/activestorage/lib/active_storage/previewer/video_previewer.rb +++ b/activestorage/lib/active_storage/previewer/video_previewer.rb @@ -2,6 +2,8 @@ module ActiveStorage class Previewer::VideoPreviewer < Previewer + class_attribute :ffmpeg_path, default: "ffmpeg" + def self.accept?(blob) blob.video? end @@ -16,7 +18,7 @@ module ActiveStorage private def draw_relevant_frame_from(file, &block) - draw "ffmpeg", "-i", file.path, "-y", "-vcodec", "png", + draw ffmpeg_path, "-i", file.path, "-y", "-vcodec", "png", "-vf", "thumbnail", "-vframes", "1", "-f", "image2", "-", &block end end -- cgit v1.2.3 From 8c5a7fbefd3cad403e7594d0b6a5488d80d4c98e Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Sat, 2 Dec 2017 22:43:28 -0500 Subject: Purge variants with their blobs --- activestorage/lib/active_storage/log_subscriber.rb | 4 +++ activestorage/lib/active_storage/service.rb | 9 +++++-- .../service/azure_storage_service.rb | 30 +++++++++++++++++----- .../lib/active_storage/service/disk_service.rb | 22 +++++++++++----- .../lib/active_storage/service/gcs_service.rb | 18 ++++++++----- .../lib/active_storage/service/mirror_service.rb | 5 ++++ .../lib/active_storage/service/s3_service.rb | 20 ++++++++++----- 7 files changed, 79 insertions(+), 29 deletions(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/log_subscriber.rb b/activestorage/lib/active_storage/log_subscriber.rb index 5cbf4bd1a5..a4e148c1a5 100644 --- a/activestorage/lib/active_storage/log_subscriber.rb +++ b/activestorage/lib/active_storage/log_subscriber.rb @@ -18,6 +18,10 @@ module ActiveStorage info event, color("Deleted file from key: #{key_in(event)}", RED) end + def service_delete_prefixed(event) + info event, color("Deleted files by key prefix: #{event.payload[:prefix]}", RED) + end + def service_exist(event) debug event, color("Checked if file exists at key: #{key_in(event)} (#{event.payload[:exist] ? "yes" : "no"})", BLUE) end diff --git a/activestorage/lib/active_storage/service.rb b/activestorage/lib/active_storage/service.rb index aa150e4d8a..c8f675db86 100644 --- a/activestorage/lib/active_storage/service.rb +++ b/activestorage/lib/active_storage/service.rb @@ -78,6 +78,11 @@ module ActiveStorage raise NotImplementedError end + # Delete files at keys starting with the +prefix+. + def delete_prefixed(prefix) + raise NotImplementedError + end + # Return +true+ if a file exists at the +key+. def exist?(key) raise NotImplementedError @@ -104,10 +109,10 @@ module ActiveStorage end private - def instrument(operation, key, payload = {}, &block) + def instrument(operation, payload = {}, &block) ActiveSupport::Notifications.instrument( "service_#{operation}.active_storage", - payload.merge(key: key, service: service_name), &block) + payload.merge(service: service_name), &block) end def service_name diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index f3877ad9c9..98de0836de 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -19,7 +19,7 @@ module ActiveStorage end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do begin blobs.create_block_blob(container, key, io, content_md5: checksum) rescue Azure::Core::Http::HTTPError @@ -30,11 +30,11 @@ module ActiveStorage def download(key, &block) if block_given? - instrument :streaming_download, key do + instrument :streaming_download, key: key do stream(key, &block) end else - instrument :download, key do + instrument :download, key: key do _, io = blobs.get_blob(container, key) io.force_encoding(Encoding::BINARY) end @@ -42,7 +42,7 @@ module ActiveStorage end def delete(key) - instrument :delete, key do + instrument :delete, key: key do begin blobs.delete_blob(container, key) rescue Azure::Core::Http::HTTPError @@ -51,8 +51,24 @@ module ActiveStorage end end + def delete_prefixed(prefix) + instrument :delete_all, prefix: prefix do + marker = nil + + loop do + results = blobs.list_blobs(container, prefix: prefix, marker: marker) + + results.each do |blob| + blobs.delete_blob(container, blob.name) + end + + break unless marker = results.continuation_token.presence + end + end + end + def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = blob_for(key).present? payload[:exist] = answer answer @@ -60,7 +76,7 @@ module ActiveStorage end def url(key, expires_in:, filename:, disposition:, content_type:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| base_url = url_for(key) generated_url = signer.signed_uri( URI(base_url), false, @@ -77,7 +93,7 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| base_url = url_for(key) generated_url = signer.signed_uri(URI(base_url), false, permissions: "rw", expiry: format_expiry(expires_in)).to_s diff --git a/activestorage/lib/active_storage/service/disk_service.rb b/activestorage/lib/active_storage/service/disk_service.rb index 52eaba4e7b..a8728c5bc3 100644 --- a/activestorage/lib/active_storage/service/disk_service.rb +++ b/activestorage/lib/active_storage/service/disk_service.rb @@ -16,7 +16,7 @@ module ActiveStorage end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do IO.copy_stream(io, make_path_for(key)) ensure_integrity_of(key, checksum) if checksum end @@ -24,7 +24,7 @@ module ActiveStorage def download(key) if block_given? - instrument :streaming_download, key do + instrument :streaming_download, key: key do File.open(path_for(key), "rb") do |file| while data = file.read(64.kilobytes) yield data @@ -32,14 +32,14 @@ module ActiveStorage end end else - instrument :download, key do + instrument :download, key: key do File.binread path_for(key) end end end def delete(key) - instrument :delete, key do + instrument :delete, key: key do begin File.delete path_for(key) rescue Errno::ENOENT @@ -48,8 +48,16 @@ module ActiveStorage end end + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + Dir.glob(path_for("#{prefix}*")).each do |path| + FileUtils.rm_rf(path) + end + end + end + def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = File.exist? path_for(key) payload[:exist] = answer answer @@ -57,7 +65,7 @@ module ActiveStorage end def url(key, expires_in:, filename:, disposition:, content_type:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| verified_key_with_expiration = ActiveStorage.verifier.generate(key, expires_in: expires_in, purpose: :blob_key) generated_url = @@ -77,7 +85,7 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| verified_token_with_expiration = ActiveStorage.verifier.generate( { key: key, diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb index fd9916634a..c13ce4786d 100644 --- a/activestorage/lib/active_storage/service/gcs_service.rb +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -14,7 +14,7 @@ module ActiveStorage end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do begin bucket.create_file(io, key, md5: checksum) rescue Google::Cloud::InvalidArgumentError @@ -25,7 +25,7 @@ module ActiveStorage # FIXME: Download in chunks when given a block. def download(key) - instrument :download, key do + instrument :download, key: key do io = file_for(key).download io.rewind @@ -38,7 +38,7 @@ module ActiveStorage end def delete(key) - instrument :delete, key do + instrument :delete, key: key do begin file_for(key).delete rescue Google::Cloud::NotFoundError @@ -47,8 +47,14 @@ module ActiveStorage end end + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + bucket.files(prefix: prefix).all(&:delete) + end + end + def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = file_for(key).exists? payload[:exist] = answer answer @@ -56,7 +62,7 @@ module ActiveStorage end def url(key, expires_in:, filename:, content_type:, disposition:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = file_for(key).signed_url expires: expires_in, query: { "response-content-disposition" => content_disposition_with(type: disposition, filename: filename), "response-content-type" => content_type @@ -69,7 +75,7 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = bucket.signed_url key, method: "PUT", expires: expires_in, content_type: content_type, content_md5: checksum diff --git a/activestorage/lib/active_storage/service/mirror_service.rb b/activestorage/lib/active_storage/service/mirror_service.rb index 39e922f7ab..7eca8ce7f4 100644 --- a/activestorage/lib/active_storage/service/mirror_service.rb +++ b/activestorage/lib/active_storage/service/mirror_service.rb @@ -35,6 +35,11 @@ module ActiveStorage perform_across_services :delete, key end + # Delete files at keys starting with the +prefix+ on all services. + def delete_prefixed(prefix) + perform_across_services :delete_prefixed, prefix + end + private def each_service(&block) [ primary, *mirrors ].each(&block) diff --git a/activestorage/lib/active_storage/service/s3_service.rb b/activestorage/lib/active_storage/service/s3_service.rb index 6957119780..c95672f338 100644 --- a/activestorage/lib/active_storage/service/s3_service.rb +++ b/activestorage/lib/active_storage/service/s3_service.rb @@ -17,7 +17,7 @@ module ActiveStorage end def upload(key, io, checksum: nil) - instrument :upload, key, checksum: checksum do + instrument :upload, key: key, checksum: checksum do begin object_for(key).put(upload_options.merge(body: io, content_md5: checksum)) rescue Aws::S3::Errors::BadDigest @@ -28,24 +28,30 @@ module ActiveStorage def download(key, &block) if block_given? - instrument :streaming_download, key do + instrument :streaming_download, key: key do stream(key, &block) end else - instrument :download, key do + instrument :download, key: key do object_for(key).get.body.read.force_encoding(Encoding::BINARY) end end end def delete(key) - instrument :delete, key do + instrument :delete, key: key do object_for(key).delete end end + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + bucket.objects(prefix: prefix).batch_delete! + end + end + def exist?(key) - instrument :exist, key do |payload| + instrument :exist, key: key do |payload| answer = object_for(key).exists? payload[:exist] = answer answer @@ -53,7 +59,7 @@ module ActiveStorage end def url(key, expires_in:, filename:, disposition:, content_type:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = object_for(key).presigned_url :get, expires_in: expires_in.to_i, response_content_disposition: content_disposition_with(type: disposition, filename: filename), response_content_type: content_type @@ -65,7 +71,7 @@ module ActiveStorage end def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key do |payload| + instrument :url, key: key do |payload| generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i, content_type: content_type, content_length: content_length, content_md5: checksum -- cgit v1.2.3 From 7609ca08ce5000689838eeb04ed37084bf364f78 Mon Sep 17 00:00:00 2001 From: Kasper Timm Hansen Date: Sun, 3 Dec 2017 18:06:29 +0100 Subject: Fix instrumention name: delete_prefixed like the others. --- activestorage/lib/active_storage/service/azure_storage_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/service/azure_storage_service.rb b/activestorage/lib/active_storage/service/azure_storage_service.rb index 98de0836de..19b09991b3 100644 --- a/activestorage/lib/active_storage/service/azure_storage_service.rb +++ b/activestorage/lib/active_storage/service/azure_storage_service.rb @@ -52,7 +52,7 @@ module ActiveStorage end def delete_prefixed(prefix) - instrument :delete_all, prefix: prefix do + instrument :delete_prefixed, prefix: prefix do marker = nil loop do -- cgit v1.2.3 From e8286ee272a3e51daebc198519accd1f6895a8d2 Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Thu, 7 Dec 2017 15:14:22 -0500 Subject: Fix customizing Content-Type via GCS service URLs --- activestorage/lib/active_storage/service/gcs_service.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/service/gcs_service.rb b/activestorage/lib/active_storage/service/gcs_service.rb index c13ce4786d..6f6f4105fe 100644 --- a/activestorage/lib/active_storage/service/gcs_service.rb +++ b/activestorage/lib/active_storage/service/gcs_service.rb @@ -16,7 +16,13 @@ module ActiveStorage def upload(key, io, checksum: nil) instrument :upload, key: key, checksum: checksum do begin - bucket.create_file(io, key, md5: checksum) + # The official GCS client library doesn't allow us to create a file with no Content-Type metadata. + # We need the file we create to have no Content-Type so we can control it via the response-content-type + # param in signed URLs. Workaround: let the GCS client create the file with an inferred + # Content-Type (usually "application/octet-stream") then clear it. + bucket.create_file(io, key, md5: checksum).update do |file| + file.content_type = nil + end rescue Google::Cloud::InvalidArgumentError raise ActiveStorage::IntegrityError end -- cgit v1.2.3 From da8e0ba03cbae33857954c0c1a228bd6dae562da Mon Sep 17 00:00:00 2001 From: George Claghorn Date: Fri, 8 Dec 2017 13:15:04 -0500 Subject: Swap raw video width and height if angle is 90 or 270 degrees --- .../lib/active_storage/analyzer/video_analyzer.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'activestorage/lib/active_storage') diff --git a/activestorage/lib/active_storage/analyzer/video_analyzer.rb b/activestorage/lib/active_storage/analyzer/video_analyzer.rb index b6fc54d917..1c144baa37 100644 --- a/activestorage/lib/active_storage/analyzer/video_analyzer.rb +++ b/activestorage/lib/active_storage/analyzer/video_analyzer.rb @@ -31,10 +31,18 @@ module ActiveStorage private def width - Integer(video_stream["width"]) if video_stream["width"] + rotated? ? raw_height : raw_width end def height + rotated? ? raw_width : raw_height + end + + def raw_width + Integer(video_stream["width"]) if video_stream["width"] + end + + def raw_height Integer(video_stream["height"]) if video_stream["height"] end @@ -52,6 +60,10 @@ module ActiveStorage end end + def rotated? + angle == 90 || angle == 270 + end + def tags @tags ||= video_stream["tags"] || {} -- cgit v1.2.3