diff options
Diffstat (limited to 'lib/active_storage')
-rw-r--r-- | lib/active_storage/attached.rb | 34 | ||||
-rw-r--r-- | lib/active_storage/attached/macros.rb | 23 | ||||
-rw-r--r-- | lib/active_storage/attached/many.rb | 31 | ||||
-rw-r--r-- | lib/active_storage/attached/one.rb | 29 | ||||
-rw-r--r-- | lib/active_storage/attachment.rb | 30 | ||||
-rw-r--r-- | lib/active_storage/blob.rb | 68 | ||||
-rw-r--r-- | lib/active_storage/config/sites.yml | 25 | ||||
-rw-r--r-- | lib/active_storage/disk_controller.rb | 28 | ||||
-rw-r--r-- | lib/active_storage/download.rb | 90 | ||||
-rw-r--r-- | lib/active_storage/filename.rb | 31 | ||||
-rw-r--r-- | lib/active_storage/migration.rb | 28 | ||||
-rw-r--r-- | lib/active_storage/purge_job.rb | 10 | ||||
-rw-r--r-- | lib/active_storage/railtie.rb | 27 | ||||
-rw-r--r-- | lib/active_storage/site.rb | 41 | ||||
-rw-r--r-- | lib/active_storage/site/disk_site.rb | 71 | ||||
-rw-r--r-- | lib/active_storage/site/gcs_site.rb | 53 | ||||
-rw-r--r-- | lib/active_storage/site/mirror_site.rb | 51 | ||||
-rw-r--r-- | lib/active_storage/site/s3_site.rb | 63 | ||||
-rw-r--r-- | lib/active_storage/verified_key_with_expiration.rb | 24 |
19 files changed, 757 insertions, 0 deletions
diff --git a/lib/active_storage/attached.rb b/lib/active_storage/attached.rb new file mode 100644 index 0000000000..7475c38999 --- /dev/null +++ b/lib/active_storage/attached.rb @@ -0,0 +1,34 @@ +require "active_storage/blob" +require "active_storage/attachment" + +require "action_dispatch/http/upload" +require "active_support/core_ext/module/delegation" + +class ActiveStorage::Attached + attr_reader :name, :record + + def initialize(name, record) + @name, @record = name, record + end + + private + def create_blob_from(attachable) + case attachable + when ActiveStorage::Blob + attachable + when ActionDispatch::Http::UploadedFile + ActiveStorage::Blob.create_after_upload! \ + io: attachable.open, + filename: attachable.original_filename, + content_type: attachable.content_type + when Hash + ActiveStorage::Blob.create_after_upload!(attachable) + else + nil + end + end +end + +require "active_storage/attached/one" +require "active_storage/attached/many" +require "active_storage/attached/macros" diff --git a/lib/active_storage/attached/macros.rb b/lib/active_storage/attached/macros.rb new file mode 100644 index 0000000000..96493d1215 --- /dev/null +++ b/lib/active_storage/attached/macros.rb @@ -0,0 +1,23 @@ +module ActiveStorage::Attached::Macros + def has_one_attached(name, dependent: :purge_later) + define_method(name) do + instance_variable_get("@active_storage_attached_#{name}") || + instance_variable_set("@active_storage_attached_#{name}", ActiveStorage::Attached::One.new(name, self)) + end + + if dependent == :purge_later + before_destroy { public_send(name).purge_later } + end + end + + def has_many_attached(name, dependent: :purge_later) + define_method(name) do + instance_variable_get("@active_storage_attached_#{name}") || + instance_variable_set("@active_storage_attached_#{name}", ActiveStorage::Attached::Many.new(name, self)) + end + + if dependent == :purge_later + before_destroy { public_send(name).purge_later } + end + end +end diff --git a/lib/active_storage/attached/many.rb b/lib/active_storage/attached/many.rb new file mode 100644 index 0000000000..f1535dfbc6 --- /dev/null +++ b/lib/active_storage/attached/many.rb @@ -0,0 +1,31 @@ +class ActiveStorage::Attached::Many < ActiveStorage::Attached + delegate_missing_to :attachments + + def attachments + @attachments ||= ActiveStorage::Attachment.where(record_gid: record.to_gid.to_s, name: name) + end + + def attach(*attachables) + @attachments = attachments | Array(attachables).flatten.collect do |attachable| + ActiveStorage::Attachment.create!(record_gid: record.to_gid.to_s, name: name, blob: create_blob_from(attachable)) + end + end + + def attached? + attachments.any? + end + + def purge + if attached? + attachments.each(&:purge) + @attachments = nil + end + end + + def purge_later + if attached? + attachments.each(&:purge_later) + @attachments = nil + end + end +end diff --git a/lib/active_storage/attached/one.rb b/lib/active_storage/attached/one.rb new file mode 100644 index 0000000000..d08d265992 --- /dev/null +++ b/lib/active_storage/attached/one.rb @@ -0,0 +1,29 @@ +class ActiveStorage::Attached::One < ActiveStorage::Attached + delegate_missing_to :attachment + + def attachment + @attachment ||= ActiveStorage::Attachment.find_by(record_gid: record.to_gid.to_s, name: name) + end + + def attach(attachable) + @attachment = ActiveStorage::Attachment.create!(record_gid: record.to_gid.to_s, name: name, blob: create_blob_from(attachable)) + end + + def attached? + attachment.present? + end + + def purge + if attached? + attachment.purge + @attachment = nil + end + end + + def purge_later + if attached? + attachment.purge_later + @attachment = nil + end + end +end diff --git a/lib/active_storage/attachment.rb b/lib/active_storage/attachment.rb new file mode 100644 index 0000000000..20c619aa5a --- /dev/null +++ b/lib/active_storage/attachment.rb @@ -0,0 +1,30 @@ +require "active_storage/blob" +require "global_id" +require "active_support/core_ext/module/delegation" + +# Schema: id, record_gid, blob_id, created_at +class ActiveStorage::Attachment < ActiveRecord::Base + self.table_name = "active_storage_attachments" + + belongs_to :blob, class_name: "ActiveStorage::Blob" + + delegate_missing_to :blob + + def record + @record ||= GlobalID::Locator.locate(record_gid) + end + + def record=(record) + @record = record + self.record_gid = record&.to_gid + end + + def purge + blob.purge + destroy + end + + def purge_later + ActiveStorage::PurgeJob.perform_later(self) + end +end diff --git a/lib/active_storage/blob.rb b/lib/active_storage/blob.rb new file mode 100644 index 0000000000..edf57b5c78 --- /dev/null +++ b/lib/active_storage/blob.rb @@ -0,0 +1,68 @@ +require "active_storage/site" +require "active_storage/filename" +require "active_storage/purge_job" + +# Schema: id, key, filename, content_type, metadata, byte_size, checksum, created_at +class ActiveStorage::Blob < ActiveRecord::Base + self.table_name = "active_storage_blobs" + + has_secure_token :key + store :metadata, coder: JSON + + class_attribute :site + + class << self + def build_after_upload(io:, filename:, content_type: nil, metadata: nil) + new.tap do |blob| + blob.filename = filename + blob.content_type = content_type + blob.metadata = metadata + + blob.upload io + end + end + + def create_after_upload!(io:, filename:, content_type: nil, metadata: nil) + build_after_upload(io: io, filename: filename, content_type: content_type, metadata: metadata).tap(&:save!) + end + end + + # We can't wait until the record is first saved to have a key for it + def key + self[:key] ||= self.class.generate_unique_secure_token + end + + def filename + ActiveStorage::Filename.new(self[:filename]) + end + + def url(expires_in: 5.minutes, disposition: :inline) + site.url key, expires_in: expires_in, disposition: disposition, filename: filename + end + + + def upload(io) + site.upload(key, io) + + self.checksum = site.checksum(key) + self.byte_size = site.byte_size(key) + end + + def download + site.download key + end + + + def delete + site.delete key + end + + def purge + delete + destroy + end + + def purge_later + ActiveStorage::PurgeJob.perform_later(self) + end +end diff --git a/lib/active_storage/config/sites.yml b/lib/active_storage/config/sites.yml new file mode 100644 index 0000000000..43bc77fbf9 --- /dev/null +++ b/lib/active_storage/config/sites.yml @@ -0,0 +1,25 @@ +# Configuration should be something like this: +# +# config/environments/development.rb +# config.active_storage.site = :local +# +# config/environments/production.rb +# config.active_storage.site = :amazon +local: + site: Disk + root: <%%= File.join(Dir.tmpdir, "active_storage") %> + +amazon: + site: S3 + access_key_id: <%%= Rails.application.secrets.aws[:access_key_id] %> + secret_access_key: <%%= Rails.application.secrets.aws[:secret_access_key] %> + region: us-east-1 + bucket: <%= Rails.application.class.name.remove(/::Application$/).underscore %> + +google: + site: GCS + +mirror: + site: Mirror + primary: amazon + secondaries: google diff --git a/lib/active_storage/disk_controller.rb b/lib/active_storage/disk_controller.rb new file mode 100644 index 0000000000..3eba86c213 --- /dev/null +++ b/lib/active_storage/disk_controller.rb @@ -0,0 +1,28 @@ +require "action_controller" +require "active_storage/blob" +require "active_storage/verified_key_with_expiration" + +require "active_support/core_ext/object/inclusion" + +class ActiveStorage::DiskController < ActionController::Base + def show + if key = decode_verified_key + blob = ActiveStorage::Blob.find_by!(key: key) + + if stale?(etag: blob.checksum) + send_data blob.download, filename: blob.filename, type: blob.content_type, disposition: disposition_param + end + else + head :not_found + end + end + + private + def decode_verified_key + ActiveStorage::VerifiedKeyWithExpiration.decode(params[:encoded_key]) + end + + def disposition_param + params[:disposition].presence_in(%w( inline attachment )) || 'inline' + end +end diff --git a/lib/active_storage/download.rb b/lib/active_storage/download.rb new file mode 100644 index 0000000000..4d656942d8 --- /dev/null +++ b/lib/active_storage/download.rb @@ -0,0 +1,90 @@ +class ActiveStorage::Download + # Sending .ai files as application/postscript to Safari opens them in a blank, grey screen. + # Downloading .ai as application/postscript files in Safari appends .ps to the extension. + # Sending HTML, SVG, XML and SWF files as binary closes XSS vulnerabilities. + # Sending JS files as binary avoids InvalidCrossOriginRequest without compromising security. + CONTENT_TYPES_TO_RENDER_AS_BINARY = %w( + text/html + text/javascript + image/svg+xml + application/postscript + application/x-shockwave-flash + text/xml + application/xml + application/xhtml+xml + ) + + BINARY_CONTENT_TYPE = 'application/octet-stream' + + def initialize(stored_file) + @stored_file = stored_file + end + + def headers(force_attachment: false) + { + x_accel_redirect: '/reproxy', + x_reproxy_url: reproxy_url, + content_type: content_type, + content_disposition: content_disposition(force_attachment), + x_frame_options: 'SAMEORIGIN' + } + end + + private + def reproxy_url + @stored_file.depot_location.paths.first + end + + def content_type + if @stored_file.content_type.in? CONTENT_TYPES_TO_RENDER_AS_BINARY + BINARY_CONTENT_TYPE + else + @stored_file.content_type + end + end + + def content_disposition(force_attachment = false) + if force_attachment || content_type == BINARY_CONTENT_TYPE + "attachment; #{escaped_filename}" + else + "inline; #{escaped_filename}" + end + end + + # RFC2231 encoding for UTF-8 filenames, with an ASCII fallback + # first for unsupported browsers (IE < 9, perhaps others?). + # http://greenbytes.de/tech/tc2231/#encoding-2231-fb + def escaped_filename + filename = @stored_file.filename.sanitized + ascii_filename = encode_ascii_filename(filename) + utf8_filename = encode_utf8_filename(filename) + "#{ascii_filename}; #{utf8_filename}" + end + + TRADITIONAL_PARAMETER_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ + + def encode_ascii_filename(filename) + # There is no reliable way to escape special or non-Latin characters + # in a traditionally quoted Content-Disposition filename parameter. + # Settle for transliterating to ASCII, then percent-escaping special + # characters, excluding spaces. + filename = I18n.transliterate(filename) + filename = percent_escape(filename, TRADITIONAL_PARAMETER_ESCAPED_CHAR) + %(filename="#{filename}") + end + + RFC5987_PARAMETER_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ + + def encode_utf8_filename(filename) + # RFC2231 filename parameters can simply be percent-escaped according + # to RFC5987. + filename = percent_escape(filename, RFC5987_PARAMETER_ESCAPED_CHAR) + %(filename*=UTF-8''#{filename}) + end + + def percent_escape(string, pattern) + string.gsub(pattern) do |char| + char.bytes.map { |byte| "%%%02X" % byte }.join("") + end + end +end diff --git a/lib/active_storage/filename.rb b/lib/active_storage/filename.rb new file mode 100644 index 0000000000..71614b5113 --- /dev/null +++ b/lib/active_storage/filename.rb @@ -0,0 +1,31 @@ +class ActiveStorage::Filename + include Comparable + + def initialize(filename) + @filename = filename + end + + def extname + File.extname(@filename) + end + + def extension + extname.from(1) + end + + def base + File.basename(@filename, extname) + end + + def sanitized + @filename.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-") + end + + def to_s + sanitized.to_s + end + + def <=>(other) + to_s.downcase <=> other.to_s.downcase + end +end diff --git a/lib/active_storage/migration.rb b/lib/active_storage/migration.rb new file mode 100644 index 0000000000..c0400abe3b --- /dev/null +++ b/lib/active_storage/migration.rb @@ -0,0 +1,28 @@ +class ActiveStorage::CreateTables < ActiveRecord::Migration[5.1] + def change + create_table :active_storage_blobs do |t| + t.string :key + t.string :filename + t.string :content_type + t.text :metadata + t.integer :byte_size + t.string :checksum + t.time :created_at + + t.index [ :key ], unique: true + end + + create_table :active_storage_attachments do |t| + t.string :name + t.string :record_gid + t.integer :blob_id + + t.time :created_at + + t.index :record_gid + t.index :blob_id + t.index [ :record_gid, :name ] + t.index [ :record_gid, :blob_id ], unique: true + end + end +end diff --git a/lib/active_storage/purge_job.rb b/lib/active_storage/purge_job.rb new file mode 100644 index 0000000000..b59d3687f8 --- /dev/null +++ b/lib/active_storage/purge_job.rb @@ -0,0 +1,10 @@ +require "active_job" + +class ActiveStorage::PurgeJob < ActiveJob::Base + # FIXME: Limit this to a custom ActiveStorage error + retry_on StandardError + + def perform(attachment_or_blob) + attachment_or_blob.purge + end +end diff --git a/lib/active_storage/railtie.rb b/lib/active_storage/railtie.rb new file mode 100644 index 0000000000..bf38d5aff5 --- /dev/null +++ b/lib/active_storage/railtie.rb @@ -0,0 +1,27 @@ +require "rails/railtie" + +module ActiveStorage + class Railtie < Rails::Railtie # :nodoc: + config.active_storage = ActiveSupport::OrderedOptions.new + + config.eager_load_namespaces << ActiveStorage + + initializer "active_storage.routes" do + require "active_storage/disk_controller" + + config.after_initialize do |app| + app.routes.prepend do + get "/rails/blobs/:encoded_key" => "active_storage/disk#show", as: :rails_disk_blob + end + end + end + + initializer "active_storage.attached" do + require "active_storage/attached" + + ActiveSupport.on_load(:active_record) do + extend ActiveStorage::Attached::Macros + end + end + end +end diff --git a/lib/active_storage/site.rb b/lib/active_storage/site.rb new file mode 100644 index 0000000000..b3b0221c63 --- /dev/null +++ b/lib/active_storage/site.rb @@ -0,0 +1,41 @@ +# Abstract class serving as an interface for concrete sites. +class ActiveStorage::Site + def self.configure(site, **options) + begin + require "active_storage/site/#{site.to_s.downcase}_site" + ActiveStorage::Site.const_get(:"#{site}Site").new(**options) + rescue LoadError => e + puts "Couldn't configure site: #{site} (#{e.message})" + end + end + + + def upload(key, io) + raise NotImplementedError + end + + def download(key) + raise NotImplementedError + end + + def delete(key) + raise NotImplementedError + end + + def exist?(key) + raise NotImplementedError + end + + + def url(key, expires_in:, disposition:, filename:) + raise NotImplementedError + end + + def bytesize(key) + raise NotImplementedError + end + + def checksum(key) + raise NotImplementedError + end +end diff --git a/lib/active_storage/site/disk_site.rb b/lib/active_storage/site/disk_site.rb new file mode 100644 index 0000000000..2ff0b22fae --- /dev/null +++ b/lib/active_storage/site/disk_site.rb @@ -0,0 +1,71 @@ +require "fileutils" +require "pathname" + +class ActiveStorage::Site::DiskSite < ActiveStorage::Site + attr_reader :root + + def initialize(root:) + @root = root + end + + def upload(key, io) + File.open(make_path_for(key), "wb") do |file| + while chunk = io.read(65536) + file.write(chunk) + end + end + end + + def download(key) + if block_given? + File.open(path_for(key)) do |file| + while data = file.read(65536) + yield data + end + end + else + File.open path_for(key), &:read + end + end + + def delete(key) + File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted + end + + def exist?(key) + File.exist? path_for(key) + end + + + def url(key, expires_in:, disposition:, filename:) + verified_key_with_expiration = ActiveStorage::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) + + if defined?(Rails) && defined?(Rails.application) + Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) + else + "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" + end + end + + def byte_size(key) + File.size path_for(key) + end + + def checksum(key) + Digest::MD5.file(path_for(key)).hexdigest + end + + + private + def path_for(key) + File.join root, folder_for(key), key + end + + def folder_for(key) + [ key[0..1], key[2..3] ].join("/") + end + + def make_path_for(key) + path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } + end +end diff --git a/lib/active_storage/site/gcs_site.rb b/lib/active_storage/site/gcs_site.rb new file mode 100644 index 0000000000..bf681ca6a2 --- /dev/null +++ b/lib/active_storage/site/gcs_site.rb @@ -0,0 +1,53 @@ +require "google/cloud/storage" +require "active_support/core_ext/object/to_query" + +class ActiveStorage::Site::GCSSite < ActiveStorage::Site + attr_reader :client, :bucket + + def initialize(project:, keyfile:, bucket:) + @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + bucket.create_file(io, key) + end + + def download(key) + io = file_for(key).download + io.rewind + io.read + end + + def delete(key) + file_for(key).try(:delete) + end + + def exist?(key) + file_for(key).present? + end + + + def url(key, expires_in:, disposition:, filename:) + file_for(key).signed_url(expires: expires_in) + "&" + + { "response-content-disposition" => "#{disposition}; filename=\"#{filename}\"" }.to_query + end + + def byte_size(key) + file_for(key).size + end + + def checksum(key) + convert_to_hex base64: file_for(key).md5 + end + + + private + def file_for(key) + bucket.file(key) + end + + def convert_to_hex(base64:) + base64.unpack("m0").first.unpack("H*").first + end +end diff --git a/lib/active_storage/site/mirror_site.rb b/lib/active_storage/site/mirror_site.rb new file mode 100644 index 0000000000..ba3ef0ef0e --- /dev/null +++ b/lib/active_storage/site/mirror_site.rb @@ -0,0 +1,51 @@ +class ActiveStorage::Site::MirrorSite < ActiveStorage::Site + attr_reader :sites + + def initialize(sites:) + @sites = sites + end + + def upload(key, io) + sites.collect do |site| + site.upload key, io + io.rewind + end + end + + def download(key) + sites.detect { |site| site.exist?(key) }.download(key) + end + + def delete(key) + perform_across_sites :delete, key + end + + def exist?(key) + perform_across_sites(:exist?, key).any? + end + + + def url(key, **options) + primary_site.url(key, **options) + end + + def byte_size(key) + primary_site.byte_size(key) + end + + def checksum(key) + primary_site.checksum(key) + end + + private + def primary_site + sites.first + end + + def perform_across_sites(method, *args) + # FIXME: Convert to be threaded + sites.collect do |site| + site.public_send method, *args + end + end +end diff --git a/lib/active_storage/site/s3_site.rb b/lib/active_storage/site/s3_site.rb new file mode 100644 index 0000000000..65dad37cfe --- /dev/null +++ b/lib/active_storage/site/s3_site.rb @@ -0,0 +1,63 @@ +require "aws-sdk" + +class ActiveStorage::Site::S3Site < ActiveStorage::Site + attr_reader :client, :bucket + + def initialize(access_key_id:, secret_access_key:, region:, bucket:) + @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + object_for(key).put(body: io) + end + + def download(key) + if block_given? + stream(key, &block) + else + object_for(key).get.body.read + end + end + + def delete(key) + object_for(key).delete + end + + def exist?(key) + object_for(key).exists? + end + + + def url(key, expires_in:, disposition:, filename:) + object_for(key).presigned_url :get, expires_in: expires_in, + response_content_disposition: "#{disposition}; filename=\"#{filename}\"" + end + + def byte_size(key) + object_for(key).size + end + + def checksum(key) + object_for(key).etag.remove(/"/) + end + + + private + def object_for(key) + bucket.object(key) + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key, options = {}, &block) + object = object_for(key) + + chunk_size = 5242880 # 5 megabytes + offset = 0 + + while offset < object.content_length + yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) + offset += chunk_size + end + end +end diff --git a/lib/active_storage/verified_key_with_expiration.rb b/lib/active_storage/verified_key_with_expiration.rb new file mode 100644 index 0000000000..8708106735 --- /dev/null +++ b/lib/active_storage/verified_key_with_expiration.rb @@ -0,0 +1,24 @@ +class ActiveStorage::VerifiedKeyWithExpiration + class_attribute :verifier, default: defined?(Rails) ? Rails.application.message_verifier('ActiveStorage') : nil + + class << self + def encode(key, expires_in: nil) + verifier.generate([ key, expires_at(expires_in) ]) + end + + def decode(encoded_key) + key, expires_at = verifier.verified(encoded_key) + + key if key && fresh?(expires_at) + end + + private + def expires_at(expires_in) + expires_in ? Time.now.utc.advance(seconds: expires_in) : nil + end + + def fresh?(expires_at) + expires_at.nil? || Time.now.utc < expires_at + end + end +end |