From c624df326a4ef36919a5195a3c5509fab97dcba3 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Thu, 6 Jul 2017 11:33:29 +0200 Subject: ActiveVault -> ActiveStorage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Yaroslav agreed to hand over the gem name ❤️ --- lib/active_storage.rb | 9 +++ lib/active_storage/attached.rb | 34 ++++++++ lib/active_storage/attached/macros.rb | 23 ++++++ lib/active_storage/attached/many.rb | 31 ++++++++ lib/active_storage/attached/one.rb | 29 +++++++ lib/active_storage/attachment.rb | 30 ++++++++ lib/active_storage/blob.rb | 68 ++++++++++++++++ lib/active_storage/config/sites.yml | 25 ++++++ lib/active_storage/disk_controller.rb | 28 +++++++ lib/active_storage/download.rb | 90 ++++++++++++++++++++++ lib/active_storage/filename.rb | 31 ++++++++ lib/active_storage/migration.rb | 28 +++++++ lib/active_storage/purge_job.rb | 10 +++ lib/active_storage/railtie.rb | 27 +++++++ lib/active_storage/site.rb | 41 ++++++++++ lib/active_storage/site/disk_site.rb | 71 +++++++++++++++++ lib/active_storage/site/gcs_site.rb | 53 +++++++++++++ lib/active_storage/site/mirror_site.rb | 51 ++++++++++++ lib/active_storage/site/s3_site.rb | 63 +++++++++++++++ lib/active_storage/verified_key_with_expiration.rb | 24 ++++++ lib/active_vault.rb | 9 --- lib/active_vault/attached.rb | 34 -------- lib/active_vault/attached/macros.rb | 23 ------ lib/active_vault/attached/many.rb | 31 -------- lib/active_vault/attached/one.rb | 29 ------- lib/active_vault/attachment.rb | 30 -------- lib/active_vault/blob.rb | 68 ---------------- lib/active_vault/config/sites.yml | 25 ------ lib/active_vault/disk_controller.rb | 28 ------- lib/active_vault/download.rb | 90 ---------------------- lib/active_vault/filename.rb | 31 -------- lib/active_vault/migration.rb | 28 ------- lib/active_vault/purge_job.rb | 10 --- lib/active_vault/railtie.rb | 27 ------- lib/active_vault/site.rb | 41 ---------- lib/active_vault/site/disk_site.rb | 71 ----------------- lib/active_vault/site/gcs_site.rb | 53 ------------- lib/active_vault/site/mirror_site.rb | 51 ------------ lib/active_vault/site/s3_site.rb | 63 --------------- lib/active_vault/verified_key_with_expiration.rb | 24 ------ 40 files changed, 766 insertions(+), 766 deletions(-) create mode 100644 lib/active_storage.rb create mode 100644 lib/active_storage/attached.rb create mode 100644 lib/active_storage/attached/macros.rb create mode 100644 lib/active_storage/attached/many.rb create mode 100644 lib/active_storage/attached/one.rb create mode 100644 lib/active_storage/attachment.rb create mode 100644 lib/active_storage/blob.rb create mode 100644 lib/active_storage/config/sites.yml create mode 100644 lib/active_storage/disk_controller.rb create mode 100644 lib/active_storage/download.rb create mode 100644 lib/active_storage/filename.rb create mode 100644 lib/active_storage/migration.rb create mode 100644 lib/active_storage/purge_job.rb create mode 100644 lib/active_storage/railtie.rb create mode 100644 lib/active_storage/site.rb create mode 100644 lib/active_storage/site/disk_site.rb create mode 100644 lib/active_storage/site/gcs_site.rb create mode 100644 lib/active_storage/site/mirror_site.rb create mode 100644 lib/active_storage/site/s3_site.rb create mode 100644 lib/active_storage/verified_key_with_expiration.rb delete mode 100644 lib/active_vault.rb delete mode 100644 lib/active_vault/attached.rb delete mode 100644 lib/active_vault/attached/macros.rb delete mode 100644 lib/active_vault/attached/many.rb delete mode 100644 lib/active_vault/attached/one.rb delete mode 100644 lib/active_vault/attachment.rb delete mode 100644 lib/active_vault/blob.rb delete mode 100644 lib/active_vault/config/sites.yml delete mode 100644 lib/active_vault/disk_controller.rb delete mode 100644 lib/active_vault/download.rb delete mode 100644 lib/active_vault/filename.rb delete mode 100644 lib/active_vault/migration.rb delete mode 100644 lib/active_vault/purge_job.rb delete mode 100644 lib/active_vault/railtie.rb delete mode 100644 lib/active_vault/site.rb delete mode 100644 lib/active_vault/site/disk_site.rb delete mode 100644 lib/active_vault/site/gcs_site.rb delete mode 100644 lib/active_vault/site/mirror_site.rb delete mode 100644 lib/active_vault/site/s3_site.rb delete mode 100644 lib/active_vault/verified_key_with_expiration.rb (limited to 'lib') diff --git a/lib/active_storage.rb b/lib/active_storage.rb new file mode 100644 index 0000000000..e87eb8a506 --- /dev/null +++ b/lib/active_storage.rb @@ -0,0 +1,9 @@ +require "active_record" +require "active_storage/railtie" if defined?(Rails) + +module ActiveStorage + extend ActiveSupport::Autoload + + autoload :Blob + autoload :Site +end diff --git a/lib/active_storage/attached.rb b/lib/active_storage/attached.rb new file mode 100644 index 0000000000..7475c38999 --- /dev/null +++ b/lib/active_storage/attached.rb @@ -0,0 +1,34 @@ +require "active_storage/blob" +require "active_storage/attachment" + +require "action_dispatch/http/upload" +require "active_support/core_ext/module/delegation" + +class ActiveStorage::Attached + attr_reader :name, :record + + def initialize(name, record) + @name, @record = name, record + end + + private + def create_blob_from(attachable) + case attachable + when ActiveStorage::Blob + attachable + when ActionDispatch::Http::UploadedFile + ActiveStorage::Blob.create_after_upload! \ + io: attachable.open, + filename: attachable.original_filename, + content_type: attachable.content_type + when Hash + ActiveStorage::Blob.create_after_upload!(attachable) + else + nil + end + end +end + +require "active_storage/attached/one" +require "active_storage/attached/many" +require "active_storage/attached/macros" diff --git a/lib/active_storage/attached/macros.rb b/lib/active_storage/attached/macros.rb new file mode 100644 index 0000000000..96493d1215 --- /dev/null +++ b/lib/active_storage/attached/macros.rb @@ -0,0 +1,23 @@ +module ActiveStorage::Attached::Macros + def has_one_attached(name, dependent: :purge_later) + define_method(name) do + instance_variable_get("@active_storage_attached_#{name}") || + instance_variable_set("@active_storage_attached_#{name}", ActiveStorage::Attached::One.new(name, self)) + end + + if dependent == :purge_later + before_destroy { public_send(name).purge_later } + end + end + + def has_many_attached(name, dependent: :purge_later) + define_method(name) do + instance_variable_get("@active_storage_attached_#{name}") || + instance_variable_set("@active_storage_attached_#{name}", ActiveStorage::Attached::Many.new(name, self)) + end + + if dependent == :purge_later + before_destroy { public_send(name).purge_later } + end + end +end diff --git a/lib/active_storage/attached/many.rb b/lib/active_storage/attached/many.rb new file mode 100644 index 0000000000..f1535dfbc6 --- /dev/null +++ b/lib/active_storage/attached/many.rb @@ -0,0 +1,31 @@ +class ActiveStorage::Attached::Many < ActiveStorage::Attached + delegate_missing_to :attachments + + def attachments + @attachments ||= ActiveStorage::Attachment.where(record_gid: record.to_gid.to_s, name: name) + end + + def attach(*attachables) + @attachments = attachments | Array(attachables).flatten.collect do |attachable| + ActiveStorage::Attachment.create!(record_gid: record.to_gid.to_s, name: name, blob: create_blob_from(attachable)) + end + end + + def attached? + attachments.any? + end + + def purge + if attached? + attachments.each(&:purge) + @attachments = nil + end + end + + def purge_later + if attached? + attachments.each(&:purge_later) + @attachments = nil + end + end +end diff --git a/lib/active_storage/attached/one.rb b/lib/active_storage/attached/one.rb new file mode 100644 index 0000000000..d08d265992 --- /dev/null +++ b/lib/active_storage/attached/one.rb @@ -0,0 +1,29 @@ +class ActiveStorage::Attached::One < ActiveStorage::Attached + delegate_missing_to :attachment + + def attachment + @attachment ||= ActiveStorage::Attachment.find_by(record_gid: record.to_gid.to_s, name: name) + end + + def attach(attachable) + @attachment = ActiveStorage::Attachment.create!(record_gid: record.to_gid.to_s, name: name, blob: create_blob_from(attachable)) + end + + def attached? + attachment.present? + end + + def purge + if attached? + attachment.purge + @attachment = nil + end + end + + def purge_later + if attached? + attachment.purge_later + @attachment = nil + end + end +end diff --git a/lib/active_storage/attachment.rb b/lib/active_storage/attachment.rb new file mode 100644 index 0000000000..20c619aa5a --- /dev/null +++ b/lib/active_storage/attachment.rb @@ -0,0 +1,30 @@ +require "active_storage/blob" +require "global_id" +require "active_support/core_ext/module/delegation" + +# Schema: id, record_gid, blob_id, created_at +class ActiveStorage::Attachment < ActiveRecord::Base + self.table_name = "active_storage_attachments" + + belongs_to :blob, class_name: "ActiveStorage::Blob" + + delegate_missing_to :blob + + def record + @record ||= GlobalID::Locator.locate(record_gid) + end + + def record=(record) + @record = record + self.record_gid = record&.to_gid + end + + def purge + blob.purge + destroy + end + + def purge_later + ActiveStorage::PurgeJob.perform_later(self) + end +end diff --git a/lib/active_storage/blob.rb b/lib/active_storage/blob.rb new file mode 100644 index 0000000000..edf57b5c78 --- /dev/null +++ b/lib/active_storage/blob.rb @@ -0,0 +1,68 @@ +require "active_storage/site" +require "active_storage/filename" +require "active_storage/purge_job" + +# Schema: id, key, filename, content_type, metadata, byte_size, checksum, created_at +class ActiveStorage::Blob < ActiveRecord::Base + self.table_name = "active_storage_blobs" + + has_secure_token :key + store :metadata, coder: JSON + + class_attribute :site + + class << self + def build_after_upload(io:, filename:, content_type: nil, metadata: nil) + new.tap do |blob| + blob.filename = filename + blob.content_type = content_type + blob.metadata = metadata + + blob.upload io + end + end + + def create_after_upload!(io:, filename:, content_type: nil, metadata: nil) + build_after_upload(io: io, filename: filename, content_type: content_type, metadata: metadata).tap(&:save!) + end + end + + # We can't wait until the record is first saved to have a key for it + def key + self[:key] ||= self.class.generate_unique_secure_token + end + + def filename + ActiveStorage::Filename.new(self[:filename]) + end + + def url(expires_in: 5.minutes, disposition: :inline) + site.url key, expires_in: expires_in, disposition: disposition, filename: filename + end + + + def upload(io) + site.upload(key, io) + + self.checksum = site.checksum(key) + self.byte_size = site.byte_size(key) + end + + def download + site.download key + end + + + def delete + site.delete key + end + + def purge + delete + destroy + end + + def purge_later + ActiveStorage::PurgeJob.perform_later(self) + end +end diff --git a/lib/active_storage/config/sites.yml b/lib/active_storage/config/sites.yml new file mode 100644 index 0000000000..43bc77fbf9 --- /dev/null +++ b/lib/active_storage/config/sites.yml @@ -0,0 +1,25 @@ +# Configuration should be something like this: +# +# config/environments/development.rb +# config.active_storage.site = :local +# +# config/environments/production.rb +# config.active_storage.site = :amazon +local: + site: Disk + root: <%%= File.join(Dir.tmpdir, "active_storage") %> + +amazon: + site: S3 + access_key_id: <%%= Rails.application.secrets.aws[:access_key_id] %> + secret_access_key: <%%= Rails.application.secrets.aws[:secret_access_key] %> + region: us-east-1 + bucket: <%= Rails.application.class.name.remove(/::Application$/).underscore %> + +google: + site: GCS + +mirror: + site: Mirror + primary: amazon + secondaries: google diff --git a/lib/active_storage/disk_controller.rb b/lib/active_storage/disk_controller.rb new file mode 100644 index 0000000000..3eba86c213 --- /dev/null +++ b/lib/active_storage/disk_controller.rb @@ -0,0 +1,28 @@ +require "action_controller" +require "active_storage/blob" +require "active_storage/verified_key_with_expiration" + +require "active_support/core_ext/object/inclusion" + +class ActiveStorage::DiskController < ActionController::Base + def show + if key = decode_verified_key + blob = ActiveStorage::Blob.find_by!(key: key) + + if stale?(etag: blob.checksum) + send_data blob.download, filename: blob.filename, type: blob.content_type, disposition: disposition_param + end + else + head :not_found + end + end + + private + def decode_verified_key + ActiveStorage::VerifiedKeyWithExpiration.decode(params[:encoded_key]) + end + + def disposition_param + params[:disposition].presence_in(%w( inline attachment )) || 'inline' + end +end diff --git a/lib/active_storage/download.rb b/lib/active_storage/download.rb new file mode 100644 index 0000000000..4d656942d8 --- /dev/null +++ b/lib/active_storage/download.rb @@ -0,0 +1,90 @@ +class ActiveStorage::Download + # Sending .ai files as application/postscript to Safari opens them in a blank, grey screen. + # Downloading .ai as application/postscript files in Safari appends .ps to the extension. + # Sending HTML, SVG, XML and SWF files as binary closes XSS vulnerabilities. + # Sending JS files as binary avoids InvalidCrossOriginRequest without compromising security. + CONTENT_TYPES_TO_RENDER_AS_BINARY = %w( + text/html + text/javascript + image/svg+xml + application/postscript + application/x-shockwave-flash + text/xml + application/xml + application/xhtml+xml + ) + + BINARY_CONTENT_TYPE = 'application/octet-stream' + + def initialize(stored_file) + @stored_file = stored_file + end + + def headers(force_attachment: false) + { + x_accel_redirect: '/reproxy', + x_reproxy_url: reproxy_url, + content_type: content_type, + content_disposition: content_disposition(force_attachment), + x_frame_options: 'SAMEORIGIN' + } + end + + private + def reproxy_url + @stored_file.depot_location.paths.first + end + + def content_type + if @stored_file.content_type.in? CONTENT_TYPES_TO_RENDER_AS_BINARY + BINARY_CONTENT_TYPE + else + @stored_file.content_type + end + end + + def content_disposition(force_attachment = false) + if force_attachment || content_type == BINARY_CONTENT_TYPE + "attachment; #{escaped_filename}" + else + "inline; #{escaped_filename}" + end + end + + # RFC2231 encoding for UTF-8 filenames, with an ASCII fallback + # first for unsupported browsers (IE < 9, perhaps others?). + # http://greenbytes.de/tech/tc2231/#encoding-2231-fb + def escaped_filename + filename = @stored_file.filename.sanitized + ascii_filename = encode_ascii_filename(filename) + utf8_filename = encode_utf8_filename(filename) + "#{ascii_filename}; #{utf8_filename}" + end + + TRADITIONAL_PARAMETER_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ + + def encode_ascii_filename(filename) + # There is no reliable way to escape special or non-Latin characters + # in a traditionally quoted Content-Disposition filename parameter. + # Settle for transliterating to ASCII, then percent-escaping special + # characters, excluding spaces. + filename = I18n.transliterate(filename) + filename = percent_escape(filename, TRADITIONAL_PARAMETER_ESCAPED_CHAR) + %(filename="#{filename}") + end + + RFC5987_PARAMETER_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ + + def encode_utf8_filename(filename) + # RFC2231 filename parameters can simply be percent-escaped according + # to RFC5987. + filename = percent_escape(filename, RFC5987_PARAMETER_ESCAPED_CHAR) + %(filename*=UTF-8''#{filename}) + end + + def percent_escape(string, pattern) + string.gsub(pattern) do |char| + char.bytes.map { |byte| "%%%02X" % byte }.join("") + end + end +end diff --git a/lib/active_storage/filename.rb b/lib/active_storage/filename.rb new file mode 100644 index 0000000000..71614b5113 --- /dev/null +++ b/lib/active_storage/filename.rb @@ -0,0 +1,31 @@ +class ActiveStorage::Filename + include Comparable + + def initialize(filename) + @filename = filename + end + + def extname + File.extname(@filename) + end + + def extension + extname.from(1) + end + + def base + File.basename(@filename, extname) + end + + def sanitized + @filename.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-") + end + + def to_s + sanitized.to_s + end + + def <=>(other) + to_s.downcase <=> other.to_s.downcase + end +end diff --git a/lib/active_storage/migration.rb b/lib/active_storage/migration.rb new file mode 100644 index 0000000000..c0400abe3b --- /dev/null +++ b/lib/active_storage/migration.rb @@ -0,0 +1,28 @@ +class ActiveStorage::CreateTables < ActiveRecord::Migration[5.1] + def change + create_table :active_storage_blobs do |t| + t.string :key + t.string :filename + t.string :content_type + t.text :metadata + t.integer :byte_size + t.string :checksum + t.time :created_at + + t.index [ :key ], unique: true + end + + create_table :active_storage_attachments do |t| + t.string :name + t.string :record_gid + t.integer :blob_id + + t.time :created_at + + t.index :record_gid + t.index :blob_id + t.index [ :record_gid, :name ] + t.index [ :record_gid, :blob_id ], unique: true + end + end +end diff --git a/lib/active_storage/purge_job.rb b/lib/active_storage/purge_job.rb new file mode 100644 index 0000000000..b59d3687f8 --- /dev/null +++ b/lib/active_storage/purge_job.rb @@ -0,0 +1,10 @@ +require "active_job" + +class ActiveStorage::PurgeJob < ActiveJob::Base + # FIXME: Limit this to a custom ActiveStorage error + retry_on StandardError + + def perform(attachment_or_blob) + attachment_or_blob.purge + end +end diff --git a/lib/active_storage/railtie.rb b/lib/active_storage/railtie.rb new file mode 100644 index 0000000000..bf38d5aff5 --- /dev/null +++ b/lib/active_storage/railtie.rb @@ -0,0 +1,27 @@ +require "rails/railtie" + +module ActiveStorage + class Railtie < Rails::Railtie # :nodoc: + config.active_storage = ActiveSupport::OrderedOptions.new + + config.eager_load_namespaces << ActiveStorage + + initializer "active_storage.routes" do + require "active_storage/disk_controller" + + config.after_initialize do |app| + app.routes.prepend do + get "/rails/blobs/:encoded_key" => "active_storage/disk#show", as: :rails_disk_blob + end + end + end + + initializer "active_storage.attached" do + require "active_storage/attached" + + ActiveSupport.on_load(:active_record) do + extend ActiveStorage::Attached::Macros + end + end + end +end diff --git a/lib/active_storage/site.rb b/lib/active_storage/site.rb new file mode 100644 index 0000000000..b3b0221c63 --- /dev/null +++ b/lib/active_storage/site.rb @@ -0,0 +1,41 @@ +# Abstract class serving as an interface for concrete sites. +class ActiveStorage::Site + def self.configure(site, **options) + begin + require "active_storage/site/#{site.to_s.downcase}_site" + ActiveStorage::Site.const_get(:"#{site}Site").new(**options) + rescue LoadError => e + puts "Couldn't configure site: #{site} (#{e.message})" + end + end + + + def upload(key, io) + raise NotImplementedError + end + + def download(key) + raise NotImplementedError + end + + def delete(key) + raise NotImplementedError + end + + def exist?(key) + raise NotImplementedError + end + + + def url(key, expires_in:, disposition:, filename:) + raise NotImplementedError + end + + def bytesize(key) + raise NotImplementedError + end + + def checksum(key) + raise NotImplementedError + end +end diff --git a/lib/active_storage/site/disk_site.rb b/lib/active_storage/site/disk_site.rb new file mode 100644 index 0000000000..2ff0b22fae --- /dev/null +++ b/lib/active_storage/site/disk_site.rb @@ -0,0 +1,71 @@ +require "fileutils" +require "pathname" + +class ActiveStorage::Site::DiskSite < ActiveStorage::Site + attr_reader :root + + def initialize(root:) + @root = root + end + + def upload(key, io) + File.open(make_path_for(key), "wb") do |file| + while chunk = io.read(65536) + file.write(chunk) + end + end + end + + def download(key) + if block_given? + File.open(path_for(key)) do |file| + while data = file.read(65536) + yield data + end + end + else + File.open path_for(key), &:read + end + end + + def delete(key) + File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted + end + + def exist?(key) + File.exist? path_for(key) + end + + + def url(key, expires_in:, disposition:, filename:) + verified_key_with_expiration = ActiveStorage::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) + + if defined?(Rails) && defined?(Rails.application) + Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) + else + "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" + end + end + + def byte_size(key) + File.size path_for(key) + end + + def checksum(key) + Digest::MD5.file(path_for(key)).hexdigest + end + + + private + def path_for(key) + File.join root, folder_for(key), key + end + + def folder_for(key) + [ key[0..1], key[2..3] ].join("/") + end + + def make_path_for(key) + path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } + end +end diff --git a/lib/active_storage/site/gcs_site.rb b/lib/active_storage/site/gcs_site.rb new file mode 100644 index 0000000000..bf681ca6a2 --- /dev/null +++ b/lib/active_storage/site/gcs_site.rb @@ -0,0 +1,53 @@ +require "google/cloud/storage" +require "active_support/core_ext/object/to_query" + +class ActiveStorage::Site::GCSSite < ActiveStorage::Site + attr_reader :client, :bucket + + def initialize(project:, keyfile:, bucket:) + @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + bucket.create_file(io, key) + end + + def download(key) + io = file_for(key).download + io.rewind + io.read + end + + def delete(key) + file_for(key).try(:delete) + end + + def exist?(key) + file_for(key).present? + end + + + def url(key, expires_in:, disposition:, filename:) + file_for(key).signed_url(expires: expires_in) + "&" + + { "response-content-disposition" => "#{disposition}; filename=\"#{filename}\"" }.to_query + end + + def byte_size(key) + file_for(key).size + end + + def checksum(key) + convert_to_hex base64: file_for(key).md5 + end + + + private + def file_for(key) + bucket.file(key) + end + + def convert_to_hex(base64:) + base64.unpack("m0").first.unpack("H*").first + end +end diff --git a/lib/active_storage/site/mirror_site.rb b/lib/active_storage/site/mirror_site.rb new file mode 100644 index 0000000000..ba3ef0ef0e --- /dev/null +++ b/lib/active_storage/site/mirror_site.rb @@ -0,0 +1,51 @@ +class ActiveStorage::Site::MirrorSite < ActiveStorage::Site + attr_reader :sites + + def initialize(sites:) + @sites = sites + end + + def upload(key, io) + sites.collect do |site| + site.upload key, io + io.rewind + end + end + + def download(key) + sites.detect { |site| site.exist?(key) }.download(key) + end + + def delete(key) + perform_across_sites :delete, key + end + + def exist?(key) + perform_across_sites(:exist?, key).any? + end + + + def url(key, **options) + primary_site.url(key, **options) + end + + def byte_size(key) + primary_site.byte_size(key) + end + + def checksum(key) + primary_site.checksum(key) + end + + private + def primary_site + sites.first + end + + def perform_across_sites(method, *args) + # FIXME: Convert to be threaded + sites.collect do |site| + site.public_send method, *args + end + end +end diff --git a/lib/active_storage/site/s3_site.rb b/lib/active_storage/site/s3_site.rb new file mode 100644 index 0000000000..65dad37cfe --- /dev/null +++ b/lib/active_storage/site/s3_site.rb @@ -0,0 +1,63 @@ +require "aws-sdk" + +class ActiveStorage::Site::S3Site < ActiveStorage::Site + attr_reader :client, :bucket + + def initialize(access_key_id:, secret_access_key:, region:, bucket:) + @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + object_for(key).put(body: io) + end + + def download(key) + if block_given? + stream(key, &block) + else + object_for(key).get.body.read + end + end + + def delete(key) + object_for(key).delete + end + + def exist?(key) + object_for(key).exists? + end + + + def url(key, expires_in:, disposition:, filename:) + object_for(key).presigned_url :get, expires_in: expires_in, + response_content_disposition: "#{disposition}; filename=\"#{filename}\"" + end + + def byte_size(key) + object_for(key).size + end + + def checksum(key) + object_for(key).etag.remove(/"/) + end + + + private + def object_for(key) + bucket.object(key) + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key, options = {}, &block) + object = object_for(key) + + chunk_size = 5242880 # 5 megabytes + offset = 0 + + while offset < object.content_length + yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) + offset += chunk_size + end + end +end diff --git a/lib/active_storage/verified_key_with_expiration.rb b/lib/active_storage/verified_key_with_expiration.rb new file mode 100644 index 0000000000..8708106735 --- /dev/null +++ b/lib/active_storage/verified_key_with_expiration.rb @@ -0,0 +1,24 @@ +class ActiveStorage::VerifiedKeyWithExpiration + class_attribute :verifier, default: defined?(Rails) ? Rails.application.message_verifier('ActiveStorage') : nil + + class << self + def encode(key, expires_in: nil) + verifier.generate([ key, expires_at(expires_in) ]) + end + + def decode(encoded_key) + key, expires_at = verifier.verified(encoded_key) + + key if key && fresh?(expires_at) + end + + private + def expires_at(expires_in) + expires_in ? Time.now.utc.advance(seconds: expires_in) : nil + end + + def fresh?(expires_at) + expires_at.nil? || Time.now.utc < expires_at + end + end +end diff --git a/lib/active_vault.rb b/lib/active_vault.rb deleted file mode 100644 index f47b09b4cd..0000000000 --- a/lib/active_vault.rb +++ /dev/null @@ -1,9 +0,0 @@ -require "active_record" -require "active_vault/railtie" if defined?(Rails) - -module ActiveVault - extend ActiveSupport::Autoload - - autoload :Blob - autoload :Site -end diff --git a/lib/active_vault/attached.rb b/lib/active_vault/attached.rb deleted file mode 100644 index a968f3500d..0000000000 --- a/lib/active_vault/attached.rb +++ /dev/null @@ -1,34 +0,0 @@ -require "active_vault/blob" -require "active_vault/attachment" - -require "action_dispatch/http/upload" -require "active_support/core_ext/module/delegation" - -class ActiveVault::Attached - attr_reader :name, :record - - def initialize(name, record) - @name, @record = name, record - end - - private - def create_blob_from(attachable) - case attachable - when ActiveVault::Blob - attachable - when ActionDispatch::Http::UploadedFile - ActiveVault::Blob.create_after_upload! \ - io: attachable.open, - filename: attachable.original_filename, - content_type: attachable.content_type - when Hash - ActiveVault::Blob.create_after_upload!(attachable) - else - nil - end - end -end - -require "active_vault/attached/one" -require "active_vault/attached/many" -require "active_vault/attached/macros" diff --git a/lib/active_vault/attached/macros.rb b/lib/active_vault/attached/macros.rb deleted file mode 100644 index 1b95c14c9c..0000000000 --- a/lib/active_vault/attached/macros.rb +++ /dev/null @@ -1,23 +0,0 @@ -module ActiveVault::Attached::Macros - def has_one_attached(name, dependent: :purge_later) - define_method(name) do - instance_variable_get("@active_vault_attached_#{name}") || - instance_variable_set("@active_vault_attached_#{name}", ActiveVault::Attached::One.new(name, self)) - end - - if dependent == :purge_later - before_destroy { public_send(name).purge_later } - end - end - - def has_many_attached(name, dependent: :purge_later) - define_method(name) do - instance_variable_get("@active_vault_attached_#{name}") || - instance_variable_set("@active_vault_attached_#{name}", ActiveVault::Attached::Many.new(name, self)) - end - - if dependent == :purge_later - before_destroy { public_send(name).purge_later } - end - end -end diff --git a/lib/active_vault/attached/many.rb b/lib/active_vault/attached/many.rb deleted file mode 100644 index 6f79a1c555..0000000000 --- a/lib/active_vault/attached/many.rb +++ /dev/null @@ -1,31 +0,0 @@ -class ActiveVault::Attached::Many < ActiveVault::Attached - delegate_missing_to :attachments - - def attachments - @attachments ||= ActiveVault::Attachment.where(record_gid: record.to_gid.to_s, name: name) - end - - def attach(*attachables) - @attachments = attachments | Array(attachables).flatten.collect do |attachable| - ActiveVault::Attachment.create!(record_gid: record.to_gid.to_s, name: name, blob: create_blob_from(attachable)) - end - end - - def attached? - attachments.any? - end - - def purge - if attached? - attachments.each(&:purge) - @attachments = nil - end - end - - def purge_later - if attached? - attachments.each(&:purge_later) - @attachments = nil - end - end -end diff --git a/lib/active_vault/attached/one.rb b/lib/active_vault/attached/one.rb deleted file mode 100644 index 01a5d0d6f0..0000000000 --- a/lib/active_vault/attached/one.rb +++ /dev/null @@ -1,29 +0,0 @@ -class ActiveVault::Attached::One < ActiveVault::Attached - delegate_missing_to :attachment - - def attachment - @attachment ||= ActiveVault::Attachment.find_by(record_gid: record.to_gid.to_s, name: name) - end - - def attach(attachable) - @attachment = ActiveVault::Attachment.create!(record_gid: record.to_gid.to_s, name: name, blob: create_blob_from(attachable)) - end - - def attached? - attachment.present? - end - - def purge - if attached? - attachment.purge - @attachment = nil - end - end - - def purge_later - if attached? - attachment.purge_later - @attachment = nil - end - end -end diff --git a/lib/active_vault/attachment.rb b/lib/active_vault/attachment.rb deleted file mode 100644 index 549a734d68..0000000000 --- a/lib/active_vault/attachment.rb +++ /dev/null @@ -1,30 +0,0 @@ -require "active_vault/blob" -require "global_id" -require "active_support/core_ext/module/delegation" - -# Schema: id, record_gid, blob_id, created_at -class ActiveVault::Attachment < ActiveRecord::Base - self.table_name = "active_vault_attachments" - - belongs_to :blob, class_name: "ActiveVault::Blob" - - delegate_missing_to :blob - - def record - @record ||= GlobalID::Locator.locate(record_gid) - end - - def record=(record) - @record = record - self.record_gid = record&.to_gid - end - - def purge - blob.purge - destroy - end - - def purge_later - ActiveVault::PurgeJob.perform_later(self) - end -end diff --git a/lib/active_vault/blob.rb b/lib/active_vault/blob.rb deleted file mode 100644 index a232ca5c1a..0000000000 --- a/lib/active_vault/blob.rb +++ /dev/null @@ -1,68 +0,0 @@ -require "active_vault/site" -require "active_vault/filename" -require "active_vault/purge_job" - -# Schema: id, key, filename, content_type, metadata, byte_size, checksum, created_at -class ActiveVault::Blob < ActiveRecord::Base - self.table_name = "active_vault_blobs" - - has_secure_token :key - store :metadata, coder: JSON - - class_attribute :site - - class << self - def build_after_upload(io:, filename:, content_type: nil, metadata: nil) - new.tap do |blob| - blob.filename = filename - blob.content_type = content_type - blob.metadata = metadata - - blob.upload io - end - end - - def create_after_upload!(io:, filename:, content_type: nil, metadata: nil) - build_after_upload(io: io, filename: filename, content_type: content_type, metadata: metadata).tap(&:save!) - end - end - - # We can't wait until the record is first saved to have a key for it - def key - self[:key] ||= self.class.generate_unique_secure_token - end - - def filename - ActiveVault::Filename.new(self[:filename]) - end - - def url(expires_in: 5.minutes, disposition: :inline) - site.url key, expires_in: expires_in, disposition: disposition, filename: filename - end - - - def upload(io) - site.upload(key, io) - - self.checksum = site.checksum(key) - self.byte_size = site.byte_size(key) - end - - def download - site.download key - end - - - def delete - site.delete key - end - - def purge - delete - destroy - end - - def purge_later - ActiveVault::PurgeJob.perform_later(self) - end -end diff --git a/lib/active_vault/config/sites.yml b/lib/active_vault/config/sites.yml deleted file mode 100644 index 334e779b28..0000000000 --- a/lib/active_vault/config/sites.yml +++ /dev/null @@ -1,25 +0,0 @@ -# Configuration should be something like this: -# -# config/environments/development.rb -# config.active_vault.site = :local -# -# config/environments/production.rb -# config.active_vault.site = :amazon -local: - site: Disk - root: <%%= File.join(Dir.tmpdir, "active_vault") %> - -amazon: - site: S3 - access_key_id: <%%= Rails.application.secrets.aws[:access_key_id] %> - secret_access_key: <%%= Rails.application.secrets.aws[:secret_access_key] %> - region: us-east-1 - bucket: <%= Rails.application.class.name.remove(/::Application$/).underscore %> - -google: - site: GCS - -mirror: - site: Mirror - primary: amazon - secondaries: google diff --git a/lib/active_vault/disk_controller.rb b/lib/active_vault/disk_controller.rb deleted file mode 100644 index 623569f0f6..0000000000 --- a/lib/active_vault/disk_controller.rb +++ /dev/null @@ -1,28 +0,0 @@ -require "action_controller" -require "active_vault/blob" -require "active_vault/verified_key_with_expiration" - -require "active_support/core_ext/object/inclusion" - -class ActiveVault::DiskController < ActionController::Base - def show - if key = decode_verified_key - blob = ActiveVault::Blob.find_by!(key: key) - - if stale?(etag: blob.checksum) - send_data blob.download, filename: blob.filename, type: blob.content_type, disposition: disposition_param - end - else - head :not_found - end - end - - private - def decode_verified_key - ActiveVault::VerifiedKeyWithExpiration.decode(params[:encoded_key]) - end - - def disposition_param - params[:disposition].presence_in(%w( inline attachment )) || 'inline' - end -end diff --git a/lib/active_vault/download.rb b/lib/active_vault/download.rb deleted file mode 100644 index 6e74056062..0000000000 --- a/lib/active_vault/download.rb +++ /dev/null @@ -1,90 +0,0 @@ -class ActiveVault::Download - # Sending .ai files as application/postscript to Safari opens them in a blank, grey screen. - # Downloading .ai as application/postscript files in Safari appends .ps to the extension. - # Sending HTML, SVG, XML and SWF files as binary closes XSS vulnerabilities. - # Sending JS files as binary avoids InvalidCrossOriginRequest without compromising security. - CONTENT_TYPES_TO_RENDER_AS_BINARY = %w( - text/html - text/javascript - image/svg+xml - application/postscript - application/x-shockwave-flash - text/xml - application/xml - application/xhtml+xml - ) - - BINARY_CONTENT_TYPE = 'application/octet-stream' - - def initialize(stored_file) - @stored_file = stored_file - end - - def headers(force_attachment: false) - { - x_accel_redirect: '/reproxy', - x_reproxy_url: reproxy_url, - content_type: content_type, - content_disposition: content_disposition(force_attachment), - x_frame_options: 'SAMEORIGIN' - } - end - - private - def reproxy_url - @stored_file.depot_location.paths.first - end - - def content_type - if @stored_file.content_type.in? CONTENT_TYPES_TO_RENDER_AS_BINARY - BINARY_CONTENT_TYPE - else - @stored_file.content_type - end - end - - def content_disposition(force_attachment = false) - if force_attachment || content_type == BINARY_CONTENT_TYPE - "attachment; #{escaped_filename}" - else - "inline; #{escaped_filename}" - end - end - - # RFC2231 encoding for UTF-8 filenames, with an ASCII fallback - # first for unsupported browsers (IE < 9, perhaps others?). - # http://greenbytes.de/tech/tc2231/#encoding-2231-fb - def escaped_filename - filename = @stored_file.filename.sanitized - ascii_filename = encode_ascii_filename(filename) - utf8_filename = encode_utf8_filename(filename) - "#{ascii_filename}; #{utf8_filename}" - end - - TRADITIONAL_PARAMETER_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ - - def encode_ascii_filename(filename) - # There is no reliable way to escape special or non-Latin characters - # in a traditionally quoted Content-Disposition filename parameter. - # Settle for transliterating to ASCII, then percent-escaping special - # characters, excluding spaces. - filename = I18n.transliterate(filename) - filename = percent_escape(filename, TRADITIONAL_PARAMETER_ESCAPED_CHAR) - %(filename="#{filename}") - end - - RFC5987_PARAMETER_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ - - def encode_utf8_filename(filename) - # RFC2231 filename parameters can simply be percent-escaped according - # to RFC5987. - filename = percent_escape(filename, RFC5987_PARAMETER_ESCAPED_CHAR) - %(filename*=UTF-8''#{filename}) - end - - def percent_escape(string, pattern) - string.gsub(pattern) do |char| - char.bytes.map { |byte| "%%%02X" % byte }.join("") - end - end -end diff --git a/lib/active_vault/filename.rb b/lib/active_vault/filename.rb deleted file mode 100644 index 647d037b1f..0000000000 --- a/lib/active_vault/filename.rb +++ /dev/null @@ -1,31 +0,0 @@ -class ActiveVault::Filename - include Comparable - - def initialize(filename) - @filename = filename - end - - def extname - File.extname(@filename) - end - - def extension - extname.from(1) - end - - def base - File.basename(@filename, extname) - end - - def sanitized - @filename.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-") - end - - def to_s - sanitized.to_s - end - - def <=>(other) - to_s.downcase <=> other.to_s.downcase - end -end diff --git a/lib/active_vault/migration.rb b/lib/active_vault/migration.rb deleted file mode 100644 index 985d26d1b9..0000000000 --- a/lib/active_vault/migration.rb +++ /dev/null @@ -1,28 +0,0 @@ -class ActiveVault::CreateTables < ActiveRecord::Migration[5.1] - def change - create_table :active_vault_blobs do |t| - t.string :key - t.string :filename - t.string :content_type - t.text :metadata - t.integer :byte_size - t.string :checksum - t.time :created_at - - t.index [ :key ], unique: true - end - - create_table :active_vault_attachments do |t| - t.string :name - t.string :record_gid - t.integer :blob_id - - t.time :created_at - - t.index :record_gid - t.index :blob_id - t.index [ :record_gid, :name ] - t.index [ :record_gid, :blob_id ], unique: true - end - end -end diff --git a/lib/active_vault/purge_job.rb b/lib/active_vault/purge_job.rb deleted file mode 100644 index b68eb370bb..0000000000 --- a/lib/active_vault/purge_job.rb +++ /dev/null @@ -1,10 +0,0 @@ -require "active_job" - -class ActiveVault::PurgeJob < ActiveJob::Base - # FIXME: Limit this to a custom ActiveVault error - retry_on StandardError - - def perform(attachment_or_blob) - attachment_or_blob.purge - end -end diff --git a/lib/active_vault/railtie.rb b/lib/active_vault/railtie.rb deleted file mode 100644 index 1830780001..0000000000 --- a/lib/active_vault/railtie.rb +++ /dev/null @@ -1,27 +0,0 @@ -require "rails/railtie" - -module ActiveVault - class Railtie < Rails::Railtie # :nodoc: - config.active_vault = ActiveSupport::OrderedOptions.new - - config.eager_load_namespaces << ActiveVault - - initializer "active_vault.routes" do - require "active_vault/disk_controller" - - config.after_initialize do |app| - app.routes.prepend do - get "/rails/blobs/:encoded_key" => "active_vault/disk#show", as: :rails_disk_blob - end - end - end - - initializer "active_vault.attached" do - require "active_vault/attached" - - ActiveSupport.on_load(:active_record) do - extend ActiveVault::Attached::Macros - end - end - end -end diff --git a/lib/active_vault/site.rb b/lib/active_vault/site.rb deleted file mode 100644 index 29eddf1566..0000000000 --- a/lib/active_vault/site.rb +++ /dev/null @@ -1,41 +0,0 @@ -# Abstract class serving as an interface for concrete sites. -class ActiveVault::Site - def self.configure(site, **options) - begin - require "active_vault/site/#{site.to_s.downcase}_site" - ActiveVault::Site.const_get(:"#{site}Site").new(**options) - rescue LoadError => e - puts "Couldn't configure site: #{site} (#{e.message})" - end - end - - - def upload(key, io) - raise NotImplementedError - end - - def download(key) - raise NotImplementedError - end - - def delete(key) - raise NotImplementedError - end - - def exist?(key) - raise NotImplementedError - end - - - def url(key, expires_in:, disposition:, filename:) - raise NotImplementedError - end - - def bytesize(key) - raise NotImplementedError - end - - def checksum(key) - raise NotImplementedError - end -end diff --git a/lib/active_vault/site/disk_site.rb b/lib/active_vault/site/disk_site.rb deleted file mode 100644 index 73f86bac6a..0000000000 --- a/lib/active_vault/site/disk_site.rb +++ /dev/null @@ -1,71 +0,0 @@ -require "fileutils" -require "pathname" - -class ActiveVault::Site::DiskSite < ActiveVault::Site - attr_reader :root - - def initialize(root:) - @root = root - end - - def upload(key, io) - File.open(make_path_for(key), "wb") do |file| - while chunk = io.read(65536) - file.write(chunk) - end - end - end - - def download(key) - if block_given? - File.open(path_for(key)) do |file| - while data = file.read(65536) - yield data - end - end - else - File.open path_for(key), &:read - end - end - - def delete(key) - File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted - end - - def exist?(key) - File.exist? path_for(key) - end - - - def url(key, expires_in:, disposition:, filename:) - verified_key_with_expiration = ActiveVault::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) - - if defined?(Rails) && defined?(Rails.application) - Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) - else - "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" - end - end - - def byte_size(key) - File.size path_for(key) - end - - def checksum(key) - Digest::MD5.file(path_for(key)).hexdigest - end - - - private - def path_for(key) - File.join root, folder_for(key), key - end - - def folder_for(key) - [ key[0..1], key[2..3] ].join("/") - end - - def make_path_for(key) - path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } - end -end diff --git a/lib/active_vault/site/gcs_site.rb b/lib/active_vault/site/gcs_site.rb deleted file mode 100644 index 8f51d486ec..0000000000 --- a/lib/active_vault/site/gcs_site.rb +++ /dev/null @@ -1,53 +0,0 @@ -require "google/cloud/storage" -require "active_support/core_ext/object/to_query" - -class ActiveVault::Site::GCSSite < ActiveVault::Site - attr_reader :client, :bucket - - def initialize(project:, keyfile:, bucket:) - @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) - @bucket = @client.bucket(bucket) - end - - def upload(key, io) - bucket.create_file(io, key) - end - - def download(key) - io = file_for(key).download - io.rewind - io.read - end - - def delete(key) - file_for(key).try(:delete) - end - - def exist?(key) - file_for(key).present? - end - - - def url(key, expires_in:, disposition:, filename:) - file_for(key).signed_url(expires: expires_in) + "&" + - { "response-content-disposition" => "#{disposition}; filename=\"#{filename}\"" }.to_query - end - - def byte_size(key) - file_for(key).size - end - - def checksum(key) - convert_to_hex base64: file_for(key).md5 - end - - - private - def file_for(key) - bucket.file(key) - end - - def convert_to_hex(base64:) - base64.unpack("m0").first.unpack("H*").first - end -end diff --git a/lib/active_vault/site/mirror_site.rb b/lib/active_vault/site/mirror_site.rb deleted file mode 100644 index 8a2fa52fcb..0000000000 --- a/lib/active_vault/site/mirror_site.rb +++ /dev/null @@ -1,51 +0,0 @@ -class ActiveVault::Site::MirrorSite < ActiveVault::Site - attr_reader :sites - - def initialize(sites:) - @sites = sites - end - - def upload(key, io) - sites.collect do |site| - site.upload key, io - io.rewind - end - end - - def download(key) - sites.detect { |site| site.exist?(key) }.download(key) - end - - def delete(key) - perform_across_sites :delete, key - end - - def exist?(key) - perform_across_sites(:exist?, key).any? - end - - - def url(key, **options) - primary_site.url(key, **options) - end - - def byte_size(key) - primary_site.byte_size(key) - end - - def checksum(key) - primary_site.checksum(key) - end - - private - def primary_site - sites.first - end - - def perform_across_sites(method, *args) - # FIXME: Convert to be threaded - sites.collect do |site| - site.public_send method, *args - end - end -end diff --git a/lib/active_vault/site/s3_site.rb b/lib/active_vault/site/s3_site.rb deleted file mode 100644 index 49a7522170..0000000000 --- a/lib/active_vault/site/s3_site.rb +++ /dev/null @@ -1,63 +0,0 @@ -require "aws-sdk" - -class ActiveVault::Site::S3Site < ActiveVault::Site - attr_reader :client, :bucket - - def initialize(access_key_id:, secret_access_key:, region:, bucket:) - @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) - @bucket = @client.bucket(bucket) - end - - def upload(key, io) - object_for(key).put(body: io) - end - - def download(key) - if block_given? - stream(key, &block) - else - object_for(key).get.body.read - end - end - - def delete(key) - object_for(key).delete - end - - def exist?(key) - object_for(key).exists? - end - - - def url(key, expires_in:, disposition:, filename:) - object_for(key).presigned_url :get, expires_in: expires_in, - response_content_disposition: "#{disposition}; filename=\"#{filename}\"" - end - - def byte_size(key) - object_for(key).size - end - - def checksum(key) - object_for(key).etag.remove(/"/) - end - - - private - def object_for(key) - bucket.object(key) - end - - # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, options = {}, &block) - object = object_for(key) - - chunk_size = 5242880 # 5 megabytes - offset = 0 - - while offset < object.content_length - yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) - offset += chunk_size - end - end -end diff --git a/lib/active_vault/verified_key_with_expiration.rb b/lib/active_vault/verified_key_with_expiration.rb deleted file mode 100644 index 95d4993ff0..0000000000 --- a/lib/active_vault/verified_key_with_expiration.rb +++ /dev/null @@ -1,24 +0,0 @@ -class ActiveVault::VerifiedKeyWithExpiration - class_attribute :verifier, default: defined?(Rails) ? Rails.application.message_verifier('ActiveVault') : nil - - class << self - def encode(key, expires_in: nil) - verifier.generate([ key, expires_at(expires_in) ]) - end - - def decode(encoded_key) - key, expires_at = verifier.verified(encoded_key) - - key if key && fresh?(expires_at) - end - - private - def expires_at(expires_in) - expires_in ? Time.now.utc.advance(seconds: expires_in) : nil - end - - def fresh?(expires_at) - expires_at.nil? || Time.now.utc < expires_at - end - end -end -- cgit v1.2.3