From c624df326a4ef36919a5195a3c5509fab97dcba3 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Thu, 6 Jul 2017 11:33:29 +0200 Subject: ActiveVault -> ActiveStorage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Yaroslav agreed to hand over the gem name ❤️ --- Gemfile.lock | 4 +- README.md | 4 +- activestorage.gemspec | 21 +++++ activevault.gemspec | 21 ----- lib/active_storage.rb | 9 +++ lib/active_storage/attached.rb | 34 ++++++++ lib/active_storage/attached/macros.rb | 23 ++++++ lib/active_storage/attached/many.rb | 31 ++++++++ lib/active_storage/attached/one.rb | 29 +++++++ lib/active_storage/attachment.rb | 30 ++++++++ lib/active_storage/blob.rb | 68 ++++++++++++++++ lib/active_storage/config/sites.yml | 25 ++++++ lib/active_storage/disk_controller.rb | 28 +++++++ lib/active_storage/download.rb | 90 ++++++++++++++++++++++ lib/active_storage/filename.rb | 31 ++++++++ lib/active_storage/migration.rb | 28 +++++++ lib/active_storage/purge_job.rb | 10 +++ lib/active_storage/railtie.rb | 27 +++++++ lib/active_storage/site.rb | 41 ++++++++++ lib/active_storage/site/disk_site.rb | 71 +++++++++++++++++ lib/active_storage/site/gcs_site.rb | 53 +++++++++++++ lib/active_storage/site/mirror_site.rb | 51 ++++++++++++ lib/active_storage/site/s3_site.rb | 63 +++++++++++++++ lib/active_storage/verified_key_with_expiration.rb | 24 ++++++ lib/active_vault.rb | 9 --- lib/active_vault/attached.rb | 34 -------- lib/active_vault/attached/macros.rb | 23 ------ lib/active_vault/attached/many.rb | 31 -------- lib/active_vault/attached/one.rb | 29 ------- lib/active_vault/attachment.rb | 30 -------- lib/active_vault/blob.rb | 68 ---------------- lib/active_vault/config/sites.yml | 25 ------ lib/active_vault/disk_controller.rb | 28 ------- lib/active_vault/download.rb | 90 ---------------------- lib/active_vault/filename.rb | 31 -------- lib/active_vault/migration.rb | 28 ------- lib/active_vault/purge_job.rb | 10 --- lib/active_vault/railtie.rb | 27 ------- lib/active_vault/site.rb | 41 ---------- lib/active_vault/site/disk_site.rb | 71 ----------------- lib/active_vault/site/gcs_site.rb | 53 ------------- lib/active_vault/site/mirror_site.rb | 51 ------------ lib/active_vault/site/s3_site.rb | 63 --------------- lib/active_vault/verified_key_with_expiration.rb | 24 ------ test/attachments_test.rb | 24 +++--- test/blob_test.rb | 6 +- test/database/create_users_migration.rb | 2 +- test/database/setup.rb | 6 +- test/disk_controller_test.rb | 14 ++-- test/filename_test.rb | 12 +-- test/site/disk_site_test.rb | 6 +- test/site/gcs_site_test.rb | 6 +- test/site/mirror_site_test.rb | 19 ++--- test/site/s3_site_test.rb | 6 +- test/site/shared_site_tests.rb | 2 +- test/test_helper.rb | 18 ++--- test/verified_key_with_expiration_test.rb | 12 +-- 57 files changed, 854 insertions(+), 861 deletions(-) create mode 100644 activestorage.gemspec delete mode 100644 activevault.gemspec create mode 100644 lib/active_storage.rb create mode 100644 lib/active_storage/attached.rb create mode 100644 lib/active_storage/attached/macros.rb create mode 100644 lib/active_storage/attached/many.rb create mode 100644 lib/active_storage/attached/one.rb create mode 100644 lib/active_storage/attachment.rb create mode 100644 lib/active_storage/blob.rb create mode 100644 lib/active_storage/config/sites.yml create mode 100644 lib/active_storage/disk_controller.rb create mode 100644 lib/active_storage/download.rb create mode 100644 lib/active_storage/filename.rb create mode 100644 lib/active_storage/migration.rb create mode 100644 lib/active_storage/purge_job.rb create mode 100644 lib/active_storage/railtie.rb create mode 100644 lib/active_storage/site.rb create mode 100644 lib/active_storage/site/disk_site.rb create mode 100644 lib/active_storage/site/gcs_site.rb create mode 100644 lib/active_storage/site/mirror_site.rb create mode 100644 lib/active_storage/site/s3_site.rb create mode 100644 lib/active_storage/verified_key_with_expiration.rb delete mode 100644 lib/active_vault.rb delete mode 100644 lib/active_vault/attached.rb delete mode 100644 lib/active_vault/attached/macros.rb delete mode 100644 lib/active_vault/attached/many.rb delete mode 100644 lib/active_vault/attached/one.rb delete mode 100644 lib/active_vault/attachment.rb delete mode 100644 lib/active_vault/blob.rb delete mode 100644 lib/active_vault/config/sites.yml delete mode 100644 lib/active_vault/disk_controller.rb delete mode 100644 lib/active_vault/download.rb delete mode 100644 lib/active_vault/filename.rb delete mode 100644 lib/active_vault/migration.rb delete mode 100644 lib/active_vault/purge_job.rb delete mode 100644 lib/active_vault/railtie.rb delete mode 100644 lib/active_vault/site.rb delete mode 100644 lib/active_vault/site/disk_site.rb delete mode 100644 lib/active_vault/site/gcs_site.rb delete mode 100644 lib/active_vault/site/mirror_site.rb delete mode 100644 lib/active_vault/site/s3_site.rb delete mode 100644 lib/active_vault/verified_key_with_expiration.rb diff --git a/Gemfile.lock b/Gemfile.lock index e20ba22218..afef3518aa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - activevault (0.1) + activestorage (0.1) actionpack (>= 5.1) activejob (>= 5.1) activerecord (>= 5.1) @@ -223,7 +223,7 @@ PLATFORMS ruby DEPENDENCIES - activevault! + activestorage! aws-sdk bundler (~> 1.15) byebug diff --git a/README.md b/README.md index 97101c6099..8803cd9f3b 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ user.avatar.exist? # => true user.avatar.purge user.avatar.exist? # => false -user.image.url(expires_in: 5.minutes) # => /rails/blobs/ +user.avatar.url(expires_in: 5.minutes) # => /rails/blobs/ class AvatarsController < ApplicationController def update @@ -55,7 +55,7 @@ end ## Configuration -Add `require "active_vault"` to config/application.rb and create a `config/initializers/active_vault_sites.rb` with the following: +Add `require "active_storage"` to config/application.rb and create a `config/initializers/active_storage_sites.rb` with the following: ```ruby diff --git a/activestorage.gemspec b/activestorage.gemspec new file mode 100644 index 0000000000..4670bd1502 --- /dev/null +++ b/activestorage.gemspec @@ -0,0 +1,21 @@ +Gem::Specification.new do |s| + s.name = "activestorage" + s.version = "0.1" + s.authors = "David Heinemeier Hansson" + s.email = "david@basecamp.com" + s.summary = "Store files in Rails applications" + s.homepage = "https://github.com/rails/activestorage" + s.license = "MIT" + + s.required_ruby_version = ">= 1.9.3" + + s.add_dependency "activesupport", ">= 5.1" + s.add_dependency "activerecord", ">= 5.1" + s.add_dependency "actionpack", ">= 5.1" + s.add_dependency "activejob", ">= 5.1" + + s.add_development_dependency "bundler", "~> 1.15" + + s.files = `git ls-files`.split("\n") + s.test_files = `git ls-files -- test/*`.split("\n") +end diff --git a/activevault.gemspec b/activevault.gemspec deleted file mode 100644 index 7144563d18..0000000000 --- a/activevault.gemspec +++ /dev/null @@ -1,21 +0,0 @@ -Gem::Specification.new do |s| - s.name = "activevault" - s.version = "0.1" - s.authors = "David Heinemeier Hansson" - s.email = "david@basecamp.com" - s.summary = "Store files in Rails applications" - s.homepage = "https://github.com/rails/activevault" - s.license = "MIT" - - s.required_ruby_version = ">= 1.9.3" - - s.add_dependency "activesupport", ">= 5.1" - s.add_dependency "activerecord", ">= 5.1" - s.add_dependency "actionpack", ">= 5.1" - s.add_dependency "activejob", ">= 5.1" - - s.add_development_dependency "bundler", "~> 1.15" - - s.files = `git ls-files`.split("\n") - s.test_files = `git ls-files -- test/*`.split("\n") -end diff --git a/lib/active_storage.rb b/lib/active_storage.rb new file mode 100644 index 0000000000..e87eb8a506 --- /dev/null +++ b/lib/active_storage.rb @@ -0,0 +1,9 @@ +require "active_record" +require "active_storage/railtie" if defined?(Rails) + +module ActiveStorage + extend ActiveSupport::Autoload + + autoload :Blob + autoload :Site +end diff --git a/lib/active_storage/attached.rb b/lib/active_storage/attached.rb new file mode 100644 index 0000000000..7475c38999 --- /dev/null +++ b/lib/active_storage/attached.rb @@ -0,0 +1,34 @@ +require "active_storage/blob" +require "active_storage/attachment" + +require "action_dispatch/http/upload" +require "active_support/core_ext/module/delegation" + +class ActiveStorage::Attached + attr_reader :name, :record + + def initialize(name, record) + @name, @record = name, record + end + + private + def create_blob_from(attachable) + case attachable + when ActiveStorage::Blob + attachable + when ActionDispatch::Http::UploadedFile + ActiveStorage::Blob.create_after_upload! \ + io: attachable.open, + filename: attachable.original_filename, + content_type: attachable.content_type + when Hash + ActiveStorage::Blob.create_after_upload!(attachable) + else + nil + end + end +end + +require "active_storage/attached/one" +require "active_storage/attached/many" +require "active_storage/attached/macros" diff --git a/lib/active_storage/attached/macros.rb b/lib/active_storage/attached/macros.rb new file mode 100644 index 0000000000..96493d1215 --- /dev/null +++ b/lib/active_storage/attached/macros.rb @@ -0,0 +1,23 @@ +module ActiveStorage::Attached::Macros + def has_one_attached(name, dependent: :purge_later) + define_method(name) do + instance_variable_get("@active_storage_attached_#{name}") || + instance_variable_set("@active_storage_attached_#{name}", ActiveStorage::Attached::One.new(name, self)) + end + + if dependent == :purge_later + before_destroy { public_send(name).purge_later } + end + end + + def has_many_attached(name, dependent: :purge_later) + define_method(name) do + instance_variable_get("@active_storage_attached_#{name}") || + instance_variable_set("@active_storage_attached_#{name}", ActiveStorage::Attached::Many.new(name, self)) + end + + if dependent == :purge_later + before_destroy { public_send(name).purge_later } + end + end +end diff --git a/lib/active_storage/attached/many.rb b/lib/active_storage/attached/many.rb new file mode 100644 index 0000000000..f1535dfbc6 --- /dev/null +++ b/lib/active_storage/attached/many.rb @@ -0,0 +1,31 @@ +class ActiveStorage::Attached::Many < ActiveStorage::Attached + delegate_missing_to :attachments + + def attachments + @attachments ||= ActiveStorage::Attachment.where(record_gid: record.to_gid.to_s, name: name) + end + + def attach(*attachables) + @attachments = attachments | Array(attachables).flatten.collect do |attachable| + ActiveStorage::Attachment.create!(record_gid: record.to_gid.to_s, name: name, blob: create_blob_from(attachable)) + end + end + + def attached? + attachments.any? + end + + def purge + if attached? + attachments.each(&:purge) + @attachments = nil + end + end + + def purge_later + if attached? + attachments.each(&:purge_later) + @attachments = nil + end + end +end diff --git a/lib/active_storage/attached/one.rb b/lib/active_storage/attached/one.rb new file mode 100644 index 0000000000..d08d265992 --- /dev/null +++ b/lib/active_storage/attached/one.rb @@ -0,0 +1,29 @@ +class ActiveStorage::Attached::One < ActiveStorage::Attached + delegate_missing_to :attachment + + def attachment + @attachment ||= ActiveStorage::Attachment.find_by(record_gid: record.to_gid.to_s, name: name) + end + + def attach(attachable) + @attachment = ActiveStorage::Attachment.create!(record_gid: record.to_gid.to_s, name: name, blob: create_blob_from(attachable)) + end + + def attached? + attachment.present? + end + + def purge + if attached? + attachment.purge + @attachment = nil + end + end + + def purge_later + if attached? + attachment.purge_later + @attachment = nil + end + end +end diff --git a/lib/active_storage/attachment.rb b/lib/active_storage/attachment.rb new file mode 100644 index 0000000000..20c619aa5a --- /dev/null +++ b/lib/active_storage/attachment.rb @@ -0,0 +1,30 @@ +require "active_storage/blob" +require "global_id" +require "active_support/core_ext/module/delegation" + +# Schema: id, record_gid, blob_id, created_at +class ActiveStorage::Attachment < ActiveRecord::Base + self.table_name = "active_storage_attachments" + + belongs_to :blob, class_name: "ActiveStorage::Blob" + + delegate_missing_to :blob + + def record + @record ||= GlobalID::Locator.locate(record_gid) + end + + def record=(record) + @record = record + self.record_gid = record&.to_gid + end + + def purge + blob.purge + destroy + end + + def purge_later + ActiveStorage::PurgeJob.perform_later(self) + end +end diff --git a/lib/active_storage/blob.rb b/lib/active_storage/blob.rb new file mode 100644 index 0000000000..edf57b5c78 --- /dev/null +++ b/lib/active_storage/blob.rb @@ -0,0 +1,68 @@ +require "active_storage/site" +require "active_storage/filename" +require "active_storage/purge_job" + +# Schema: id, key, filename, content_type, metadata, byte_size, checksum, created_at +class ActiveStorage::Blob < ActiveRecord::Base + self.table_name = "active_storage_blobs" + + has_secure_token :key + store :metadata, coder: JSON + + class_attribute :site + + class << self + def build_after_upload(io:, filename:, content_type: nil, metadata: nil) + new.tap do |blob| + blob.filename = filename + blob.content_type = content_type + blob.metadata = metadata + + blob.upload io + end + end + + def create_after_upload!(io:, filename:, content_type: nil, metadata: nil) + build_after_upload(io: io, filename: filename, content_type: content_type, metadata: metadata).tap(&:save!) + end + end + + # We can't wait until the record is first saved to have a key for it + def key + self[:key] ||= self.class.generate_unique_secure_token + end + + def filename + ActiveStorage::Filename.new(self[:filename]) + end + + def url(expires_in: 5.minutes, disposition: :inline) + site.url key, expires_in: expires_in, disposition: disposition, filename: filename + end + + + def upload(io) + site.upload(key, io) + + self.checksum = site.checksum(key) + self.byte_size = site.byte_size(key) + end + + def download + site.download key + end + + + def delete + site.delete key + end + + def purge + delete + destroy + end + + def purge_later + ActiveStorage::PurgeJob.perform_later(self) + end +end diff --git a/lib/active_storage/config/sites.yml b/lib/active_storage/config/sites.yml new file mode 100644 index 0000000000..43bc77fbf9 --- /dev/null +++ b/lib/active_storage/config/sites.yml @@ -0,0 +1,25 @@ +# Configuration should be something like this: +# +# config/environments/development.rb +# config.active_storage.site = :local +# +# config/environments/production.rb +# config.active_storage.site = :amazon +local: + site: Disk + root: <%%= File.join(Dir.tmpdir, "active_storage") %> + +amazon: + site: S3 + access_key_id: <%%= Rails.application.secrets.aws[:access_key_id] %> + secret_access_key: <%%= Rails.application.secrets.aws[:secret_access_key] %> + region: us-east-1 + bucket: <%= Rails.application.class.name.remove(/::Application$/).underscore %> + +google: + site: GCS + +mirror: + site: Mirror + primary: amazon + secondaries: google diff --git a/lib/active_storage/disk_controller.rb b/lib/active_storage/disk_controller.rb new file mode 100644 index 0000000000..3eba86c213 --- /dev/null +++ b/lib/active_storage/disk_controller.rb @@ -0,0 +1,28 @@ +require "action_controller" +require "active_storage/blob" +require "active_storage/verified_key_with_expiration" + +require "active_support/core_ext/object/inclusion" + +class ActiveStorage::DiskController < ActionController::Base + def show + if key = decode_verified_key + blob = ActiveStorage::Blob.find_by!(key: key) + + if stale?(etag: blob.checksum) + send_data blob.download, filename: blob.filename, type: blob.content_type, disposition: disposition_param + end + else + head :not_found + end + end + + private + def decode_verified_key + ActiveStorage::VerifiedKeyWithExpiration.decode(params[:encoded_key]) + end + + def disposition_param + params[:disposition].presence_in(%w( inline attachment )) || 'inline' + end +end diff --git a/lib/active_storage/download.rb b/lib/active_storage/download.rb new file mode 100644 index 0000000000..4d656942d8 --- /dev/null +++ b/lib/active_storage/download.rb @@ -0,0 +1,90 @@ +class ActiveStorage::Download + # Sending .ai files as application/postscript to Safari opens them in a blank, grey screen. + # Downloading .ai as application/postscript files in Safari appends .ps to the extension. + # Sending HTML, SVG, XML and SWF files as binary closes XSS vulnerabilities. + # Sending JS files as binary avoids InvalidCrossOriginRequest without compromising security. + CONTENT_TYPES_TO_RENDER_AS_BINARY = %w( + text/html + text/javascript + image/svg+xml + application/postscript + application/x-shockwave-flash + text/xml + application/xml + application/xhtml+xml + ) + + BINARY_CONTENT_TYPE = 'application/octet-stream' + + def initialize(stored_file) + @stored_file = stored_file + end + + def headers(force_attachment: false) + { + x_accel_redirect: '/reproxy', + x_reproxy_url: reproxy_url, + content_type: content_type, + content_disposition: content_disposition(force_attachment), + x_frame_options: 'SAMEORIGIN' + } + end + + private + def reproxy_url + @stored_file.depot_location.paths.first + end + + def content_type + if @stored_file.content_type.in? CONTENT_TYPES_TO_RENDER_AS_BINARY + BINARY_CONTENT_TYPE + else + @stored_file.content_type + end + end + + def content_disposition(force_attachment = false) + if force_attachment || content_type == BINARY_CONTENT_TYPE + "attachment; #{escaped_filename}" + else + "inline; #{escaped_filename}" + end + end + + # RFC2231 encoding for UTF-8 filenames, with an ASCII fallback + # first for unsupported browsers (IE < 9, perhaps others?). + # http://greenbytes.de/tech/tc2231/#encoding-2231-fb + def escaped_filename + filename = @stored_file.filename.sanitized + ascii_filename = encode_ascii_filename(filename) + utf8_filename = encode_utf8_filename(filename) + "#{ascii_filename}; #{utf8_filename}" + end + + TRADITIONAL_PARAMETER_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ + + def encode_ascii_filename(filename) + # There is no reliable way to escape special or non-Latin characters + # in a traditionally quoted Content-Disposition filename parameter. + # Settle for transliterating to ASCII, then percent-escaping special + # characters, excluding spaces. + filename = I18n.transliterate(filename) + filename = percent_escape(filename, TRADITIONAL_PARAMETER_ESCAPED_CHAR) + %(filename="#{filename}") + end + + RFC5987_PARAMETER_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ + + def encode_utf8_filename(filename) + # RFC2231 filename parameters can simply be percent-escaped according + # to RFC5987. + filename = percent_escape(filename, RFC5987_PARAMETER_ESCAPED_CHAR) + %(filename*=UTF-8''#{filename}) + end + + def percent_escape(string, pattern) + string.gsub(pattern) do |char| + char.bytes.map { |byte| "%%%02X" % byte }.join("") + end + end +end diff --git a/lib/active_storage/filename.rb b/lib/active_storage/filename.rb new file mode 100644 index 0000000000..71614b5113 --- /dev/null +++ b/lib/active_storage/filename.rb @@ -0,0 +1,31 @@ +class ActiveStorage::Filename + include Comparable + + def initialize(filename) + @filename = filename + end + + def extname + File.extname(@filename) + end + + def extension + extname.from(1) + end + + def base + File.basename(@filename, extname) + end + + def sanitized + @filename.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-") + end + + def to_s + sanitized.to_s + end + + def <=>(other) + to_s.downcase <=> other.to_s.downcase + end +end diff --git a/lib/active_storage/migration.rb b/lib/active_storage/migration.rb new file mode 100644 index 0000000000..c0400abe3b --- /dev/null +++ b/lib/active_storage/migration.rb @@ -0,0 +1,28 @@ +class ActiveStorage::CreateTables < ActiveRecord::Migration[5.1] + def change + create_table :active_storage_blobs do |t| + t.string :key + t.string :filename + t.string :content_type + t.text :metadata + t.integer :byte_size + t.string :checksum + t.time :created_at + + t.index [ :key ], unique: true + end + + create_table :active_storage_attachments do |t| + t.string :name + t.string :record_gid + t.integer :blob_id + + t.time :created_at + + t.index :record_gid + t.index :blob_id + t.index [ :record_gid, :name ] + t.index [ :record_gid, :blob_id ], unique: true + end + end +end diff --git a/lib/active_storage/purge_job.rb b/lib/active_storage/purge_job.rb new file mode 100644 index 0000000000..b59d3687f8 --- /dev/null +++ b/lib/active_storage/purge_job.rb @@ -0,0 +1,10 @@ +require "active_job" + +class ActiveStorage::PurgeJob < ActiveJob::Base + # FIXME: Limit this to a custom ActiveStorage error + retry_on StandardError + + def perform(attachment_or_blob) + attachment_or_blob.purge + end +end diff --git a/lib/active_storage/railtie.rb b/lib/active_storage/railtie.rb new file mode 100644 index 0000000000..bf38d5aff5 --- /dev/null +++ b/lib/active_storage/railtie.rb @@ -0,0 +1,27 @@ +require "rails/railtie" + +module ActiveStorage + class Railtie < Rails::Railtie # :nodoc: + config.active_storage = ActiveSupport::OrderedOptions.new + + config.eager_load_namespaces << ActiveStorage + + initializer "active_storage.routes" do + require "active_storage/disk_controller" + + config.after_initialize do |app| + app.routes.prepend do + get "/rails/blobs/:encoded_key" => "active_storage/disk#show", as: :rails_disk_blob + end + end + end + + initializer "active_storage.attached" do + require "active_storage/attached" + + ActiveSupport.on_load(:active_record) do + extend ActiveStorage::Attached::Macros + end + end + end +end diff --git a/lib/active_storage/site.rb b/lib/active_storage/site.rb new file mode 100644 index 0000000000..b3b0221c63 --- /dev/null +++ b/lib/active_storage/site.rb @@ -0,0 +1,41 @@ +# Abstract class serving as an interface for concrete sites. +class ActiveStorage::Site + def self.configure(site, **options) + begin + require "active_storage/site/#{site.to_s.downcase}_site" + ActiveStorage::Site.const_get(:"#{site}Site").new(**options) + rescue LoadError => e + puts "Couldn't configure site: #{site} (#{e.message})" + end + end + + + def upload(key, io) + raise NotImplementedError + end + + def download(key) + raise NotImplementedError + end + + def delete(key) + raise NotImplementedError + end + + def exist?(key) + raise NotImplementedError + end + + + def url(key, expires_in:, disposition:, filename:) + raise NotImplementedError + end + + def bytesize(key) + raise NotImplementedError + end + + def checksum(key) + raise NotImplementedError + end +end diff --git a/lib/active_storage/site/disk_site.rb b/lib/active_storage/site/disk_site.rb new file mode 100644 index 0000000000..2ff0b22fae --- /dev/null +++ b/lib/active_storage/site/disk_site.rb @@ -0,0 +1,71 @@ +require "fileutils" +require "pathname" + +class ActiveStorage::Site::DiskSite < ActiveStorage::Site + attr_reader :root + + def initialize(root:) + @root = root + end + + def upload(key, io) + File.open(make_path_for(key), "wb") do |file| + while chunk = io.read(65536) + file.write(chunk) + end + end + end + + def download(key) + if block_given? + File.open(path_for(key)) do |file| + while data = file.read(65536) + yield data + end + end + else + File.open path_for(key), &:read + end + end + + def delete(key) + File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted + end + + def exist?(key) + File.exist? path_for(key) + end + + + def url(key, expires_in:, disposition:, filename:) + verified_key_with_expiration = ActiveStorage::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) + + if defined?(Rails) && defined?(Rails.application) + Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) + else + "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" + end + end + + def byte_size(key) + File.size path_for(key) + end + + def checksum(key) + Digest::MD5.file(path_for(key)).hexdigest + end + + + private + def path_for(key) + File.join root, folder_for(key), key + end + + def folder_for(key) + [ key[0..1], key[2..3] ].join("/") + end + + def make_path_for(key) + path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } + end +end diff --git a/lib/active_storage/site/gcs_site.rb b/lib/active_storage/site/gcs_site.rb new file mode 100644 index 0000000000..bf681ca6a2 --- /dev/null +++ b/lib/active_storage/site/gcs_site.rb @@ -0,0 +1,53 @@ +require "google/cloud/storage" +require "active_support/core_ext/object/to_query" + +class ActiveStorage::Site::GCSSite < ActiveStorage::Site + attr_reader :client, :bucket + + def initialize(project:, keyfile:, bucket:) + @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + bucket.create_file(io, key) + end + + def download(key) + io = file_for(key).download + io.rewind + io.read + end + + def delete(key) + file_for(key).try(:delete) + end + + def exist?(key) + file_for(key).present? + end + + + def url(key, expires_in:, disposition:, filename:) + file_for(key).signed_url(expires: expires_in) + "&" + + { "response-content-disposition" => "#{disposition}; filename=\"#{filename}\"" }.to_query + end + + def byte_size(key) + file_for(key).size + end + + def checksum(key) + convert_to_hex base64: file_for(key).md5 + end + + + private + def file_for(key) + bucket.file(key) + end + + def convert_to_hex(base64:) + base64.unpack("m0").first.unpack("H*").first + end +end diff --git a/lib/active_storage/site/mirror_site.rb b/lib/active_storage/site/mirror_site.rb new file mode 100644 index 0000000000..ba3ef0ef0e --- /dev/null +++ b/lib/active_storage/site/mirror_site.rb @@ -0,0 +1,51 @@ +class ActiveStorage::Site::MirrorSite < ActiveStorage::Site + attr_reader :sites + + def initialize(sites:) + @sites = sites + end + + def upload(key, io) + sites.collect do |site| + site.upload key, io + io.rewind + end + end + + def download(key) + sites.detect { |site| site.exist?(key) }.download(key) + end + + def delete(key) + perform_across_sites :delete, key + end + + def exist?(key) + perform_across_sites(:exist?, key).any? + end + + + def url(key, **options) + primary_site.url(key, **options) + end + + def byte_size(key) + primary_site.byte_size(key) + end + + def checksum(key) + primary_site.checksum(key) + end + + private + def primary_site + sites.first + end + + def perform_across_sites(method, *args) + # FIXME: Convert to be threaded + sites.collect do |site| + site.public_send method, *args + end + end +end diff --git a/lib/active_storage/site/s3_site.rb b/lib/active_storage/site/s3_site.rb new file mode 100644 index 0000000000..65dad37cfe --- /dev/null +++ b/lib/active_storage/site/s3_site.rb @@ -0,0 +1,63 @@ +require "aws-sdk" + +class ActiveStorage::Site::S3Site < ActiveStorage::Site + attr_reader :client, :bucket + + def initialize(access_key_id:, secret_access_key:, region:, bucket:) + @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + object_for(key).put(body: io) + end + + def download(key) + if block_given? + stream(key, &block) + else + object_for(key).get.body.read + end + end + + def delete(key) + object_for(key).delete + end + + def exist?(key) + object_for(key).exists? + end + + + def url(key, expires_in:, disposition:, filename:) + object_for(key).presigned_url :get, expires_in: expires_in, + response_content_disposition: "#{disposition}; filename=\"#{filename}\"" + end + + def byte_size(key) + object_for(key).size + end + + def checksum(key) + object_for(key).etag.remove(/"/) + end + + + private + def object_for(key) + bucket.object(key) + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key, options = {}, &block) + object = object_for(key) + + chunk_size = 5242880 # 5 megabytes + offset = 0 + + while offset < object.content_length + yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) + offset += chunk_size + end + end +end diff --git a/lib/active_storage/verified_key_with_expiration.rb b/lib/active_storage/verified_key_with_expiration.rb new file mode 100644 index 0000000000..8708106735 --- /dev/null +++ b/lib/active_storage/verified_key_with_expiration.rb @@ -0,0 +1,24 @@ +class ActiveStorage::VerifiedKeyWithExpiration + class_attribute :verifier, default: defined?(Rails) ? Rails.application.message_verifier('ActiveStorage') : nil + + class << self + def encode(key, expires_in: nil) + verifier.generate([ key, expires_at(expires_in) ]) + end + + def decode(encoded_key) + key, expires_at = verifier.verified(encoded_key) + + key if key && fresh?(expires_at) + end + + private + def expires_at(expires_in) + expires_in ? Time.now.utc.advance(seconds: expires_in) : nil + end + + def fresh?(expires_at) + expires_at.nil? || Time.now.utc < expires_at + end + end +end diff --git a/lib/active_vault.rb b/lib/active_vault.rb deleted file mode 100644 index f47b09b4cd..0000000000 --- a/lib/active_vault.rb +++ /dev/null @@ -1,9 +0,0 @@ -require "active_record" -require "active_vault/railtie" if defined?(Rails) - -module ActiveVault - extend ActiveSupport::Autoload - - autoload :Blob - autoload :Site -end diff --git a/lib/active_vault/attached.rb b/lib/active_vault/attached.rb deleted file mode 100644 index a968f3500d..0000000000 --- a/lib/active_vault/attached.rb +++ /dev/null @@ -1,34 +0,0 @@ -require "active_vault/blob" -require "active_vault/attachment" - -require "action_dispatch/http/upload" -require "active_support/core_ext/module/delegation" - -class ActiveVault::Attached - attr_reader :name, :record - - def initialize(name, record) - @name, @record = name, record - end - - private - def create_blob_from(attachable) - case attachable - when ActiveVault::Blob - attachable - when ActionDispatch::Http::UploadedFile - ActiveVault::Blob.create_after_upload! \ - io: attachable.open, - filename: attachable.original_filename, - content_type: attachable.content_type - when Hash - ActiveVault::Blob.create_after_upload!(attachable) - else - nil - end - end -end - -require "active_vault/attached/one" -require "active_vault/attached/many" -require "active_vault/attached/macros" diff --git a/lib/active_vault/attached/macros.rb b/lib/active_vault/attached/macros.rb deleted file mode 100644 index 1b95c14c9c..0000000000 --- a/lib/active_vault/attached/macros.rb +++ /dev/null @@ -1,23 +0,0 @@ -module ActiveVault::Attached::Macros - def has_one_attached(name, dependent: :purge_later) - define_method(name) do - instance_variable_get("@active_vault_attached_#{name}") || - instance_variable_set("@active_vault_attached_#{name}", ActiveVault::Attached::One.new(name, self)) - end - - if dependent == :purge_later - before_destroy { public_send(name).purge_later } - end - end - - def has_many_attached(name, dependent: :purge_later) - define_method(name) do - instance_variable_get("@active_vault_attached_#{name}") || - instance_variable_set("@active_vault_attached_#{name}", ActiveVault::Attached::Many.new(name, self)) - end - - if dependent == :purge_later - before_destroy { public_send(name).purge_later } - end - end -end diff --git a/lib/active_vault/attached/many.rb b/lib/active_vault/attached/many.rb deleted file mode 100644 index 6f79a1c555..0000000000 --- a/lib/active_vault/attached/many.rb +++ /dev/null @@ -1,31 +0,0 @@ -class ActiveVault::Attached::Many < ActiveVault::Attached - delegate_missing_to :attachments - - def attachments - @attachments ||= ActiveVault::Attachment.where(record_gid: record.to_gid.to_s, name: name) - end - - def attach(*attachables) - @attachments = attachments | Array(attachables).flatten.collect do |attachable| - ActiveVault::Attachment.create!(record_gid: record.to_gid.to_s, name: name, blob: create_blob_from(attachable)) - end - end - - def attached? - attachments.any? - end - - def purge - if attached? - attachments.each(&:purge) - @attachments = nil - end - end - - def purge_later - if attached? - attachments.each(&:purge_later) - @attachments = nil - end - end -end diff --git a/lib/active_vault/attached/one.rb b/lib/active_vault/attached/one.rb deleted file mode 100644 index 01a5d0d6f0..0000000000 --- a/lib/active_vault/attached/one.rb +++ /dev/null @@ -1,29 +0,0 @@ -class ActiveVault::Attached::One < ActiveVault::Attached - delegate_missing_to :attachment - - def attachment - @attachment ||= ActiveVault::Attachment.find_by(record_gid: record.to_gid.to_s, name: name) - end - - def attach(attachable) - @attachment = ActiveVault::Attachment.create!(record_gid: record.to_gid.to_s, name: name, blob: create_blob_from(attachable)) - end - - def attached? - attachment.present? - end - - def purge - if attached? - attachment.purge - @attachment = nil - end - end - - def purge_later - if attached? - attachment.purge_later - @attachment = nil - end - end -end diff --git a/lib/active_vault/attachment.rb b/lib/active_vault/attachment.rb deleted file mode 100644 index 549a734d68..0000000000 --- a/lib/active_vault/attachment.rb +++ /dev/null @@ -1,30 +0,0 @@ -require "active_vault/blob" -require "global_id" -require "active_support/core_ext/module/delegation" - -# Schema: id, record_gid, blob_id, created_at -class ActiveVault::Attachment < ActiveRecord::Base - self.table_name = "active_vault_attachments" - - belongs_to :blob, class_name: "ActiveVault::Blob" - - delegate_missing_to :blob - - def record - @record ||= GlobalID::Locator.locate(record_gid) - end - - def record=(record) - @record = record - self.record_gid = record&.to_gid - end - - def purge - blob.purge - destroy - end - - def purge_later - ActiveVault::PurgeJob.perform_later(self) - end -end diff --git a/lib/active_vault/blob.rb b/lib/active_vault/blob.rb deleted file mode 100644 index a232ca5c1a..0000000000 --- a/lib/active_vault/blob.rb +++ /dev/null @@ -1,68 +0,0 @@ -require "active_vault/site" -require "active_vault/filename" -require "active_vault/purge_job" - -# Schema: id, key, filename, content_type, metadata, byte_size, checksum, created_at -class ActiveVault::Blob < ActiveRecord::Base - self.table_name = "active_vault_blobs" - - has_secure_token :key - store :metadata, coder: JSON - - class_attribute :site - - class << self - def build_after_upload(io:, filename:, content_type: nil, metadata: nil) - new.tap do |blob| - blob.filename = filename - blob.content_type = content_type - blob.metadata = metadata - - blob.upload io - end - end - - def create_after_upload!(io:, filename:, content_type: nil, metadata: nil) - build_after_upload(io: io, filename: filename, content_type: content_type, metadata: metadata).tap(&:save!) - end - end - - # We can't wait until the record is first saved to have a key for it - def key - self[:key] ||= self.class.generate_unique_secure_token - end - - def filename - ActiveVault::Filename.new(self[:filename]) - end - - def url(expires_in: 5.minutes, disposition: :inline) - site.url key, expires_in: expires_in, disposition: disposition, filename: filename - end - - - def upload(io) - site.upload(key, io) - - self.checksum = site.checksum(key) - self.byte_size = site.byte_size(key) - end - - def download - site.download key - end - - - def delete - site.delete key - end - - def purge - delete - destroy - end - - def purge_later - ActiveVault::PurgeJob.perform_later(self) - end -end diff --git a/lib/active_vault/config/sites.yml b/lib/active_vault/config/sites.yml deleted file mode 100644 index 334e779b28..0000000000 --- a/lib/active_vault/config/sites.yml +++ /dev/null @@ -1,25 +0,0 @@ -# Configuration should be something like this: -# -# config/environments/development.rb -# config.active_vault.site = :local -# -# config/environments/production.rb -# config.active_vault.site = :amazon -local: - site: Disk - root: <%%= File.join(Dir.tmpdir, "active_vault") %> - -amazon: - site: S3 - access_key_id: <%%= Rails.application.secrets.aws[:access_key_id] %> - secret_access_key: <%%= Rails.application.secrets.aws[:secret_access_key] %> - region: us-east-1 - bucket: <%= Rails.application.class.name.remove(/::Application$/).underscore %> - -google: - site: GCS - -mirror: - site: Mirror - primary: amazon - secondaries: google diff --git a/lib/active_vault/disk_controller.rb b/lib/active_vault/disk_controller.rb deleted file mode 100644 index 623569f0f6..0000000000 --- a/lib/active_vault/disk_controller.rb +++ /dev/null @@ -1,28 +0,0 @@ -require "action_controller" -require "active_vault/blob" -require "active_vault/verified_key_with_expiration" - -require "active_support/core_ext/object/inclusion" - -class ActiveVault::DiskController < ActionController::Base - def show - if key = decode_verified_key - blob = ActiveVault::Blob.find_by!(key: key) - - if stale?(etag: blob.checksum) - send_data blob.download, filename: blob.filename, type: blob.content_type, disposition: disposition_param - end - else - head :not_found - end - end - - private - def decode_verified_key - ActiveVault::VerifiedKeyWithExpiration.decode(params[:encoded_key]) - end - - def disposition_param - params[:disposition].presence_in(%w( inline attachment )) || 'inline' - end -end diff --git a/lib/active_vault/download.rb b/lib/active_vault/download.rb deleted file mode 100644 index 6e74056062..0000000000 --- a/lib/active_vault/download.rb +++ /dev/null @@ -1,90 +0,0 @@ -class ActiveVault::Download - # Sending .ai files as application/postscript to Safari opens them in a blank, grey screen. - # Downloading .ai as application/postscript files in Safari appends .ps to the extension. - # Sending HTML, SVG, XML and SWF files as binary closes XSS vulnerabilities. - # Sending JS files as binary avoids InvalidCrossOriginRequest without compromising security. - CONTENT_TYPES_TO_RENDER_AS_BINARY = %w( - text/html - text/javascript - image/svg+xml - application/postscript - application/x-shockwave-flash - text/xml - application/xml - application/xhtml+xml - ) - - BINARY_CONTENT_TYPE = 'application/octet-stream' - - def initialize(stored_file) - @stored_file = stored_file - end - - def headers(force_attachment: false) - { - x_accel_redirect: '/reproxy', - x_reproxy_url: reproxy_url, - content_type: content_type, - content_disposition: content_disposition(force_attachment), - x_frame_options: 'SAMEORIGIN' - } - end - - private - def reproxy_url - @stored_file.depot_location.paths.first - end - - def content_type - if @stored_file.content_type.in? CONTENT_TYPES_TO_RENDER_AS_BINARY - BINARY_CONTENT_TYPE - else - @stored_file.content_type - end - end - - def content_disposition(force_attachment = false) - if force_attachment || content_type == BINARY_CONTENT_TYPE - "attachment; #{escaped_filename}" - else - "inline; #{escaped_filename}" - end - end - - # RFC2231 encoding for UTF-8 filenames, with an ASCII fallback - # first for unsupported browsers (IE < 9, perhaps others?). - # http://greenbytes.de/tech/tc2231/#encoding-2231-fb - def escaped_filename - filename = @stored_file.filename.sanitized - ascii_filename = encode_ascii_filename(filename) - utf8_filename = encode_utf8_filename(filename) - "#{ascii_filename}; #{utf8_filename}" - end - - TRADITIONAL_PARAMETER_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ - - def encode_ascii_filename(filename) - # There is no reliable way to escape special or non-Latin characters - # in a traditionally quoted Content-Disposition filename parameter. - # Settle for transliterating to ASCII, then percent-escaping special - # characters, excluding spaces. - filename = I18n.transliterate(filename) - filename = percent_escape(filename, TRADITIONAL_PARAMETER_ESCAPED_CHAR) - %(filename="#{filename}") - end - - RFC5987_PARAMETER_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ - - def encode_utf8_filename(filename) - # RFC2231 filename parameters can simply be percent-escaped according - # to RFC5987. - filename = percent_escape(filename, RFC5987_PARAMETER_ESCAPED_CHAR) - %(filename*=UTF-8''#{filename}) - end - - def percent_escape(string, pattern) - string.gsub(pattern) do |char| - char.bytes.map { |byte| "%%%02X" % byte }.join("") - end - end -end diff --git a/lib/active_vault/filename.rb b/lib/active_vault/filename.rb deleted file mode 100644 index 647d037b1f..0000000000 --- a/lib/active_vault/filename.rb +++ /dev/null @@ -1,31 +0,0 @@ -class ActiveVault::Filename - include Comparable - - def initialize(filename) - @filename = filename - end - - def extname - File.extname(@filename) - end - - def extension - extname.from(1) - end - - def base - File.basename(@filename, extname) - end - - def sanitized - @filename.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-") - end - - def to_s - sanitized.to_s - end - - def <=>(other) - to_s.downcase <=> other.to_s.downcase - end -end diff --git a/lib/active_vault/migration.rb b/lib/active_vault/migration.rb deleted file mode 100644 index 985d26d1b9..0000000000 --- a/lib/active_vault/migration.rb +++ /dev/null @@ -1,28 +0,0 @@ -class ActiveVault::CreateTables < ActiveRecord::Migration[5.1] - def change - create_table :active_vault_blobs do |t| - t.string :key - t.string :filename - t.string :content_type - t.text :metadata - t.integer :byte_size - t.string :checksum - t.time :created_at - - t.index [ :key ], unique: true - end - - create_table :active_vault_attachments do |t| - t.string :name - t.string :record_gid - t.integer :blob_id - - t.time :created_at - - t.index :record_gid - t.index :blob_id - t.index [ :record_gid, :name ] - t.index [ :record_gid, :blob_id ], unique: true - end - end -end diff --git a/lib/active_vault/purge_job.rb b/lib/active_vault/purge_job.rb deleted file mode 100644 index b68eb370bb..0000000000 --- a/lib/active_vault/purge_job.rb +++ /dev/null @@ -1,10 +0,0 @@ -require "active_job" - -class ActiveVault::PurgeJob < ActiveJob::Base - # FIXME: Limit this to a custom ActiveVault error - retry_on StandardError - - def perform(attachment_or_blob) - attachment_or_blob.purge - end -end diff --git a/lib/active_vault/railtie.rb b/lib/active_vault/railtie.rb deleted file mode 100644 index 1830780001..0000000000 --- a/lib/active_vault/railtie.rb +++ /dev/null @@ -1,27 +0,0 @@ -require "rails/railtie" - -module ActiveVault - class Railtie < Rails::Railtie # :nodoc: - config.active_vault = ActiveSupport::OrderedOptions.new - - config.eager_load_namespaces << ActiveVault - - initializer "active_vault.routes" do - require "active_vault/disk_controller" - - config.after_initialize do |app| - app.routes.prepend do - get "/rails/blobs/:encoded_key" => "active_vault/disk#show", as: :rails_disk_blob - end - end - end - - initializer "active_vault.attached" do - require "active_vault/attached" - - ActiveSupport.on_load(:active_record) do - extend ActiveVault::Attached::Macros - end - end - end -end diff --git a/lib/active_vault/site.rb b/lib/active_vault/site.rb deleted file mode 100644 index 29eddf1566..0000000000 --- a/lib/active_vault/site.rb +++ /dev/null @@ -1,41 +0,0 @@ -# Abstract class serving as an interface for concrete sites. -class ActiveVault::Site - def self.configure(site, **options) - begin - require "active_vault/site/#{site.to_s.downcase}_site" - ActiveVault::Site.const_get(:"#{site}Site").new(**options) - rescue LoadError => e - puts "Couldn't configure site: #{site} (#{e.message})" - end - end - - - def upload(key, io) - raise NotImplementedError - end - - def download(key) - raise NotImplementedError - end - - def delete(key) - raise NotImplementedError - end - - def exist?(key) - raise NotImplementedError - end - - - def url(key, expires_in:, disposition:, filename:) - raise NotImplementedError - end - - def bytesize(key) - raise NotImplementedError - end - - def checksum(key) - raise NotImplementedError - end -end diff --git a/lib/active_vault/site/disk_site.rb b/lib/active_vault/site/disk_site.rb deleted file mode 100644 index 73f86bac6a..0000000000 --- a/lib/active_vault/site/disk_site.rb +++ /dev/null @@ -1,71 +0,0 @@ -require "fileutils" -require "pathname" - -class ActiveVault::Site::DiskSite < ActiveVault::Site - attr_reader :root - - def initialize(root:) - @root = root - end - - def upload(key, io) - File.open(make_path_for(key), "wb") do |file| - while chunk = io.read(65536) - file.write(chunk) - end - end - end - - def download(key) - if block_given? - File.open(path_for(key)) do |file| - while data = file.read(65536) - yield data - end - end - else - File.open path_for(key), &:read - end - end - - def delete(key) - File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted - end - - def exist?(key) - File.exist? path_for(key) - end - - - def url(key, expires_in:, disposition:, filename:) - verified_key_with_expiration = ActiveVault::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) - - if defined?(Rails) && defined?(Rails.application) - Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) - else - "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" - end - end - - def byte_size(key) - File.size path_for(key) - end - - def checksum(key) - Digest::MD5.file(path_for(key)).hexdigest - end - - - private - def path_for(key) - File.join root, folder_for(key), key - end - - def folder_for(key) - [ key[0..1], key[2..3] ].join("/") - end - - def make_path_for(key) - path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } - end -end diff --git a/lib/active_vault/site/gcs_site.rb b/lib/active_vault/site/gcs_site.rb deleted file mode 100644 index 8f51d486ec..0000000000 --- a/lib/active_vault/site/gcs_site.rb +++ /dev/null @@ -1,53 +0,0 @@ -require "google/cloud/storage" -require "active_support/core_ext/object/to_query" - -class ActiveVault::Site::GCSSite < ActiveVault::Site - attr_reader :client, :bucket - - def initialize(project:, keyfile:, bucket:) - @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) - @bucket = @client.bucket(bucket) - end - - def upload(key, io) - bucket.create_file(io, key) - end - - def download(key) - io = file_for(key).download - io.rewind - io.read - end - - def delete(key) - file_for(key).try(:delete) - end - - def exist?(key) - file_for(key).present? - end - - - def url(key, expires_in:, disposition:, filename:) - file_for(key).signed_url(expires: expires_in) + "&" + - { "response-content-disposition" => "#{disposition}; filename=\"#{filename}\"" }.to_query - end - - def byte_size(key) - file_for(key).size - end - - def checksum(key) - convert_to_hex base64: file_for(key).md5 - end - - - private - def file_for(key) - bucket.file(key) - end - - def convert_to_hex(base64:) - base64.unpack("m0").first.unpack("H*").first - end -end diff --git a/lib/active_vault/site/mirror_site.rb b/lib/active_vault/site/mirror_site.rb deleted file mode 100644 index 8a2fa52fcb..0000000000 --- a/lib/active_vault/site/mirror_site.rb +++ /dev/null @@ -1,51 +0,0 @@ -class ActiveVault::Site::MirrorSite < ActiveVault::Site - attr_reader :sites - - def initialize(sites:) - @sites = sites - end - - def upload(key, io) - sites.collect do |site| - site.upload key, io - io.rewind - end - end - - def download(key) - sites.detect { |site| site.exist?(key) }.download(key) - end - - def delete(key) - perform_across_sites :delete, key - end - - def exist?(key) - perform_across_sites(:exist?, key).any? - end - - - def url(key, **options) - primary_site.url(key, **options) - end - - def byte_size(key) - primary_site.byte_size(key) - end - - def checksum(key) - primary_site.checksum(key) - end - - private - def primary_site - sites.first - end - - def perform_across_sites(method, *args) - # FIXME: Convert to be threaded - sites.collect do |site| - site.public_send method, *args - end - end -end diff --git a/lib/active_vault/site/s3_site.rb b/lib/active_vault/site/s3_site.rb deleted file mode 100644 index 49a7522170..0000000000 --- a/lib/active_vault/site/s3_site.rb +++ /dev/null @@ -1,63 +0,0 @@ -require "aws-sdk" - -class ActiveVault::Site::S3Site < ActiveVault::Site - attr_reader :client, :bucket - - def initialize(access_key_id:, secret_access_key:, region:, bucket:) - @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) - @bucket = @client.bucket(bucket) - end - - def upload(key, io) - object_for(key).put(body: io) - end - - def download(key) - if block_given? - stream(key, &block) - else - object_for(key).get.body.read - end - end - - def delete(key) - object_for(key).delete - end - - def exist?(key) - object_for(key).exists? - end - - - def url(key, expires_in:, disposition:, filename:) - object_for(key).presigned_url :get, expires_in: expires_in, - response_content_disposition: "#{disposition}; filename=\"#{filename}\"" - end - - def byte_size(key) - object_for(key).size - end - - def checksum(key) - object_for(key).etag.remove(/"/) - end - - - private - def object_for(key) - bucket.object(key) - end - - # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, options = {}, &block) - object = object_for(key) - - chunk_size = 5242880 # 5 megabytes - offset = 0 - - while offset < object.content_length - yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) - offset += chunk_size - end - end -end diff --git a/lib/active_vault/verified_key_with_expiration.rb b/lib/active_vault/verified_key_with_expiration.rb deleted file mode 100644 index 95d4993ff0..0000000000 --- a/lib/active_vault/verified_key_with_expiration.rb +++ /dev/null @@ -1,24 +0,0 @@ -class ActiveVault::VerifiedKeyWithExpiration - class_attribute :verifier, default: defined?(Rails) ? Rails.application.message_verifier('ActiveVault') : nil - - class << self - def encode(key, expires_in: nil) - verifier.generate([ key, expires_at(expires_in) ]) - end - - def decode(encoded_key) - key, expires_at = verifier.verified(encoded_key) - - key if key && fresh?(expires_at) - end - - private - def expires_at(expires_in) - expires_in ? Time.now.utc.advance(seconds: expires_in) : nil - end - - def fresh?(expires_at) - expires_at.nil? || Time.now.utc < expires_at - end - end -end diff --git a/test/attachments_test.rb b/test/attachments_test.rb index 1b784b50c1..6e25002bb1 100644 --- a/test/attachments_test.rb +++ b/test/attachments_test.rb @@ -1,6 +1,6 @@ require "test_helper" require "database/setup" -require "active_vault/blob" +require "active_storage/blob" require "active_job" ActiveJob::Base.queue_adapter = :test @@ -13,12 +13,12 @@ class User < ActiveRecord::Base has_many_attached :highlights end -class ActiveVault::AttachmentsTest < ActiveSupport::TestCase +class ActiveStorage::AttachmentsTest < ActiveSupport::TestCase include ActiveJob::TestHelper setup { @user = User.create!(name: "DHH") } - teardown { ActiveVault::Blob.all.each(&:purge) } + teardown { ActiveStorage::Blob.all.each(&:purge) } test "attach existing blob" do @user.avatar.attach create_blob(filename: "funky.jpg") @@ -36,7 +36,7 @@ class ActiveVault::AttachmentsTest < ActiveSupport::TestCase @user.avatar.purge assert_not @user.avatar.attached? - assert_not ActiveVault::Blob.site.exist?(avatar_key) + assert_not ActiveStorage::Blob.site.exist?(avatar_key) end test "purge attached blob later when the record is destroyed" do @@ -46,8 +46,8 @@ class ActiveVault::AttachmentsTest < ActiveSupport::TestCase perform_enqueued_jobs do @user.destroy - assert_nil ActiveVault::Blob.find_by(key: avatar_key) - assert_not ActiveVault::Blob.site.exist?(avatar_key) + assert_nil ActiveStorage::Blob.find_by(key: avatar_key) + assert_not ActiveStorage::Blob.site.exist?(avatar_key) end end @@ -74,8 +74,8 @@ class ActiveVault::AttachmentsTest < ActiveSupport::TestCase @user.highlights.purge assert_not @user.highlights.attached? - assert_not ActiveVault::Blob.site.exist?(highlight_keys.first) - assert_not ActiveVault::Blob.site.exist?(highlight_keys.second) + assert_not ActiveStorage::Blob.site.exist?(highlight_keys.first) + assert_not ActiveStorage::Blob.site.exist?(highlight_keys.second) end test "purge attached blobs later when the record is destroyed" do @@ -85,11 +85,11 @@ class ActiveVault::AttachmentsTest < ActiveSupport::TestCase perform_enqueued_jobs do @user.destroy - assert_nil ActiveVault::Blob.find_by(key: highlight_keys.first) - assert_not ActiveVault::Blob.site.exist?(highlight_keys.first) + assert_nil ActiveStorage::Blob.find_by(key: highlight_keys.first) + assert_not ActiveStorage::Blob.site.exist?(highlight_keys.first) - assert_nil ActiveVault::Blob.find_by(key: highlight_keys.second) - assert_not ActiveVault::Blob.site.exist?(highlight_keys.second) + assert_nil ActiveStorage::Blob.find_by(key: highlight_keys.second) + assert_not ActiveStorage::Blob.site.exist?(highlight_keys.second) end end end diff --git a/test/blob_test.rb b/test/blob_test.rb index c7b4aeed39..880c656ecc 100644 --- a/test/blob_test.rb +++ b/test/blob_test.rb @@ -1,8 +1,8 @@ require "test_helper" require "database/setup" -require "active_vault/blob" +require "active_storage/blob" -class ActiveVault::BlobTest < ActiveSupport::TestCase +class ActiveStorage::BlobTest < ActiveSupport::TestCase test "create after upload sets byte size and checksum" do data = "Hello world!" blob = create_blob data: data @@ -23,6 +23,6 @@ class ActiveVault::BlobTest < ActiveSupport::TestCase private def expected_url_for(blob, disposition: :inline) - "/rails/blobs/#{ActiveVault::VerifiedKeyWithExpiration.encode(blob.key, expires_in: 5.minutes)}?disposition=#{disposition}" + "/rails/blobs/#{ActiveStorage::VerifiedKeyWithExpiration.encode(blob.key, expires_in: 5.minutes)}?disposition=#{disposition}" end end diff --git a/test/database/create_users_migration.rb b/test/database/create_users_migration.rb index 38dcdc129b..15be1938a9 100644 --- a/test/database/create_users_migration.rb +++ b/test/database/create_users_migration.rb @@ -1,4 +1,4 @@ -class ActiveVault::CreateUsers < ActiveRecord::Migration[5.1] +class ActiveStorage::CreateUsers < ActiveRecord::Migration[5.1] def change create_table :users do |t| t.string :name diff --git a/test/database/setup.rb b/test/database/setup.rb index 7373d72237..828edd86dd 100644 --- a/test/database/setup.rb +++ b/test/database/setup.rb @@ -1,6 +1,6 @@ -require "active_vault/migration" +require "active_storage/migration" require_relative "create_users_migration" ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:') -ActiveVault::CreateTables.migrate(:up) -ActiveVault::CreateUsers.migrate(:up) +ActiveStorage::CreateTables.migrate(:up) +ActiveStorage::CreateUsers.migrate(:up) diff --git a/test/disk_controller_test.rb b/test/disk_controller_test.rb index eaf0b497ac..3d7f4ba6bd 100644 --- a/test/disk_controller_test.rb +++ b/test/disk_controller_test.rb @@ -4,30 +4,30 @@ require "database/setup" require "action_controller" require "action_controller/test_case" -require "active_vault/disk_controller" -require "active_vault/verified_key_with_expiration" +require "active_storage/disk_controller" +require "active_storage/verified_key_with_expiration" -class ActiveVault::DiskControllerTest < ActionController::TestCase +class ActiveStorage::DiskControllerTest < ActionController::TestCase Routes = ActionDispatch::Routing::RouteSet.new.tap do |routes| routes.draw do - get "/rails/blobs/:encoded_key" => "active_vault/disk#show", as: :rails_disk_blob + get "/rails/blobs/:encoded_key" => "active_storage/disk#show", as: :rails_disk_blob end end setup do @blob = create_blob @routes = Routes - @controller = ActiveVault::DiskController.new + @controller = ActiveStorage::DiskController.new end test "showing blob inline" do - get :show, params: { encoded_key: ActiveVault::VerifiedKeyWithExpiration.encode(@blob.key, expires_in: 5.minutes) } + get :show, params: { encoded_key: ActiveStorage::VerifiedKeyWithExpiration.encode(@blob.key, expires_in: 5.minutes) } assert_equal "inline; filename=\"#{@blob.filename}\"", @response.headers["Content-Disposition"] assert_equal "text/plain", @response.headers["Content-Type"] end test "sending blob as attachment" do - get :show, params: { encoded_key: ActiveVault::VerifiedKeyWithExpiration.encode(@blob.key, expires_in: 5.minutes), disposition: :attachment } + get :show, params: { encoded_key: ActiveStorage::VerifiedKeyWithExpiration.encode(@blob.key, expires_in: 5.minutes), disposition: :attachment } assert_equal "attachment; filename=\"#{@blob.filename}\"", @response.headers["Content-Disposition"] assert_equal "text/plain", @response.headers["Content-Type"] end diff --git a/test/filename_test.rb b/test/filename_test.rb index 5cb67016c0..448ba7f766 100644 --- a/test/filename_test.rb +++ b/test/filename_test.rb @@ -1,9 +1,9 @@ require "test_helper" -class ActiveVault::FilenameTest < ActiveSupport::TestCase +class ActiveStorage::FilenameTest < ActiveSupport::TestCase test "sanitize" do "%$|:;/\t\r\n\\".each_char do |character| - filename = ActiveVault::Filename.new("foo#{character}bar.pdf") + filename = ActiveStorage::Filename.new("foo#{character}bar.pdf") assert_equal 'foo-bar.pdf', filename.sanitized assert_equal 'foo-bar.pdf', filename.to_s end @@ -16,21 +16,21 @@ class ActiveVault::FilenameTest < ActiveSupport::TestCase "\xCF" => "�", "\x00" => "", }.each do |actual, expected| - assert_equal expected, ActiveVault::Filename.new(actual).sanitized + assert_equal expected, ActiveStorage::Filename.new(actual).sanitized end end test "strips RTL override chars used to spoof unsafe executables as docs" do # Would be displayed in Windows as "evilexe.pdf" due to the right-to-left # (RTL) override char! - assert_equal 'evil-fdp.exe', ActiveVault::Filename.new("evil\u{202E}fdp.exe").sanitized + assert_equal 'evil-fdp.exe', ActiveStorage::Filename.new("evil\u{202E}fdp.exe").sanitized end test "compare case-insensitively" do - assert_operator ActiveVault::Filename.new('foobar.pdf'), :==, ActiveVault::Filename.new('FooBar.PDF') + assert_operator ActiveStorage::Filename.new('foobar.pdf'), :==, ActiveStorage::Filename.new('FooBar.PDF') end test "compare sanitized" do - assert_operator ActiveVault::Filename.new('foo-bar.pdf'), :==, ActiveVault::Filename.new("foo\tbar.pdf") + assert_operator ActiveStorage::Filename.new('foo-bar.pdf'), :==, ActiveStorage::Filename.new("foo\tbar.pdf") end end diff --git a/test/site/disk_site_test.rb b/test/site/disk_site_test.rb index e9ebdcb0be..a04414ea68 100644 --- a/test/site/disk_site_test.rb +++ b/test/site/disk_site_test.rb @@ -1,8 +1,8 @@ require "tmpdir" require "site/shared_site_tests" -class ActiveVault::Site::DiskSiteTest < ActiveSupport::TestCase - SITE = ActiveVault::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_vault")) +class ActiveStorage::Site::DiskSiteTest < ActiveSupport::TestCase + SITE = ActiveStorage::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage")) - include ActiveVault::Site::SharedSiteTests + include ActiveStorage::Site::SharedSiteTests end diff --git a/test/site/gcs_site_test.rb b/test/site/gcs_site_test.rb index b3712a2116..98b1a3d767 100644 --- a/test/site/gcs_site_test.rb +++ b/test/site/gcs_site_test.rb @@ -1,10 +1,10 @@ require "site/shared_site_tests" if SITE_CONFIGURATIONS[:gcs] - class ActiveVault::Site::GCSSiteTest < ActiveSupport::TestCase - SITE = ActiveVault::Site.configure(:GCS, SITE_CONFIGURATIONS[:gcs]) + class ActiveStorage::Site::GCSSiteTest < ActiveSupport::TestCase + SITE = ActiveStorage::Site.configure(:GCS, SITE_CONFIGURATIONS[:gcs]) - include ActiveVault::Site::SharedSiteTests + include ActiveStorage::Site::SharedSiteTests test "signed URL generation" do travel_to Time.now do diff --git a/test/site/mirror_site_test.rb b/test/site/mirror_site_test.rb index bdb0b4c357..7ced377cde 100644 --- a/test/site/mirror_site_test.rb +++ b/test/site/mirror_site_test.rb @@ -1,15 +1,15 @@ require "tmpdir" require "site/shared_site_tests" -class ActiveVault::Site::MirrorSiteTest < ActiveSupport::TestCase - PRIMARY_DISK_SITE = ActiveVault::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_vault")) - SECONDARY_DISK_SITE = ActiveVault::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_vault_mirror")) +class ActiveStorage::Site::MirrorSiteTest < ActiveSupport::TestCase + PRIMARY_DISK_SITE = ActiveStorage::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage")) + SECONDARY_DISK_SITE = ActiveStorage::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage_mirror")) - SITE = ActiveVault::Site.configure :Mirror, sites: [ PRIMARY_DISK_SITE, SECONDARY_DISK_SITE ] + SITE = ActiveStorage::Site.configure :Mirror, sites: [ PRIMARY_DISK_SITE, SECONDARY_DISK_SITE ] - include ActiveVault::Site::SharedSiteTests + include ActiveStorage::Site::SharedSiteTests - test "uploading to all sites" do + test "uploading was done to all sites" do begin key = SecureRandom.base58(24) data = "Something else entirely!" @@ -27,11 +27,4 @@ class ActiveVault::Site::MirrorSiteTest < ActiveSupport::TestCase assert PRIMARY_DISK_SITE.exist?(FIXTURE_KEY) assert SECONDARY_DISK_SITE.exist?(FIXTURE_KEY) end - - test "URL generation for primary site" do - travel_to Time.now do - assert_equal PRIMARY_DISK_SITE.url(FIXTURE_KEY, expires_in: 5.minutes, disposition: :inline, filename: "test.txt"), - SITE.url(FIXTURE_KEY, expires_in: 5.minutes, disposition: :inline, filename: "test.txt") - end - end end diff --git a/test/site/s3_site_test.rb b/test/site/s3_site_test.rb index 6daeaac2ea..a9cb6ca618 100644 --- a/test/site/s3_site_test.rb +++ b/test/site/s3_site_test.rb @@ -1,10 +1,10 @@ require "site/shared_site_tests" if SITE_CONFIGURATIONS[:s3] - class ActiveVault::Site::S3SiteTest < ActiveSupport::TestCase - SITE = ActiveVault::Site.configure(:S3, SITE_CONFIGURATIONS[:s3]) + class ActiveStorage::Site::S3SiteTest < ActiveSupport::TestCase + SITE = ActiveStorage::Site.configure(:S3, SITE_CONFIGURATIONS[:s3]) - include ActiveVault::Site::SharedSiteTests + include ActiveStorage::Site::SharedSiteTests end else puts "Skipping S3 Site tests because no S3 configuration was supplied" diff --git a/test/site/shared_site_tests.rb b/test/site/shared_site_tests.rb index 56f1a13742..687c35e941 100644 --- a/test/site/shared_site_tests.rb +++ b/test/site/shared_site_tests.rb @@ -8,7 +8,7 @@ rescue Errno::ENOENT puts "Missing site configuration file in test/sites/configurations.yml" end -module ActiveVault::Site::SharedSiteTests +module ActiveStorage::Site::SharedSiteTests extend ActiveSupport::Concern FIXTURE_KEY = SecureRandom.base58(24) diff --git a/test/test_helper.rb b/test/test_helper.rb index b18613c365..f354a995e4 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -4,25 +4,25 @@ require "active_support/test_case" require "active_support/testing/autorun" require "byebug" -require "active_vault" +require "active_storage" -require "active_vault/site" -ActiveVault::Blob.site = ActiveVault::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_vault")) +require "active_storage/site" +ActiveStorage::Blob.site = ActiveStorage::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage")) -require "active_vault/verified_key_with_expiration" -ActiveVault::VerifiedKeyWithExpiration.verifier = ActiveSupport::MessageVerifier.new("Testing") +require "active_storage/verified_key_with_expiration" +ActiveStorage::VerifiedKeyWithExpiration.verifier = ActiveSupport::MessageVerifier.new("Testing") class ActiveSupport::TestCase private def create_blob(data: "Hello world!", filename: "hello.txt", content_type: "text/plain") - ActiveVault::Blob.create_after_upload! io: StringIO.new(data), filename: filename, content_type: content_type + ActiveStorage::Blob.create_after_upload! io: StringIO.new(data), filename: filename, content_type: content_type end end -require "active_vault/attached" -ActiveRecord::Base.send :extend, ActiveVault::Attached::Macros +require "active_storage/attached" +ActiveRecord::Base.send :extend, ActiveStorage::Attached::Macros require "global_id" -GlobalID.app = "ActiveVaultExampleApp" +GlobalID.app = "ActiveStorageExampleApp" ActiveRecord::Base.send :include, GlobalID::Identification diff --git a/test/verified_key_with_expiration_test.rb b/test/verified_key_with_expiration_test.rb index 073bb047f6..ee4dc7e02e 100644 --- a/test/verified_key_with_expiration_test.rb +++ b/test/verified_key_with_expiration_test.rb @@ -1,19 +1,19 @@ require "test_helper" require "active_support/core_ext/securerandom" -class ActiveVault::VerifiedKeyWithExpirationTest < ActiveSupport::TestCase +class ActiveStorage::VerifiedKeyWithExpirationTest < ActiveSupport::TestCase FIXTURE_KEY = SecureRandom.base58(24) test "without expiration" do - encoded_key = ActiveVault::VerifiedKeyWithExpiration.encode(FIXTURE_KEY) - assert_equal FIXTURE_KEY, ActiveVault::VerifiedKeyWithExpiration.decode(encoded_key) + encoded_key = ActiveStorage::VerifiedKeyWithExpiration.encode(FIXTURE_KEY) + assert_equal FIXTURE_KEY, ActiveStorage::VerifiedKeyWithExpiration.decode(encoded_key) end test "with expiration" do - encoded_key = ActiveVault::VerifiedKeyWithExpiration.encode(FIXTURE_KEY, expires_in: 1.minute) - assert_equal FIXTURE_KEY, ActiveVault::VerifiedKeyWithExpiration.decode(encoded_key) + encoded_key = ActiveStorage::VerifiedKeyWithExpiration.encode(FIXTURE_KEY, expires_in: 1.minute) + assert_equal FIXTURE_KEY, ActiveStorage::VerifiedKeyWithExpiration.decode(encoded_key) travel 2.minutes - assert_nil ActiveVault::VerifiedKeyWithExpiration.decode(encoded_key) + assert_nil ActiveStorage::VerifiedKeyWithExpiration.decode(encoded_key) end end -- cgit v1.2.3