diff options
author | David Heinemeier Hansson <david@loudthinking.com> | 2017-07-05 13:06:29 +0200 |
---|---|---|
committer | David Heinemeier Hansson <david@loudthinking.com> | 2017-07-05 13:06:29 +0200 |
commit | 571509ad12bf3bcb3190efd7494a38c4796302b8 (patch) | |
tree | 083bcf7f1a56e6856c486de7accef2d3778826e7 /lib/active_vault | |
parent | e635dac88f0dfcc36a2313c10f860cb6e3a52cfa (diff) | |
download | rails-571509ad12bf3bcb3190efd7494a38c4796302b8.tar.gz rails-571509ad12bf3bcb3190efd7494a38c4796302b8.tar.bz2 rails-571509ad12bf3bcb3190efd7494a38c4796302b8.zip |
Rename from ActiveFile to ActiveVault since activefile gem name was taken
Diffstat (limited to 'lib/active_vault')
-rw-r--r-- | lib/active_vault/blob.rb | 66 | ||||
-rw-r--r-- | lib/active_vault/config/sites.yml | 25 | ||||
-rw-r--r-- | lib/active_vault/disk_controller.rb | 28 | ||||
-rw-r--r-- | lib/active_vault/download.rb | 90 | ||||
-rw-r--r-- | lib/active_vault/filename.rb | 31 | ||||
-rw-r--r-- | lib/active_vault/migration.rb | 15 | ||||
-rw-r--r-- | lib/active_vault/purge_job.rb | 7 | ||||
-rw-r--r-- | lib/active_vault/railtie.rb | 19 | ||||
-rw-r--r-- | lib/active_vault/site.rb | 41 | ||||
-rw-r--r-- | lib/active_vault/site/disk_site.rb | 71 | ||||
-rw-r--r-- | lib/active_vault/site/gcs_site.rb | 47 | ||||
-rw-r--r-- | lib/active_vault/site/mirror_site.rb | 44 | ||||
-rw-r--r-- | lib/active_vault/site/s3_site.rb | 63 | ||||
-rw-r--r-- | lib/active_vault/verified_key_with_expiration.rb | 24 |
14 files changed, 571 insertions, 0 deletions
diff --git a/lib/active_vault/blob.rb b/lib/active_vault/blob.rb new file mode 100644 index 0000000000..4948d43ec7 --- /dev/null +++ b/lib/active_vault/blob.rb @@ -0,0 +1,66 @@ +require "active_vault/site" +require "active_vault/filename" + +# Schema: id, key, filename, content_type, metadata, byte_size, checksum, created_at +class ActiveVault::Blob < ActiveRecord::Base + self.table_name = "rails_blobs" + + has_secure_token :key + store :metadata, coder: JSON + + class_attribute :site + + class << self + def build_after_upload(io:, filename:, content_type: nil, metadata: nil) + new.tap do |blob| + blob.filename = filename + blob.content_type = content_type + + blob.upload io + end + end + + def create_after_upload!(io:, filename:, content_type: nil, metadata: nil) + build_after_upload(io: io, filename: filename, content_type: content_type, metadata: metadata).tap(&:save!) + end + end + + # We can't wait until the record is first saved to have a key for it + def key + self[:key] ||= self.class.generate_unique_secure_token + end + + def filename + ActiveVault::Filename.new(self[:filename]) + end + + def url(expires_in: 5.minutes, disposition: :inline) + site.url key, expires_in: expires_in, disposition: disposition, filename: filename + end + + + def upload(io) + site.upload(key, io) + + self.checksum = site.checksum(key) + self.byte_size = site.byte_size(key) + end + + def download + site.download key + end + + + def delete + site.delete key + end + + def purge + delete + destroy + end + + def purge_later + ActiveVault::PurgeJob.perform_later(self) + end +end diff --git a/lib/active_vault/config/sites.yml b/lib/active_vault/config/sites.yml new file mode 100644 index 0000000000..334e779b28 --- /dev/null +++ b/lib/active_vault/config/sites.yml @@ -0,0 +1,25 @@ +# Configuration should be something like this: +# +# config/environments/development.rb +# config.active_vault.site = :local +# +# config/environments/production.rb +# config.active_vault.site = :amazon +local: + site: Disk + root: <%%= File.join(Dir.tmpdir, "active_vault") %> + +amazon: + site: S3 + access_key_id: <%%= Rails.application.secrets.aws[:access_key_id] %> + secret_access_key: <%%= Rails.application.secrets.aws[:secret_access_key] %> + region: us-east-1 + bucket: <%= Rails.application.class.name.remove(/::Application$/).underscore %> + +google: + site: GCS + +mirror: + site: Mirror + primary: amazon + secondaries: google diff --git a/lib/active_vault/disk_controller.rb b/lib/active_vault/disk_controller.rb new file mode 100644 index 0000000000..623569f0f6 --- /dev/null +++ b/lib/active_vault/disk_controller.rb @@ -0,0 +1,28 @@ +require "action_controller" +require "active_vault/blob" +require "active_vault/verified_key_with_expiration" + +require "active_support/core_ext/object/inclusion" + +class ActiveVault::DiskController < ActionController::Base + def show + if key = decode_verified_key + blob = ActiveVault::Blob.find_by!(key: key) + + if stale?(etag: blob.checksum) + send_data blob.download, filename: blob.filename, type: blob.content_type, disposition: disposition_param + end + else + head :not_found + end + end + + private + def decode_verified_key + ActiveVault::VerifiedKeyWithExpiration.decode(params[:encoded_key]) + end + + def disposition_param + params[:disposition].presence_in(%w( inline attachment )) || 'inline' + end +end diff --git a/lib/active_vault/download.rb b/lib/active_vault/download.rb new file mode 100644 index 0000000000..6e74056062 --- /dev/null +++ b/lib/active_vault/download.rb @@ -0,0 +1,90 @@ +class ActiveVault::Download + # Sending .ai files as application/postscript to Safari opens them in a blank, grey screen. + # Downloading .ai as application/postscript files in Safari appends .ps to the extension. + # Sending HTML, SVG, XML and SWF files as binary closes XSS vulnerabilities. + # Sending JS files as binary avoids InvalidCrossOriginRequest without compromising security. + CONTENT_TYPES_TO_RENDER_AS_BINARY = %w( + text/html + text/javascript + image/svg+xml + application/postscript + application/x-shockwave-flash + text/xml + application/xml + application/xhtml+xml + ) + + BINARY_CONTENT_TYPE = 'application/octet-stream' + + def initialize(stored_file) + @stored_file = stored_file + end + + def headers(force_attachment: false) + { + x_accel_redirect: '/reproxy', + x_reproxy_url: reproxy_url, + content_type: content_type, + content_disposition: content_disposition(force_attachment), + x_frame_options: 'SAMEORIGIN' + } + end + + private + def reproxy_url + @stored_file.depot_location.paths.first + end + + def content_type + if @stored_file.content_type.in? CONTENT_TYPES_TO_RENDER_AS_BINARY + BINARY_CONTENT_TYPE + else + @stored_file.content_type + end + end + + def content_disposition(force_attachment = false) + if force_attachment || content_type == BINARY_CONTENT_TYPE + "attachment; #{escaped_filename}" + else + "inline; #{escaped_filename}" + end + end + + # RFC2231 encoding for UTF-8 filenames, with an ASCII fallback + # first for unsupported browsers (IE < 9, perhaps others?). + # http://greenbytes.de/tech/tc2231/#encoding-2231-fb + def escaped_filename + filename = @stored_file.filename.sanitized + ascii_filename = encode_ascii_filename(filename) + utf8_filename = encode_utf8_filename(filename) + "#{ascii_filename}; #{utf8_filename}" + end + + TRADITIONAL_PARAMETER_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ + + def encode_ascii_filename(filename) + # There is no reliable way to escape special or non-Latin characters + # in a traditionally quoted Content-Disposition filename parameter. + # Settle for transliterating to ASCII, then percent-escaping special + # characters, excluding spaces. + filename = I18n.transliterate(filename) + filename = percent_escape(filename, TRADITIONAL_PARAMETER_ESCAPED_CHAR) + %(filename="#{filename}") + end + + RFC5987_PARAMETER_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ + + def encode_utf8_filename(filename) + # RFC2231 filename parameters can simply be percent-escaped according + # to RFC5987. + filename = percent_escape(filename, RFC5987_PARAMETER_ESCAPED_CHAR) + %(filename*=UTF-8''#{filename}) + end + + def percent_escape(string, pattern) + string.gsub(pattern) do |char| + char.bytes.map { |byte| "%%%02X" % byte }.join("") + end + end +end diff --git a/lib/active_vault/filename.rb b/lib/active_vault/filename.rb new file mode 100644 index 0000000000..647d037b1f --- /dev/null +++ b/lib/active_vault/filename.rb @@ -0,0 +1,31 @@ +class ActiveVault::Filename + include Comparable + + def initialize(filename) + @filename = filename + end + + def extname + File.extname(@filename) + end + + def extension + extname.from(1) + end + + def base + File.basename(@filename, extname) + end + + def sanitized + @filename.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-") + end + + def to_s + sanitized.to_s + end + + def <=>(other) + to_s.downcase <=> other.to_s.downcase + end +end diff --git a/lib/active_vault/migration.rb b/lib/active_vault/migration.rb new file mode 100644 index 0000000000..cc7a535f39 --- /dev/null +++ b/lib/active_vault/migration.rb @@ -0,0 +1,15 @@ +class ActiveVault::CreateBlobs < ActiveRecord::Migration[5.1] + def change + create_table :rails_blobs do |t| + t.string :key + t.string :filename + t.string :content_type + t.text :metadata + t.integer :byte_size + t.string :checksum + t.time :created_at + + t.index [ :key ], unique: true + end + end +end diff --git a/lib/active_vault/purge_job.rb b/lib/active_vault/purge_job.rb new file mode 100644 index 0000000000..d7634af2bb --- /dev/null +++ b/lib/active_vault/purge_job.rb @@ -0,0 +1,7 @@ +class ActiveVault::PurgeJob < ActiveJob::Base + retry_on ActiveVault::StorageException + + def perform(blob) + blob.purge + end +end diff --git a/lib/active_vault/railtie.rb b/lib/active_vault/railtie.rb new file mode 100644 index 0000000000..c254f4c77c --- /dev/null +++ b/lib/active_vault/railtie.rb @@ -0,0 +1,19 @@ +require "rails/railtie" + +module ActiveVault + class Railtie < Rails::Railtie # :nodoc: + config.action_file = ActiveSupport::OrderedOptions.new + + config.eager_load_namespaces << ActiveVault + + initializer "action_file.routes" do + require "active_vault/disk_controller" + + config.after_initialize do |app| + app.routes.prepend do + get "/rails/blobs/:encoded_key" => "active_vault/disk#show", as: :rails_disk_blob + end + end + end + end +end diff --git a/lib/active_vault/site.rb b/lib/active_vault/site.rb new file mode 100644 index 0000000000..29eddf1566 --- /dev/null +++ b/lib/active_vault/site.rb @@ -0,0 +1,41 @@ +# Abstract class serving as an interface for concrete sites. +class ActiveVault::Site + def self.configure(site, **options) + begin + require "active_vault/site/#{site.to_s.downcase}_site" + ActiveVault::Site.const_get(:"#{site}Site").new(**options) + rescue LoadError => e + puts "Couldn't configure site: #{site} (#{e.message})" + end + end + + + def upload(key, io) + raise NotImplementedError + end + + def download(key) + raise NotImplementedError + end + + def delete(key) + raise NotImplementedError + end + + def exist?(key) + raise NotImplementedError + end + + + def url(key, expires_in:, disposition:, filename:) + raise NotImplementedError + end + + def bytesize(key) + raise NotImplementedError + end + + def checksum(key) + raise NotImplementedError + end +end diff --git a/lib/active_vault/site/disk_site.rb b/lib/active_vault/site/disk_site.rb new file mode 100644 index 0000000000..73f86bac6a --- /dev/null +++ b/lib/active_vault/site/disk_site.rb @@ -0,0 +1,71 @@ +require "fileutils" +require "pathname" + +class ActiveVault::Site::DiskSite < ActiveVault::Site + attr_reader :root + + def initialize(root:) + @root = root + end + + def upload(key, io) + File.open(make_path_for(key), "wb") do |file| + while chunk = io.read(65536) + file.write(chunk) + end + end + end + + def download(key) + if block_given? + File.open(path_for(key)) do |file| + while data = file.read(65536) + yield data + end + end + else + File.open path_for(key), &:read + end + end + + def delete(key) + File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted + end + + def exist?(key) + File.exist? path_for(key) + end + + + def url(key, expires_in:, disposition:, filename:) + verified_key_with_expiration = ActiveVault::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) + + if defined?(Rails) && defined?(Rails.application) + Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) + else + "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" + end + end + + def byte_size(key) + File.size path_for(key) + end + + def checksum(key) + Digest::MD5.file(path_for(key)).hexdigest + end + + + private + def path_for(key) + File.join root, folder_for(key), key + end + + def folder_for(key) + [ key[0..1], key[2..3] ].join("/") + end + + def make_path_for(key) + path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } + end +end diff --git a/lib/active_vault/site/gcs_site.rb b/lib/active_vault/site/gcs_site.rb new file mode 100644 index 0000000000..e509ebbbd2 --- /dev/null +++ b/lib/active_vault/site/gcs_site.rb @@ -0,0 +1,47 @@ +require "google/cloud/storage" + +class ActiveVault::Site::GCSSite < ActiveVault::Site + attr_reader :client, :bucket + + def initialize(project:, keyfile:, bucket:) + @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + bucket.create_file(io, key) + end + + def download(key) + io = file_for(key).download + io.rewind + io.read + end + + def delete(key) + file_for(key).try(:delete) + end + + def exist?(key) + file_for(key).present? + end + + + def byte_size(key) + file_for(key).size + end + + def checksum(key) + convert_to_hex base64: file_for(key).md5 + end + + + private + def file_for(key) + bucket.file(key) + end + + def convert_to_hex(base64:) + base64.unpack("m0").first.unpack("H*").first + end +end diff --git a/lib/active_vault/site/mirror_site.rb b/lib/active_vault/site/mirror_site.rb new file mode 100644 index 0000000000..67d79a2607 --- /dev/null +++ b/lib/active_vault/site/mirror_site.rb @@ -0,0 +1,44 @@ +class ActiveVault::Site::MirrorSite < ActiveVault::Site + attr_reader :sites + + def initialize(sites:) + @sites = sites + end + + def upload(key, io) + perform_across_sites :upload, key, io + end + + def download(key) + sites.detect { |site| site.exist?(key) }.download(key) + end + + def delete(key) + perform_across_sites :delete, key + end + + def exist?(key) + perform_across_sites(:exist?, key).any? + end + + + def byte_size(key) + primary_site.byte_size(key) + end + + def checksum(key) + primary_site.checksum(key) + end + + private + def primary_site + sites.first + end + + def perform_across_sites(method, **args) + # FIXME: Convert to be threaded + sites.collect do |site| + site.send method, **args + end + end +end diff --git a/lib/active_vault/site/s3_site.rb b/lib/active_vault/site/s3_site.rb new file mode 100644 index 0000000000..49a7522170 --- /dev/null +++ b/lib/active_vault/site/s3_site.rb @@ -0,0 +1,63 @@ +require "aws-sdk" + +class ActiveVault::Site::S3Site < ActiveVault::Site + attr_reader :client, :bucket + + def initialize(access_key_id:, secret_access_key:, region:, bucket:) + @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + object_for(key).put(body: io) + end + + def download(key) + if block_given? + stream(key, &block) + else + object_for(key).get.body.read + end + end + + def delete(key) + object_for(key).delete + end + + def exist?(key) + object_for(key).exists? + end + + + def url(key, expires_in:, disposition:, filename:) + object_for(key).presigned_url :get, expires_in: expires_in, + response_content_disposition: "#{disposition}; filename=\"#{filename}\"" + end + + def byte_size(key) + object_for(key).size + end + + def checksum(key) + object_for(key).etag.remove(/"/) + end + + + private + def object_for(key) + bucket.object(key) + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key, options = {}, &block) + object = object_for(key) + + chunk_size = 5242880 # 5 megabytes + offset = 0 + + while offset < object.content_length + yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) + offset += chunk_size + end + end +end diff --git a/lib/active_vault/verified_key_with_expiration.rb b/lib/active_vault/verified_key_with_expiration.rb new file mode 100644 index 0000000000..95d4993ff0 --- /dev/null +++ b/lib/active_vault/verified_key_with_expiration.rb @@ -0,0 +1,24 @@ +class ActiveVault::VerifiedKeyWithExpiration + class_attribute :verifier, default: defined?(Rails) ? Rails.application.message_verifier('ActiveVault') : nil + + class << self + def encode(key, expires_in: nil) + verifier.generate([ key, expires_at(expires_in) ]) + end + + def decode(encoded_key) + key, expires_at = verifier.verified(encoded_key) + + key if key && fresh?(expires_at) + end + + private + def expires_at(expires_in) + expires_in ? Time.now.utc.advance(seconds: expires_in) : nil + end + + def fresh?(expires_at) + expires_at.nil? || Time.now.utc < expires_at + end + end +end |