From 571509ad12bf3bcb3190efd7494a38c4796302b8 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Wed, 5 Jul 2017 13:06:29 +0200 Subject: Rename from ActiveFile to ActiveVault since activefile gem name was taken --- lib/active_file.rb | 9 --- lib/active_file/blob.rb | 66 ----------------- lib/active_file/config/sites.yml | 25 ------- lib/active_file/disk_controller.rb | 28 -------- lib/active_file/download.rb | 90 ------------------------ lib/active_file/filename.rb | 31 -------- lib/active_file/migration.rb | 15 ---- lib/active_file/purge_job.rb | 7 -- lib/active_file/railtie.rb | 19 ----- lib/active_file/site.rb | 41 ----------- lib/active_file/site/disk_site.rb | 71 ------------------- lib/active_file/site/gcs_site.rb | 47 ------------- lib/active_file/site/mirror_site.rb | 44 ------------ lib/active_file/site/s3_site.rb | 63 ----------------- lib/active_file/verified_key_with_expiration.rb | 24 ------- lib/active_vault.rb | 9 +++ lib/active_vault/blob.rb | 66 +++++++++++++++++ lib/active_vault/config/sites.yml | 25 +++++++ lib/active_vault/disk_controller.rb | 28 ++++++++ lib/active_vault/download.rb | 90 ++++++++++++++++++++++++ lib/active_vault/filename.rb | 31 ++++++++ lib/active_vault/migration.rb | 15 ++++ lib/active_vault/purge_job.rb | 7 ++ lib/active_vault/railtie.rb | 19 +++++ lib/active_vault/site.rb | 41 +++++++++++ lib/active_vault/site/disk_site.rb | 71 +++++++++++++++++++ lib/active_vault/site/gcs_site.rb | 47 +++++++++++++ lib/active_vault/site/mirror_site.rb | 44 ++++++++++++ lib/active_vault/site/s3_site.rb | 63 +++++++++++++++++ lib/active_vault/verified_key_with_expiration.rb | 24 +++++++ 30 files changed, 580 insertions(+), 580 deletions(-) delete mode 100644 lib/active_file.rb delete mode 100644 lib/active_file/blob.rb delete mode 100644 lib/active_file/config/sites.yml delete mode 100644 lib/active_file/disk_controller.rb delete mode 100644 lib/active_file/download.rb delete mode 100644 lib/active_file/filename.rb delete mode 100644 lib/active_file/migration.rb delete mode 100644 lib/active_file/purge_job.rb delete mode 100644 lib/active_file/railtie.rb delete mode 100644 lib/active_file/site.rb delete mode 100644 lib/active_file/site/disk_site.rb delete mode 100644 lib/active_file/site/gcs_site.rb delete mode 100644 lib/active_file/site/mirror_site.rb delete mode 100644 lib/active_file/site/s3_site.rb delete mode 100644 lib/active_file/verified_key_with_expiration.rb create mode 100644 lib/active_vault.rb create mode 100644 lib/active_vault/blob.rb create mode 100644 lib/active_vault/config/sites.yml create mode 100644 lib/active_vault/disk_controller.rb create mode 100644 lib/active_vault/download.rb create mode 100644 lib/active_vault/filename.rb create mode 100644 lib/active_vault/migration.rb create mode 100644 lib/active_vault/purge_job.rb create mode 100644 lib/active_vault/railtie.rb create mode 100644 lib/active_vault/site.rb create mode 100644 lib/active_vault/site/disk_site.rb create mode 100644 lib/active_vault/site/gcs_site.rb create mode 100644 lib/active_vault/site/mirror_site.rb create mode 100644 lib/active_vault/site/s3_site.rb create mode 100644 lib/active_vault/verified_key_with_expiration.rb (limited to 'lib') diff --git a/lib/active_file.rb b/lib/active_file.rb deleted file mode 100644 index b4b319fc8e..0000000000 --- a/lib/active_file.rb +++ /dev/null @@ -1,9 +0,0 @@ -require "active_record" -require "active_file/railtie" if defined?(Rails) - -module ActiveFile - extend ActiveSupport::Autoload - - autoload :Blob - autoload :Site -end diff --git a/lib/active_file/blob.rb b/lib/active_file/blob.rb deleted file mode 100644 index d8b9cd07d2..0000000000 --- a/lib/active_file/blob.rb +++ /dev/null @@ -1,66 +0,0 @@ -require "active_file/site" -require "active_file/filename" - -# Schema: id, key, filename, content_type, metadata, byte_size, checksum, created_at -class ActiveFile::Blob < ActiveRecord::Base - self.table_name = "rails_blobs" - - has_secure_token :key - store :metadata, coder: JSON - - class_attribute :site - - class << self - def build_after_upload(io:, filename:, content_type: nil, metadata: nil) - new.tap do |blob| - blob.filename = filename - blob.content_type = content_type - - blob.upload io - end - end - - def create_after_upload!(io:, filename:, content_type: nil, metadata: nil) - build_after_upload(io: io, filename: filename, content_type: content_type, metadata: metadata).tap(&:save!) - end - end - - # We can't wait until the record is first saved to have a key for it - def key - self[:key] ||= self.class.generate_unique_secure_token - end - - def filename - ActiveFile::Filename.new(self[:filename]) - end - - def url(expires_in: 5.minutes, disposition: :inline) - site.url key, expires_in: expires_in, disposition: disposition, filename: filename - end - - - def upload(io) - site.upload(key, io) - - self.checksum = site.checksum(key) - self.byte_size = site.byte_size(key) - end - - def download - site.download key - end - - - def delete - site.delete key - end - - def purge - delete - destroy - end - - def purge_later - ActiveFile::PurgeJob.perform_later(self) - end -end diff --git a/lib/active_file/config/sites.yml b/lib/active_file/config/sites.yml deleted file mode 100644 index bb550aed7a..0000000000 --- a/lib/active_file/config/sites.yml +++ /dev/null @@ -1,25 +0,0 @@ -# Configuration should be something like this: -# -# config/environments/development.rb -# config.active_file.site = :local -# -# config/environments/production.rb -# config.active_file.site = :amazon -local: - site: Disk - root: <%%= File.join(Dir.tmpdir, "active_file") %> - -amazon: - site: S3 - access_key_id: <%%= Rails.application.secrets.aws[:access_key_id] %> - secret_access_key: <%%= Rails.application.secrets.aws[:secret_access_key] %> - region: us-east-1 - bucket: <%= Rails.application.class.name.remove(/::Application$/).underscore %> - -google: - site: GCS - -mirror: - site: Mirror - primary: amazon - secondaries: google diff --git a/lib/active_file/disk_controller.rb b/lib/active_file/disk_controller.rb deleted file mode 100644 index d778cf066f..0000000000 --- a/lib/active_file/disk_controller.rb +++ /dev/null @@ -1,28 +0,0 @@ -require "action_controller" -require "active_file/blob" -require "active_file/verified_key_with_expiration" - -require "active_support/core_ext/object/inclusion" - -class ActiveFile::DiskController < ActionController::Base - def show - if key = decode_verified_key - blob = ActiveFile::Blob.find_by!(key: key) - - if stale?(etag: blob.checksum) - send_data blob.download, filename: blob.filename, type: blob.content_type, disposition: disposition_param - end - else - head :not_found - end - end - - private - def decode_verified_key - ActiveFile::VerifiedKeyWithExpiration.decode(params[:encoded_key]) - end - - def disposition_param - params[:disposition].presence_in(%w( inline attachment )) || 'inline' - end -end diff --git a/lib/active_file/download.rb b/lib/active_file/download.rb deleted file mode 100644 index 74f69a9dfc..0000000000 --- a/lib/active_file/download.rb +++ /dev/null @@ -1,90 +0,0 @@ -class ActiveFile::Download - # Sending .ai files as application/postscript to Safari opens them in a blank, grey screen. - # Downloading .ai as application/postscript files in Safari appends .ps to the extension. - # Sending HTML, SVG, XML and SWF files as binary closes XSS vulnerabilities. - # Sending JS files as binary avoids InvalidCrossOriginRequest without compromising security. - CONTENT_TYPES_TO_RENDER_AS_BINARY = %w( - text/html - text/javascript - image/svg+xml - application/postscript - application/x-shockwave-flash - text/xml - application/xml - application/xhtml+xml - ) - - BINARY_CONTENT_TYPE = 'application/octet-stream' - - def initialize(stored_file) - @stored_file = stored_file - end - - def headers(force_attachment: false) - { - x_accel_redirect: '/reproxy', - x_reproxy_url: reproxy_url, - content_type: content_type, - content_disposition: content_disposition(force_attachment), - x_frame_options: 'SAMEORIGIN' - } - end - - private - def reproxy_url - @stored_file.depot_location.paths.first - end - - def content_type - if @stored_file.content_type.in? CONTENT_TYPES_TO_RENDER_AS_BINARY - BINARY_CONTENT_TYPE - else - @stored_file.content_type - end - end - - def content_disposition(force_attachment = false) - if force_attachment || content_type == BINARY_CONTENT_TYPE - "attachment; #{escaped_filename}" - else - "inline; #{escaped_filename}" - end - end - - # RFC2231 encoding for UTF-8 filenames, with an ASCII fallback - # first for unsupported browsers (IE < 9, perhaps others?). - # http://greenbytes.de/tech/tc2231/#encoding-2231-fb - def escaped_filename - filename = @stored_file.filename.sanitized - ascii_filename = encode_ascii_filename(filename) - utf8_filename = encode_utf8_filename(filename) - "#{ascii_filename}; #{utf8_filename}" - end - - TRADITIONAL_PARAMETER_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ - - def encode_ascii_filename(filename) - # There is no reliable way to escape special or non-Latin characters - # in a traditionally quoted Content-Disposition filename parameter. - # Settle for transliterating to ASCII, then percent-escaping special - # characters, excluding spaces. - filename = I18n.transliterate(filename) - filename = percent_escape(filename, TRADITIONAL_PARAMETER_ESCAPED_CHAR) - %(filename="#{filename}") - end - - RFC5987_PARAMETER_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ - - def encode_utf8_filename(filename) - # RFC2231 filename parameters can simply be percent-escaped according - # to RFC5987. - filename = percent_escape(filename, RFC5987_PARAMETER_ESCAPED_CHAR) - %(filename*=UTF-8''#{filename}) - end - - def percent_escape(string, pattern) - string.gsub(pattern) do |char| - char.bytes.map { |byte| "%%%02X" % byte }.join("") - end - end -end diff --git a/lib/active_file/filename.rb b/lib/active_file/filename.rb deleted file mode 100644 index b3c184e26c..0000000000 --- a/lib/active_file/filename.rb +++ /dev/null @@ -1,31 +0,0 @@ -class ActiveFile::Filename - include Comparable - - def initialize(filename) - @filename = filename - end - - def extname - File.extname(@filename) - end - - def extension - extname.from(1) - end - - def base - File.basename(@filename, extname) - end - - def sanitized - @filename.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-") - end - - def to_s - sanitized.to_s - end - - def <=>(other) - to_s.downcase <=> other.to_s.downcase - end -end diff --git a/lib/active_file/migration.rb b/lib/active_file/migration.rb deleted file mode 100644 index 1c87444dd4..0000000000 --- a/lib/active_file/migration.rb +++ /dev/null @@ -1,15 +0,0 @@ -class ActiveFile::CreateBlobs < ActiveRecord::Migration[5.1] - def change - create_table :rails_blobs do |t| - t.string :key - t.string :filename - t.string :content_type - t.text :metadata - t.integer :byte_size - t.string :checksum - t.time :created_at - - t.index [ :key ], unique: true - end - end -end diff --git a/lib/active_file/purge_job.rb b/lib/active_file/purge_job.rb deleted file mode 100644 index 1a967db2f0..0000000000 --- a/lib/active_file/purge_job.rb +++ /dev/null @@ -1,7 +0,0 @@ -class ActiveFile::PurgeJob < ActiveJob::Base - retry_on ActiveFile::StorageException - - def perform(blob) - blob.purge - end -end diff --git a/lib/active_file/railtie.rb b/lib/active_file/railtie.rb deleted file mode 100644 index 18e4779229..0000000000 --- a/lib/active_file/railtie.rb +++ /dev/null @@ -1,19 +0,0 @@ -require "rails/railtie" - -module ActiveFile - class Railtie < Rails::Railtie # :nodoc: - config.action_file = ActiveSupport::OrderedOptions.new - - config.eager_load_namespaces << ActiveFile - - initializer "action_file.routes" do - require "active_file/disk_controller" - - config.after_initialize do |app| - app.routes.prepend do - get "/rails/blobs/:encoded_key" => "active_file/disk#show", as: :rails_disk_blob - end - end - end - end -end diff --git a/lib/active_file/site.rb b/lib/active_file/site.rb deleted file mode 100644 index 19cbbc754e..0000000000 --- a/lib/active_file/site.rb +++ /dev/null @@ -1,41 +0,0 @@ -# Abstract class serving as an interface for concrete sites. -class ActiveFile::Site - def self.configure(site, **options) - begin - require "active_file/site/#{site.to_s.downcase}_site" - ActiveFile::Site.const_get(:"#{site}Site").new(**options) - rescue LoadError => e - puts "Couldn't configure site: #{site} (#{e.message})" - end - end - - - def upload(key, io) - raise NotImplementedError - end - - def download(key) - raise NotImplementedError - end - - def delete(key) - raise NotImplementedError - end - - def exist?(key) - raise NotImplementedError - end - - - def url(key, expires_in:, disposition:, filename:) - raise NotImplementedError - end - - def bytesize(key) - raise NotImplementedError - end - - def checksum(key) - raise NotImplementedError - end -end diff --git a/lib/active_file/site/disk_site.rb b/lib/active_file/site/disk_site.rb deleted file mode 100644 index 7916a642c0..0000000000 --- a/lib/active_file/site/disk_site.rb +++ /dev/null @@ -1,71 +0,0 @@ -require "fileutils" -require "pathname" - -class ActiveFile::Site::DiskSite < ActiveFile::Site - attr_reader :root - - def initialize(root:) - @root = root - end - - def upload(key, io) - File.open(make_path_for(key), "wb") do |file| - while chunk = io.read(65536) - file.write(chunk) - end - end - end - - def download(key) - if block_given? - File.open(path_for(key)) do |file| - while data = file.read(65536) - yield data - end - end - else - File.open path_for(key), &:read - end - end - - def delete(key) - File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted - end - - def exist?(key) - File.exist? path_for(key) - end - - - def url(key, expires_in:, disposition:, filename:) - verified_key_with_expiration = ActiveFile::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) - - if defined?(Rails) && defined?(Rails.application) - Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) - else - "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" - end - end - - def byte_size(key) - File.size path_for(key) - end - - def checksum(key) - Digest::MD5.file(path_for(key)).hexdigest - end - - - private - def path_for(key) - File.join root, folder_for(key), key - end - - def folder_for(key) - [ key[0..1], key[2..3] ].join("/") - end - - def make_path_for(key) - path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } - end -end diff --git a/lib/active_file/site/gcs_site.rb b/lib/active_file/site/gcs_site.rb deleted file mode 100644 index c5f3d634cf..0000000000 --- a/lib/active_file/site/gcs_site.rb +++ /dev/null @@ -1,47 +0,0 @@ -require "google/cloud/storage" - -class ActiveFile::Site::GCSSite < ActiveFile::Site - attr_reader :client, :bucket - - def initialize(project:, keyfile:, bucket:) - @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) - @bucket = @client.bucket(bucket) - end - - def upload(key, io) - bucket.create_file(io, key) - end - - def download(key) - io = file_for(key).download - io.rewind - io.read - end - - def delete(key) - file_for(key).try(:delete) - end - - def exist?(key) - file_for(key).present? - end - - - def byte_size(key) - file_for(key).size - end - - def checksum(key) - convert_to_hex base64: file_for(key).md5 - end - - - private - def file_for(key) - bucket.file(key) - end - - def convert_to_hex(base64:) - base64.unpack("m0").first.unpack("H*").first - end -end diff --git a/lib/active_file/site/mirror_site.rb b/lib/active_file/site/mirror_site.rb deleted file mode 100644 index 65f28cd437..0000000000 --- a/lib/active_file/site/mirror_site.rb +++ /dev/null @@ -1,44 +0,0 @@ -class ActiveFile::Site::MirrorSite < ActiveFile::Site - attr_reader :sites - - def initialize(sites:) - @sites = sites - end - - def upload(key, io) - perform_across_sites :upload, key, io - end - - def download(key) - sites.detect { |site| site.exist?(key) }.download(key) - end - - def delete(key) - perform_across_sites :delete, key - end - - def exist?(key) - perform_across_sites(:exist?, key).any? - end - - - def byte_size(key) - primary_site.byte_size(key) - end - - def checksum(key) - primary_site.checksum(key) - end - - private - def primary_site - sites.first - end - - def perform_across_sites(method, **args) - # FIXME: Convert to be threaded - sites.collect do |site| - site.send method, **args - end - end -end diff --git a/lib/active_file/site/s3_site.rb b/lib/active_file/site/s3_site.rb deleted file mode 100644 index 25a876c697..0000000000 --- a/lib/active_file/site/s3_site.rb +++ /dev/null @@ -1,63 +0,0 @@ -require "aws-sdk" - -class ActiveFile::Site::S3Site < ActiveFile::Site - attr_reader :client, :bucket - - def initialize(access_key_id:, secret_access_key:, region:, bucket:) - @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) - @bucket = @client.bucket(bucket) - end - - def upload(key, io) - object_for(key).put(body: io) - end - - def download(key) - if block_given? - stream(key, &block) - else - object_for(key).get.body.read - end - end - - def delete(key) - object_for(key).delete - end - - def exist?(key) - object_for(key).exists? - end - - - def url(key, expires_in:, disposition:, filename:) - object_for(key).presigned_url :get, expires_in: expires_in, - response_content_disposition: "#{disposition}; filename=\"#{filename}\"" - end - - def byte_size(key) - object_for(key).size - end - - def checksum(key) - object_for(key).etag.remove(/"/) - end - - - private - def object_for(key) - bucket.object(key) - end - - # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, options = {}, &block) - object = object_for(key) - - chunk_size = 5242880 # 5 megabytes - offset = 0 - - while offset < object.content_length - yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) - offset += chunk_size - end - end -end diff --git a/lib/active_file/verified_key_with_expiration.rb b/lib/active_file/verified_key_with_expiration.rb deleted file mode 100644 index e9e811d364..0000000000 --- a/lib/active_file/verified_key_with_expiration.rb +++ /dev/null @@ -1,24 +0,0 @@ -class ActiveFile::VerifiedKeyWithExpiration - class_attribute :verifier, default: defined?(Rails) ? Rails.application.message_verifier('ActiveFile') : nil - - class << self - def encode(key, expires_in: nil) - verifier.generate([ key, expires_at(expires_in) ]) - end - - def decode(encoded_key) - key, expires_at = verifier.verified(encoded_key) - - key if key && fresh?(expires_at) - end - - private - def expires_at(expires_in) - expires_in ? Time.now.utc.advance(seconds: expires_in) : nil - end - - def fresh?(expires_at) - expires_at.nil? || Time.now.utc < expires_at - end - end -end diff --git a/lib/active_vault.rb b/lib/active_vault.rb new file mode 100644 index 0000000000..f47b09b4cd --- /dev/null +++ b/lib/active_vault.rb @@ -0,0 +1,9 @@ +require "active_record" +require "active_vault/railtie" if defined?(Rails) + +module ActiveVault + extend ActiveSupport::Autoload + + autoload :Blob + autoload :Site +end diff --git a/lib/active_vault/blob.rb b/lib/active_vault/blob.rb new file mode 100644 index 0000000000..4948d43ec7 --- /dev/null +++ b/lib/active_vault/blob.rb @@ -0,0 +1,66 @@ +require "active_vault/site" +require "active_vault/filename" + +# Schema: id, key, filename, content_type, metadata, byte_size, checksum, created_at +class ActiveVault::Blob < ActiveRecord::Base + self.table_name = "rails_blobs" + + has_secure_token :key + store :metadata, coder: JSON + + class_attribute :site + + class << self + def build_after_upload(io:, filename:, content_type: nil, metadata: nil) + new.tap do |blob| + blob.filename = filename + blob.content_type = content_type + + blob.upload io + end + end + + def create_after_upload!(io:, filename:, content_type: nil, metadata: nil) + build_after_upload(io: io, filename: filename, content_type: content_type, metadata: metadata).tap(&:save!) + end + end + + # We can't wait until the record is first saved to have a key for it + def key + self[:key] ||= self.class.generate_unique_secure_token + end + + def filename + ActiveVault::Filename.new(self[:filename]) + end + + def url(expires_in: 5.minutes, disposition: :inline) + site.url key, expires_in: expires_in, disposition: disposition, filename: filename + end + + + def upload(io) + site.upload(key, io) + + self.checksum = site.checksum(key) + self.byte_size = site.byte_size(key) + end + + def download + site.download key + end + + + def delete + site.delete key + end + + def purge + delete + destroy + end + + def purge_later + ActiveVault::PurgeJob.perform_later(self) + end +end diff --git a/lib/active_vault/config/sites.yml b/lib/active_vault/config/sites.yml new file mode 100644 index 0000000000..334e779b28 --- /dev/null +++ b/lib/active_vault/config/sites.yml @@ -0,0 +1,25 @@ +# Configuration should be something like this: +# +# config/environments/development.rb +# config.active_vault.site = :local +# +# config/environments/production.rb +# config.active_vault.site = :amazon +local: + site: Disk + root: <%%= File.join(Dir.tmpdir, "active_vault") %> + +amazon: + site: S3 + access_key_id: <%%= Rails.application.secrets.aws[:access_key_id] %> + secret_access_key: <%%= Rails.application.secrets.aws[:secret_access_key] %> + region: us-east-1 + bucket: <%= Rails.application.class.name.remove(/::Application$/).underscore %> + +google: + site: GCS + +mirror: + site: Mirror + primary: amazon + secondaries: google diff --git a/lib/active_vault/disk_controller.rb b/lib/active_vault/disk_controller.rb new file mode 100644 index 0000000000..623569f0f6 --- /dev/null +++ b/lib/active_vault/disk_controller.rb @@ -0,0 +1,28 @@ +require "action_controller" +require "active_vault/blob" +require "active_vault/verified_key_with_expiration" + +require "active_support/core_ext/object/inclusion" + +class ActiveVault::DiskController < ActionController::Base + def show + if key = decode_verified_key + blob = ActiveVault::Blob.find_by!(key: key) + + if stale?(etag: blob.checksum) + send_data blob.download, filename: blob.filename, type: blob.content_type, disposition: disposition_param + end + else + head :not_found + end + end + + private + def decode_verified_key + ActiveVault::VerifiedKeyWithExpiration.decode(params[:encoded_key]) + end + + def disposition_param + params[:disposition].presence_in(%w( inline attachment )) || 'inline' + end +end diff --git a/lib/active_vault/download.rb b/lib/active_vault/download.rb new file mode 100644 index 0000000000..6e74056062 --- /dev/null +++ b/lib/active_vault/download.rb @@ -0,0 +1,90 @@ +class ActiveVault::Download + # Sending .ai files as application/postscript to Safari opens them in a blank, grey screen. + # Downloading .ai as application/postscript files in Safari appends .ps to the extension. + # Sending HTML, SVG, XML and SWF files as binary closes XSS vulnerabilities. + # Sending JS files as binary avoids InvalidCrossOriginRequest without compromising security. + CONTENT_TYPES_TO_RENDER_AS_BINARY = %w( + text/html + text/javascript + image/svg+xml + application/postscript + application/x-shockwave-flash + text/xml + application/xml + application/xhtml+xml + ) + + BINARY_CONTENT_TYPE = 'application/octet-stream' + + def initialize(stored_file) + @stored_file = stored_file + end + + def headers(force_attachment: false) + { + x_accel_redirect: '/reproxy', + x_reproxy_url: reproxy_url, + content_type: content_type, + content_disposition: content_disposition(force_attachment), + x_frame_options: 'SAMEORIGIN' + } + end + + private + def reproxy_url + @stored_file.depot_location.paths.first + end + + def content_type + if @stored_file.content_type.in? CONTENT_TYPES_TO_RENDER_AS_BINARY + BINARY_CONTENT_TYPE + else + @stored_file.content_type + end + end + + def content_disposition(force_attachment = false) + if force_attachment || content_type == BINARY_CONTENT_TYPE + "attachment; #{escaped_filename}" + else + "inline; #{escaped_filename}" + end + end + + # RFC2231 encoding for UTF-8 filenames, with an ASCII fallback + # first for unsupported browsers (IE < 9, perhaps others?). + # http://greenbytes.de/tech/tc2231/#encoding-2231-fb + def escaped_filename + filename = @stored_file.filename.sanitized + ascii_filename = encode_ascii_filename(filename) + utf8_filename = encode_utf8_filename(filename) + "#{ascii_filename}; #{utf8_filename}" + end + + TRADITIONAL_PARAMETER_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ + + def encode_ascii_filename(filename) + # There is no reliable way to escape special or non-Latin characters + # in a traditionally quoted Content-Disposition filename parameter. + # Settle for transliterating to ASCII, then percent-escaping special + # characters, excluding spaces. + filename = I18n.transliterate(filename) + filename = percent_escape(filename, TRADITIONAL_PARAMETER_ESCAPED_CHAR) + %(filename="#{filename}") + end + + RFC5987_PARAMETER_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ + + def encode_utf8_filename(filename) + # RFC2231 filename parameters can simply be percent-escaped according + # to RFC5987. + filename = percent_escape(filename, RFC5987_PARAMETER_ESCAPED_CHAR) + %(filename*=UTF-8''#{filename}) + end + + def percent_escape(string, pattern) + string.gsub(pattern) do |char| + char.bytes.map { |byte| "%%%02X" % byte }.join("") + end + end +end diff --git a/lib/active_vault/filename.rb b/lib/active_vault/filename.rb new file mode 100644 index 0000000000..647d037b1f --- /dev/null +++ b/lib/active_vault/filename.rb @@ -0,0 +1,31 @@ +class ActiveVault::Filename + include Comparable + + def initialize(filename) + @filename = filename + end + + def extname + File.extname(@filename) + end + + def extension + extname.from(1) + end + + def base + File.basename(@filename, extname) + end + + def sanitized + @filename.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-") + end + + def to_s + sanitized.to_s + end + + def <=>(other) + to_s.downcase <=> other.to_s.downcase + end +end diff --git a/lib/active_vault/migration.rb b/lib/active_vault/migration.rb new file mode 100644 index 0000000000..cc7a535f39 --- /dev/null +++ b/lib/active_vault/migration.rb @@ -0,0 +1,15 @@ +class ActiveVault::CreateBlobs < ActiveRecord::Migration[5.1] + def change + create_table :rails_blobs do |t| + t.string :key + t.string :filename + t.string :content_type + t.text :metadata + t.integer :byte_size + t.string :checksum + t.time :created_at + + t.index [ :key ], unique: true + end + end +end diff --git a/lib/active_vault/purge_job.rb b/lib/active_vault/purge_job.rb new file mode 100644 index 0000000000..d7634af2bb --- /dev/null +++ b/lib/active_vault/purge_job.rb @@ -0,0 +1,7 @@ +class ActiveVault::PurgeJob < ActiveJob::Base + retry_on ActiveVault::StorageException + + def perform(blob) + blob.purge + end +end diff --git a/lib/active_vault/railtie.rb b/lib/active_vault/railtie.rb new file mode 100644 index 0000000000..c254f4c77c --- /dev/null +++ b/lib/active_vault/railtie.rb @@ -0,0 +1,19 @@ +require "rails/railtie" + +module ActiveVault + class Railtie < Rails::Railtie # :nodoc: + config.action_file = ActiveSupport::OrderedOptions.new + + config.eager_load_namespaces << ActiveVault + + initializer "action_file.routes" do + require "active_vault/disk_controller" + + config.after_initialize do |app| + app.routes.prepend do + get "/rails/blobs/:encoded_key" => "active_vault/disk#show", as: :rails_disk_blob + end + end + end + end +end diff --git a/lib/active_vault/site.rb b/lib/active_vault/site.rb new file mode 100644 index 0000000000..29eddf1566 --- /dev/null +++ b/lib/active_vault/site.rb @@ -0,0 +1,41 @@ +# Abstract class serving as an interface for concrete sites. +class ActiveVault::Site + def self.configure(site, **options) + begin + require "active_vault/site/#{site.to_s.downcase}_site" + ActiveVault::Site.const_get(:"#{site}Site").new(**options) + rescue LoadError => e + puts "Couldn't configure site: #{site} (#{e.message})" + end + end + + + def upload(key, io) + raise NotImplementedError + end + + def download(key) + raise NotImplementedError + end + + def delete(key) + raise NotImplementedError + end + + def exist?(key) + raise NotImplementedError + end + + + def url(key, expires_in:, disposition:, filename:) + raise NotImplementedError + end + + def bytesize(key) + raise NotImplementedError + end + + def checksum(key) + raise NotImplementedError + end +end diff --git a/lib/active_vault/site/disk_site.rb b/lib/active_vault/site/disk_site.rb new file mode 100644 index 0000000000..73f86bac6a --- /dev/null +++ b/lib/active_vault/site/disk_site.rb @@ -0,0 +1,71 @@ +require "fileutils" +require "pathname" + +class ActiveVault::Site::DiskSite < ActiveVault::Site + attr_reader :root + + def initialize(root:) + @root = root + end + + def upload(key, io) + File.open(make_path_for(key), "wb") do |file| + while chunk = io.read(65536) + file.write(chunk) + end + end + end + + def download(key) + if block_given? + File.open(path_for(key)) do |file| + while data = file.read(65536) + yield data + end + end + else + File.open path_for(key), &:read + end + end + + def delete(key) + File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted + end + + def exist?(key) + File.exist? path_for(key) + end + + + def url(key, expires_in:, disposition:, filename:) + verified_key_with_expiration = ActiveVault::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) + + if defined?(Rails) && defined?(Rails.application) + Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) + else + "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" + end + end + + def byte_size(key) + File.size path_for(key) + end + + def checksum(key) + Digest::MD5.file(path_for(key)).hexdigest + end + + + private + def path_for(key) + File.join root, folder_for(key), key + end + + def folder_for(key) + [ key[0..1], key[2..3] ].join("/") + end + + def make_path_for(key) + path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } + end +end diff --git a/lib/active_vault/site/gcs_site.rb b/lib/active_vault/site/gcs_site.rb new file mode 100644 index 0000000000..e509ebbbd2 --- /dev/null +++ b/lib/active_vault/site/gcs_site.rb @@ -0,0 +1,47 @@ +require "google/cloud/storage" + +class ActiveVault::Site::GCSSite < ActiveVault::Site + attr_reader :client, :bucket + + def initialize(project:, keyfile:, bucket:) + @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + bucket.create_file(io, key) + end + + def download(key) + io = file_for(key).download + io.rewind + io.read + end + + def delete(key) + file_for(key).try(:delete) + end + + def exist?(key) + file_for(key).present? + end + + + def byte_size(key) + file_for(key).size + end + + def checksum(key) + convert_to_hex base64: file_for(key).md5 + end + + + private + def file_for(key) + bucket.file(key) + end + + def convert_to_hex(base64:) + base64.unpack("m0").first.unpack("H*").first + end +end diff --git a/lib/active_vault/site/mirror_site.rb b/lib/active_vault/site/mirror_site.rb new file mode 100644 index 0000000000..67d79a2607 --- /dev/null +++ b/lib/active_vault/site/mirror_site.rb @@ -0,0 +1,44 @@ +class ActiveVault::Site::MirrorSite < ActiveVault::Site + attr_reader :sites + + def initialize(sites:) + @sites = sites + end + + def upload(key, io) + perform_across_sites :upload, key, io + end + + def download(key) + sites.detect { |site| site.exist?(key) }.download(key) + end + + def delete(key) + perform_across_sites :delete, key + end + + def exist?(key) + perform_across_sites(:exist?, key).any? + end + + + def byte_size(key) + primary_site.byte_size(key) + end + + def checksum(key) + primary_site.checksum(key) + end + + private + def primary_site + sites.first + end + + def perform_across_sites(method, **args) + # FIXME: Convert to be threaded + sites.collect do |site| + site.send method, **args + end + end +end diff --git a/lib/active_vault/site/s3_site.rb b/lib/active_vault/site/s3_site.rb new file mode 100644 index 0000000000..49a7522170 --- /dev/null +++ b/lib/active_vault/site/s3_site.rb @@ -0,0 +1,63 @@ +require "aws-sdk" + +class ActiveVault::Site::S3Site < ActiveVault::Site + attr_reader :client, :bucket + + def initialize(access_key_id:, secret_access_key:, region:, bucket:) + @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + object_for(key).put(body: io) + end + + def download(key) + if block_given? + stream(key, &block) + else + object_for(key).get.body.read + end + end + + def delete(key) + object_for(key).delete + end + + def exist?(key) + object_for(key).exists? + end + + + def url(key, expires_in:, disposition:, filename:) + object_for(key).presigned_url :get, expires_in: expires_in, + response_content_disposition: "#{disposition}; filename=\"#{filename}\"" + end + + def byte_size(key) + object_for(key).size + end + + def checksum(key) + object_for(key).etag.remove(/"/) + end + + + private + def object_for(key) + bucket.object(key) + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key, options = {}, &block) + object = object_for(key) + + chunk_size = 5242880 # 5 megabytes + offset = 0 + + while offset < object.content_length + yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) + offset += chunk_size + end + end +end diff --git a/lib/active_vault/verified_key_with_expiration.rb b/lib/active_vault/verified_key_with_expiration.rb new file mode 100644 index 0000000000..95d4993ff0 --- /dev/null +++ b/lib/active_vault/verified_key_with_expiration.rb @@ -0,0 +1,24 @@ +class ActiveVault::VerifiedKeyWithExpiration + class_attribute :verifier, default: defined?(Rails) ? Rails.application.message_verifier('ActiveVault') : nil + + class << self + def encode(key, expires_in: nil) + verifier.generate([ key, expires_at(expires_in) ]) + end + + def decode(encoded_key) + key, expires_at = verifier.verified(encoded_key) + + key if key && fresh?(expires_at) + end + + private + def expires_at(expires_in) + expires_in ? Time.now.utc.advance(seconds: expires_in) : nil + end + + def fresh?(expires_at) + expires_at.nil? || Time.now.utc < expires_at + end + end +end -- cgit v1.2.3