From 571509ad12bf3bcb3190efd7494a38c4796302b8 Mon Sep 17 00:00:00 2001
From: David Heinemeier Hansson <david@loudthinking.com>
Date: Wed, 5 Jul 2017 13:06:29 +0200
Subject: Rename from ActiveFile to ActiveVault since activefile gem name was
 taken

---
 lib/active_vault/blob.rb                         | 66 +++++++++++++++++
 lib/active_vault/config/sites.yml                | 25 +++++++
 lib/active_vault/disk_controller.rb              | 28 ++++++++
 lib/active_vault/download.rb                     | 90 ++++++++++++++++++++++++
 lib/active_vault/filename.rb                     | 31 ++++++++
 lib/active_vault/migration.rb                    | 15 ++++
 lib/active_vault/purge_job.rb                    |  7 ++
 lib/active_vault/railtie.rb                      | 19 +++++
 lib/active_vault/site.rb                         | 41 +++++++++++
 lib/active_vault/site/disk_site.rb               | 71 +++++++++++++++++++
 lib/active_vault/site/gcs_site.rb                | 47 +++++++++++++
 lib/active_vault/site/mirror_site.rb             | 44 ++++++++++++
 lib/active_vault/site/s3_site.rb                 | 63 +++++++++++++++++
 lib/active_vault/verified_key_with_expiration.rb | 24 +++++++
 14 files changed, 571 insertions(+)
 create mode 100644 lib/active_vault/blob.rb
 create mode 100644 lib/active_vault/config/sites.yml
 create mode 100644 lib/active_vault/disk_controller.rb
 create mode 100644 lib/active_vault/download.rb
 create mode 100644 lib/active_vault/filename.rb
 create mode 100644 lib/active_vault/migration.rb
 create mode 100644 lib/active_vault/purge_job.rb
 create mode 100644 lib/active_vault/railtie.rb
 create mode 100644 lib/active_vault/site.rb
 create mode 100644 lib/active_vault/site/disk_site.rb
 create mode 100644 lib/active_vault/site/gcs_site.rb
 create mode 100644 lib/active_vault/site/mirror_site.rb
 create mode 100644 lib/active_vault/site/s3_site.rb
 create mode 100644 lib/active_vault/verified_key_with_expiration.rb

(limited to 'lib/active_vault')

diff --git a/lib/active_vault/blob.rb b/lib/active_vault/blob.rb
new file mode 100644
index 0000000000..4948d43ec7
--- /dev/null
+++ b/lib/active_vault/blob.rb
@@ -0,0 +1,66 @@
+require "active_vault/site"
+require "active_vault/filename"
+
+# Schema: id, key, filename, content_type, metadata, byte_size, checksum, created_at
+class ActiveVault::Blob < ActiveRecord::Base
+  self.table_name = "rails_blobs"
+
+  has_secure_token :key
+  store :metadata, coder: JSON
+
+  class_attribute :site
+
+  class << self
+    def build_after_upload(io:, filename:, content_type: nil, metadata: nil)
+      new.tap do |blob|
+        blob.filename     = filename
+        blob.content_type = content_type
+
+        blob.upload io
+      end
+    end
+
+    def create_after_upload!(io:, filename:, content_type: nil, metadata: nil)
+      build_after_upload(io: io, filename: filename, content_type: content_type, metadata: metadata).tap(&:save!)
+    end
+  end
+
+  # We can't wait until the record is first saved to have a key for it
+  def key
+    self[:key] ||= self.class.generate_unique_secure_token
+  end
+
+  def filename
+    ActiveVault::Filename.new(self[:filename])
+  end
+
+  def url(expires_in: 5.minutes, disposition: :inline)
+    site.url key, expires_in: expires_in, disposition: disposition, filename: filename
+  end
+
+
+  def upload(io)
+    site.upload(key, io)
+
+    self.checksum  = site.checksum(key)
+    self.byte_size = site.byte_size(key)
+  end
+
+  def download
+    site.download key
+  end
+
+
+  def delete
+    site.delete key
+  end
+
+  def purge
+    delete
+    destroy
+  end
+
+  def purge_later
+    ActiveVault::PurgeJob.perform_later(self)
+  end
+end
diff --git a/lib/active_vault/config/sites.yml b/lib/active_vault/config/sites.yml
new file mode 100644
index 0000000000..334e779b28
--- /dev/null
+++ b/lib/active_vault/config/sites.yml
@@ -0,0 +1,25 @@
+# Configuration should be something like this:
+#
+# config/environments/development.rb
+#   config.active_vault.site = :local
+#
+# config/environments/production.rb
+#   config.active_vault.site = :amazon
+local:
+  site: Disk
+  root: <%%= File.join(Dir.tmpdir, "active_vault") %>
+
+amazon:
+  site: S3
+  access_key_id: <%%= Rails.application.secrets.aws[:access_key_id] %>
+  secret_access_key: <%%= Rails.application.secrets.aws[:secret_access_key] %>
+  region: us-east-1
+  bucket: <%= Rails.application.class.name.remove(/::Application$/).underscore %>
+
+google:
+  site: GCS
+
+mirror:
+  site: Mirror
+  primary: amazon
+  secondaries: google
diff --git a/lib/active_vault/disk_controller.rb b/lib/active_vault/disk_controller.rb
new file mode 100644
index 0000000000..623569f0f6
--- /dev/null
+++ b/lib/active_vault/disk_controller.rb
@@ -0,0 +1,28 @@
+require "action_controller"
+require "active_vault/blob"
+require "active_vault/verified_key_with_expiration"
+
+require "active_support/core_ext/object/inclusion"
+
+class ActiveVault::DiskController < ActionController::Base
+  def show
+    if key = decode_verified_key
+      blob = ActiveVault::Blob.find_by!(key: key)
+      
+      if stale?(etag: blob.checksum)
+        send_data blob.download, filename: blob.filename, type: blob.content_type, disposition: disposition_param
+      end
+    else
+      head :not_found
+    end
+  end
+
+  private
+    def decode_verified_key
+      ActiveVault::VerifiedKeyWithExpiration.decode(params[:encoded_key])
+    end
+
+    def disposition_param
+      params[:disposition].presence_in(%w( inline attachment )) || 'inline'
+    end
+end
diff --git a/lib/active_vault/download.rb b/lib/active_vault/download.rb
new file mode 100644
index 0000000000..6e74056062
--- /dev/null
+++ b/lib/active_vault/download.rb
@@ -0,0 +1,90 @@
+class ActiveVault::Download
+  # Sending .ai files as application/postscript to Safari opens them in a blank, grey screen.
+  # Downloading .ai as application/postscript files in Safari appends .ps to the extension.
+  # Sending HTML, SVG, XML and SWF files as binary closes XSS vulnerabilities.
+  # Sending JS files as binary avoids InvalidCrossOriginRequest without compromising security.
+  CONTENT_TYPES_TO_RENDER_AS_BINARY = %w(
+    text/html
+    text/javascript
+    image/svg+xml
+    application/postscript
+    application/x-shockwave-flash
+    text/xml
+    application/xml
+    application/xhtml+xml
+  )
+
+  BINARY_CONTENT_TYPE = 'application/octet-stream'
+
+  def initialize(stored_file)
+    @stored_file = stored_file
+  end
+
+  def headers(force_attachment: false)
+    {
+      x_accel_redirect:    '/reproxy',
+      x_reproxy_url:       reproxy_url,
+      content_type:        content_type,
+      content_disposition: content_disposition(force_attachment),
+      x_frame_options:     'SAMEORIGIN'
+    }
+  end
+
+  private
+    def reproxy_url
+      @stored_file.depot_location.paths.first
+    end
+
+    def content_type
+      if @stored_file.content_type.in? CONTENT_TYPES_TO_RENDER_AS_BINARY
+        BINARY_CONTENT_TYPE
+      else
+        @stored_file.content_type
+      end
+    end
+
+    def content_disposition(force_attachment = false)
+      if force_attachment || content_type == BINARY_CONTENT_TYPE
+        "attachment; #{escaped_filename}"
+      else
+        "inline; #{escaped_filename}"
+      end
+    end
+
+    # RFC2231 encoding for UTF-8 filenames, with an ASCII fallback
+    # first for unsupported browsers (IE < 9, perhaps others?).
+    # http://greenbytes.de/tech/tc2231/#encoding-2231-fb
+    def escaped_filename
+      filename = @stored_file.filename.sanitized
+      ascii_filename = encode_ascii_filename(filename)
+      utf8_filename = encode_utf8_filename(filename)
+      "#{ascii_filename}; #{utf8_filename}"
+    end
+
+    TRADITIONAL_PARAMETER_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/
+
+    def encode_ascii_filename(filename)
+      # There is no reliable way to escape special or non-Latin characters
+      # in a traditionally quoted Content-Disposition filename parameter.
+      # Settle for transliterating to ASCII, then percent-escaping special
+      # characters, excluding spaces.
+      filename = I18n.transliterate(filename)
+      filename = percent_escape(filename, TRADITIONAL_PARAMETER_ESCAPED_CHAR)
+      %(filename="#{filename}")
+    end
+
+    RFC5987_PARAMETER_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/
+
+    def encode_utf8_filename(filename)
+      # RFC2231 filename parameters can simply be percent-escaped according
+      # to RFC5987.
+      filename = percent_escape(filename, RFC5987_PARAMETER_ESCAPED_CHAR)
+      %(filename*=UTF-8''#{filename})
+    end
+
+    def percent_escape(string, pattern)
+      string.gsub(pattern) do |char|
+        char.bytes.map { |byte| "%%%02X" % byte }.join("")
+      end
+    end
+end
diff --git a/lib/active_vault/filename.rb b/lib/active_vault/filename.rb
new file mode 100644
index 0000000000..647d037b1f
--- /dev/null
+++ b/lib/active_vault/filename.rb
@@ -0,0 +1,31 @@
+class ActiveVault::Filename
+  include Comparable
+
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def extname
+    File.extname(@filename)
+  end
+
+  def extension
+    extname.from(1)
+  end
+
+  def base
+    File.basename(@filename, extname)
+  end
+
+  def sanitized
+    @filename.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-")
+  end
+
+  def to_s
+    sanitized.to_s
+  end
+
+  def <=>(other)
+    to_s.downcase <=> other.to_s.downcase
+  end
+end
diff --git a/lib/active_vault/migration.rb b/lib/active_vault/migration.rb
new file mode 100644
index 0000000000..cc7a535f39
--- /dev/null
+++ b/lib/active_vault/migration.rb
@@ -0,0 +1,15 @@
+class ActiveVault::CreateBlobs < ActiveRecord::Migration[5.1]
+  def change
+    create_table :rails_blobs do |t|
+      t.string :key
+      t.string :filename
+      t.string :content_type
+      t.text :metadata
+      t.integer :byte_size
+      t.string :checksum
+      t.time :created_at
+
+      t.index [ :key ], unique: true
+    end
+  end
+end
diff --git a/lib/active_vault/purge_job.rb b/lib/active_vault/purge_job.rb
new file mode 100644
index 0000000000..d7634af2bb
--- /dev/null
+++ b/lib/active_vault/purge_job.rb
@@ -0,0 +1,7 @@
+class ActiveVault::PurgeJob < ActiveJob::Base
+  retry_on ActiveVault::StorageException
+
+  def perform(blob)
+    blob.purge
+  end
+end
diff --git a/lib/active_vault/railtie.rb b/lib/active_vault/railtie.rb
new file mode 100644
index 0000000000..c254f4c77c
--- /dev/null
+++ b/lib/active_vault/railtie.rb
@@ -0,0 +1,19 @@
+require "rails/railtie"
+
+module ActiveVault
+  class Railtie < Rails::Railtie # :nodoc:
+    config.action_file = ActiveSupport::OrderedOptions.new
+
+    config.eager_load_namespaces << ActiveVault
+
+    initializer "action_file.routes" do
+      require "active_vault/disk_controller"
+
+      config.after_initialize do |app|
+        app.routes.prepend do
+          get "/rails/blobs/:encoded_key" => "active_vault/disk#show", as: :rails_disk_blob
+        end
+      end
+    end
+  end
+end
diff --git a/lib/active_vault/site.rb b/lib/active_vault/site.rb
new file mode 100644
index 0000000000..29eddf1566
--- /dev/null
+++ b/lib/active_vault/site.rb
@@ -0,0 +1,41 @@
+# Abstract class serving as an interface for concrete sites.
+class ActiveVault::Site
+  def self.configure(site, **options)
+    begin
+      require "active_vault/site/#{site.to_s.downcase}_site"
+      ActiveVault::Site.const_get(:"#{site}Site").new(**options)
+    rescue LoadError => e
+      puts "Couldn't configure site: #{site} (#{e.message})"
+    end
+  end
+
+
+  def upload(key, io)
+    raise NotImplementedError
+  end
+
+  def download(key)
+    raise NotImplementedError
+  end
+
+  def delete(key)
+    raise NotImplementedError
+  end
+
+  def exist?(key)
+    raise NotImplementedError
+  end
+
+
+  def url(key, expires_in:, disposition:, filename:)
+    raise NotImplementedError
+  end
+
+  def bytesize(key)
+    raise NotImplementedError
+  end
+
+  def checksum(key)
+    raise NotImplementedError
+  end
+end
diff --git a/lib/active_vault/site/disk_site.rb b/lib/active_vault/site/disk_site.rb
new file mode 100644
index 0000000000..73f86bac6a
--- /dev/null
+++ b/lib/active_vault/site/disk_site.rb
@@ -0,0 +1,71 @@
+require "fileutils"
+require "pathname"
+
+class ActiveVault::Site::DiskSite < ActiveVault::Site
+  attr_reader :root
+
+  def initialize(root:)
+    @root = root
+  end
+
+  def upload(key, io)
+    File.open(make_path_for(key), "wb") do |file|
+      while chunk = io.read(65536)
+        file.write(chunk)
+      end
+    end
+  end
+
+  def download(key)
+    if block_given?
+      File.open(path_for(key)) do |file|
+        while data = file.read(65536)
+          yield data
+        end
+      end
+    else
+      File.open path_for(key), &:read
+    end
+  end
+
+  def delete(key)
+    File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted
+  end
+
+  def exist?(key)
+    File.exist? path_for(key)
+  end
+
+
+  def url(key, expires_in:, disposition:, filename:)
+    verified_key_with_expiration = ActiveVault::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in)
+
+    if defined?(Rails) && defined?(Rails.application)
+      Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition)
+    else
+      "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}"
+    end
+  end
+
+  def byte_size(key)
+    File.size path_for(key)
+  end
+
+  def checksum(key)
+    Digest::MD5.file(path_for(key)).hexdigest
+  end
+
+
+  private
+    def path_for(key)
+      File.join root, folder_for(key), key
+    end
+
+    def folder_for(key)
+      [ key[0..1], key[2..3] ].join("/")
+    end
+
+    def make_path_for(key)
+      path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) }
+    end
+end
diff --git a/lib/active_vault/site/gcs_site.rb b/lib/active_vault/site/gcs_site.rb
new file mode 100644
index 0000000000..e509ebbbd2
--- /dev/null
+++ b/lib/active_vault/site/gcs_site.rb
@@ -0,0 +1,47 @@
+require "google/cloud/storage"
+
+class ActiveVault::Site::GCSSite < ActiveVault::Site
+  attr_reader :client, :bucket
+
+  def initialize(project:, keyfile:, bucket:)
+    @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile)
+    @bucket = @client.bucket(bucket)
+  end
+
+  def upload(key, io)
+    bucket.create_file(io, key)
+  end
+
+  def download(key)
+    io = file_for(key).download
+    io.rewind
+    io.read
+  end
+
+  def delete(key)
+    file_for(key).try(:delete)
+  end
+
+  def exist?(key)
+    file_for(key).present?
+  end
+
+
+  def byte_size(key)
+    file_for(key).size
+  end
+
+  def checksum(key)
+    convert_to_hex base64: file_for(key).md5
+  end
+
+
+  private
+    def file_for(key)
+      bucket.file(key)
+    end
+
+    def convert_to_hex(base64:)
+      base64.unpack("m0").first.unpack("H*").first
+    end
+end
diff --git a/lib/active_vault/site/mirror_site.rb b/lib/active_vault/site/mirror_site.rb
new file mode 100644
index 0000000000..67d79a2607
--- /dev/null
+++ b/lib/active_vault/site/mirror_site.rb
@@ -0,0 +1,44 @@
+class ActiveVault::Site::MirrorSite < ActiveVault::Site
+  attr_reader :sites
+
+  def initialize(sites:)
+    @sites = sites
+  end
+
+  def upload(key, io)
+    perform_across_sites :upload, key, io
+  end
+
+  def download(key)
+    sites.detect { |site| site.exist?(key) }.download(key)
+  end
+
+  def delete(key)
+    perform_across_sites :delete, key
+  end
+
+  def exist?(key)
+    perform_across_sites(:exist?, key).any?
+  end
+
+
+  def byte_size(key)
+    primary_site.byte_size(key)
+  end
+
+  def checksum(key)
+    primary_site.checksum(key)
+  end
+
+  private
+    def primary_site
+      sites.first
+    end
+
+    def perform_across_sites(method, **args)
+      # FIXME: Convert to be threaded
+      sites.collect do |site|
+        site.send method, **args
+      end   
+    end
+end
diff --git a/lib/active_vault/site/s3_site.rb b/lib/active_vault/site/s3_site.rb
new file mode 100644
index 0000000000..49a7522170
--- /dev/null
+++ b/lib/active_vault/site/s3_site.rb
@@ -0,0 +1,63 @@
+require "aws-sdk"
+
+class ActiveVault::Site::S3Site < ActiveVault::Site
+  attr_reader :client, :bucket
+
+  def initialize(access_key_id:, secret_access_key:, region:, bucket:)
+    @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region)
+    @bucket = @client.bucket(bucket)
+  end
+
+  def upload(key, io)
+    object_for(key).put(body: io)
+  end
+
+  def download(key)
+    if block_given?
+      stream(key, &block)
+    else
+      object_for(key).get.body.read
+    end
+  end
+
+  def delete(key)
+    object_for(key).delete
+  end
+
+  def exist?(key)
+    object_for(key).exists?
+  end
+
+
+  def url(key, expires_in:, disposition:, filename:)
+    object_for(key).presigned_url :get, expires_in: expires_in,
+      response_content_disposition: "#{disposition}; filename=\"#{filename}\""
+  end
+
+  def byte_size(key)
+    object_for(key).size
+  end
+
+  def checksum(key)
+    object_for(key).etag.remove(/"/)
+  end
+
+
+  private
+    def object_for(key)
+      bucket.object(key)
+    end
+
+    # Reads the object for the given key in chunks, yielding each to the block.
+    def stream(key, options = {}, &block)
+      object = object_for(key)
+
+      chunk_size = 5242880 # 5 megabytes
+      offset = 0
+
+      while offset < object.content_length
+        yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}"))
+        offset += chunk_size
+      end
+    end
+end
diff --git a/lib/active_vault/verified_key_with_expiration.rb b/lib/active_vault/verified_key_with_expiration.rb
new file mode 100644
index 0000000000..95d4993ff0
--- /dev/null
+++ b/lib/active_vault/verified_key_with_expiration.rb
@@ -0,0 +1,24 @@
+class ActiveVault::VerifiedKeyWithExpiration
+  class_attribute :verifier, default: defined?(Rails) ? Rails.application.message_verifier('ActiveVault') : nil
+
+  class << self
+    def encode(key, expires_in: nil)
+      verifier.generate([ key, expires_at(expires_in) ])
+    end
+
+    def decode(encoded_key)
+      key, expires_at = verifier.verified(encoded_key)
+
+      key if key && fresh?(expires_at)
+    end
+
+    private
+      def expires_at(expires_in)
+        expires_in ? Time.now.utc.advance(seconds: expires_in) : nil
+      end
+
+      def fresh?(expires_at)
+        expires_at.nil? || Time.now.utc < expires_at
+      end
+  end
+end
-- 
cgit v1.2.3