Move models and jobs to the app setup

Follow engine conventions more closely

12 files changed, 669 insertions, 0 deletions

diff --git a/app/models/active_storage/attachment.rb b/app/models/active_storage/attachment.rb
new file mode 100644
index 0000000000..20c619aa5a
--- /dev/null
+++ b/app/models/active_storage/attachment.rb
@@ -0,0 +1,30 @@
+require "active_storage/blob"
+require "global_id"
+require "active_support/core_ext/module/delegation"
+
+# Schema: id, record_gid, blob_id, created_at
+class ActiveStorage::Attachment < ActiveRecord::Base
+  self.table_name = "active_storage_attachments"
+
+  belongs_to :blob, class_name: "ActiveStorage::Blob"
+
+  delegate_missing_to :blob
+
+  def record
+    @record ||= GlobalID::Locator.locate(record_gid)
+  end
+
+  def record=(record)
+    @record = record
+    self.record_gid = record&.to_gid
+  end
+
+  def purge
+    blob.purge
+    destroy
+  end
+
+  def purge_later
+    ActiveStorage::PurgeJob.perform_later(self)
+  end
+end
diff --git a/app/models/active_storage/blob.rb b/app/models/active_storage/blob.rb
new file mode 100644
index 0000000000..6bd3941cd8
--- /dev/null
+++ b/app/models/active_storage/blob.rb
@@ -0,0 +1,95 @@
+require "active_storage/service"
+require "active_storage/filename"
+require "active_storage/purge_job"
+require "active_storage/variant"
+
+# Schema: id, key, filename, content_type, metadata, byte_size, checksum, created_at
+class ActiveStorage::Blob < ActiveRecord::Base
+  self.table_name = "active_storage_blobs"
+
+  has_secure_token :key
+  store :metadata, coder: JSON
+
+  class_attribute :service
+
+  class << self
+    def build_after_upload(io:, filename:, content_type: nil, metadata: nil)
+      new.tap do |blob|
+        blob.filename     = filename
+        blob.content_type = content_type
+        blob.metadata     = metadata
+
+        blob.upload io
+      end
+    end
+
+    def create_after_upload!(io:, filename:, content_type: nil, metadata: nil)
+      build_after_upload(io: io, filename: filename, content_type: content_type, metadata: metadata).tap(&:save!)
+    end
+
+    def create_before_direct_upload!(filename:, byte_size:, checksum:, content_type: nil, metadata: nil)
+      create! filename: filename, byte_size: byte_size, checksum: checksum, content_type: content_type, metadata: metadata
+    end
+  end
+
+
+  def key
+    # We can't wait until the record is first saved to have a key for it
+    self[:key] ||= self.class.generate_unique_secure_token
+  end
+
+  def filename
+    ActiveStorage::Filename.new(self[:filename])
+  end
+
+  def variant(transformations)
+    ActiveStorage::Variant.new(self, ActiveStorage::Variation.new(transformations))
+  end
+
+
+  def url(expires_in: 5.minutes, disposition: :inline)
+    service.url key, expires_in: expires_in, disposition: disposition, filename: filename
+  end
+
+  def url_for_direct_upload(expires_in: 5.minutes)
+    service.url_for_direct_upload key, expires_in: expires_in, content_type: content_type, content_length: byte_size
+  end
+
+
+  def upload(io)
+    self.checksum  = compute_checksum_in_chunks(io)
+    self.byte_size = io.size
+
+    service.upload(key, io, checksum: checksum)
+  end
+
+  def download(&block)
+    service.download key, &block
+  end
+
+
+  def delete
+    service.delete key
+  end
+
+  def purge
+    delete
+    destroy
+  end
+
+  def purge_later
+    ActiveStorage::PurgeJob.perform_later(self)
+  end
+
+
+  private
+    def compute_checksum_in_chunks(io)
+      Digest::MD5.new.tap do |checksum|
+        while chunk = io.read(5.megabytes)
+          checksum << chunk
+        end
+
+        io.rewind
+      end.base64digest
+    end
+end
diff --git a/app/models/active_storage/filename.rb b/app/models/active_storage/filename.rb
new file mode 100644
index 0000000000..71614b5113
--- /dev/null
+++ b/app/models/active_storage/filename.rb
@@ -0,0 +1,31 @@
+class ActiveStorage::Filename
+  include Comparable
+
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def extname
+    File.extname(@filename)
+  end
+
+  def extension
+    extname.from(1)
+  end
+
+  def base
+    File.basename(@filename, extname)
+  end
+
+  def sanitized
+    @filename.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-")
+  end
+
+  def to_s
+    sanitized.to_s
+  end
+
+  def <=>(other)
+    to_s.downcase <=> other.to_s.downcase
+  end
+end
diff --git a/app/models/active_storage/service.rb b/app/models/active_storage/service.rb
new file mode 100644
index 0000000000..745b1a615f
--- /dev/null
+++ b/app/models/active_storage/service.rb
@@ -0,0 +1,96 @@
+require "active_storage/log_subscriber"
+
+# Abstract class serving as an interface for concrete services.
+#
+# The available services are:
+#
+# * +Disk+, to manage attachments saved directly on the hard drive.
+# * +GCS+, to manage attachments through Google Cloud Storage.
+# * +S3+, to manage attachments through Amazon S3.
+# * +Mirror+, to be able to use several services to manage attachments.
+#
+# Inside a Rails application, you can set-up your services through the
+# generated <tt>config/storage_services.yml</tt> file and reference one
+# of the aforementioned constant under the +service+ key. For example:
+#
+#   local:
+#     service: Disk
+#     root: <%= Rails.root.join("storage") %>
+#
+# You can checkout the service's constructor to know which keys are required.
+#
+# Then, in your application's configuration, you can specify the service to
+# use like this:
+#
+#   config.active_storage.service = :local
+#
+# If you are using Active Storage outside of a Ruby on Rails application, you
+# can configure the service to use like this:
+#
+#   ActiveStorage::Blob.service = ActiveStorage::Service.configure(
+#     :Disk,
+#     root: Pathname("/foo/bar/storage")
+#   )
+class ActiveStorage::Service
+  class ActiveStorage::IntegrityError < StandardError; end
+
+  extend ActiveSupport::Autoload
+  autoload :Configurator
+
+  class_attribute :logger
+
+  class << self
+    # Configure an Active Storage service by name from a set of configurations,
+    # typically loaded from a YAML file. The Active Storage engine uses this
+    # to set the global Active Storage service when the app boots.
+    def configure(service_name, configurations)
+      Configurator.build(service_name, configurations)
+    end
+
+    # Override in subclasses that stitch together multiple services and hence
+    # need to build additional services using the configurator.
+    #
+    # Passes the configurator and all of the service's config as keyword args.
+    #
+    # See MirrorService for an example.
+    def build(configurator:, service: nil, **service_config) #:nodoc:
+      new(**service_config)
+    end
+  end
+
+  def upload(key, io, checksum: nil)
+    raise NotImplementedError
+  end
+
+  def download(key)
+    raise NotImplementedError
+  end
+
+  def delete(key)
+    raise NotImplementedError
+  end
+
+  def exist?(key)
+    raise NotImplementedError
+  end
+
+  def url(key, expires_in:, disposition:, filename:)
+    raise NotImplementedError
+  end
+
+  def url_for_direct_upload(key, expires_in:, content_type:, content_length:)
+    raise NotImplementedError
+  end
+
+  private
+    def instrument(operation, key, payload = {}, &block)
+      ActiveSupport::Notifications.instrument(
+        "service_#{operation}.active_storage",
+        payload.merge(key: key, service: service_name), &block)
+    end
+
+    def service_name
+      # ActiveStorage::Service::DiskService => Disk
+      self.class.name.split("::").third.remove("Service")
+    end
+end
diff --git a/app/models/active_storage/service/configurator.rb b/app/models/active_storage/service/configurator.rb
new file mode 100644
index 0000000000..00ae24d251
--- /dev/null
+++ b/app/models/active_storage/service/configurator.rb
@@ -0,0 +1,28 @@
+class ActiveStorage::Service::Configurator #:nodoc:
+  attr_reader :configurations
+
+  def self.build(service_name, configurations)
+    new(configurations).build(service_name)
+  end
+
+  def initialize(configurations)
+    @configurations = configurations.deep_symbolize_keys
+  end
+
+  def build(service_name)
+    config = config_for(service_name.to_sym)
+    resolve(config.fetch(:service)).build(**config, configurator: self)
+  end
+
+  private
+    def config_for(name)
+      configurations.fetch name do
+        raise "Missing configuration for the #{name.inspect} Active Storage service. Configurations available for #{configurations.keys.inspect}"
+      end
+    end
+
+    def resolve(class_name)
+      require "active_storage/service/#{class_name.to_s.downcase}_service"
+      ActiveStorage::Service.const_get(:"#{class_name}Service")
+    end
+end
diff --git a/app/models/active_storage/service/disk_service.rb b/app/models/active_storage/service/disk_service.rb
new file mode 100644
index 0000000000..a2a27528c1
--- /dev/null
+++ b/app/models/active_storage/service/disk_service.rb
@@ -0,0 +1,89 @@
+require "fileutils"
+require "pathname"
+require "digest/md5"
+require "active_support/core_ext/numeric/bytes"
+
+class ActiveStorage::Service::DiskService < ActiveStorage::Service
+  attr_reader :root
+
+  def initialize(root:)
+    @root = root
+  end
+
+  def upload(key, io, checksum: nil)
+    instrument :upload, key, checksum: checksum do
+      IO.copy_stream(io, make_path_for(key))
+      ensure_integrity_of(key, checksum) if checksum
+    end
+  end
+
+  def download(key)
+    if block_given?
+      instrument :streaming_download, key do
+        File.open(path_for(key), "rb") do |file|
+          while data = file.read(64.kilobytes)
+            yield data
+          end
+        end
+      end
+    else
+      instrument :download, key do
+        File.binread path_for(key)
+      end
+    end
+  end
+
+  def delete(key)
+    instrument :delete, key do
+      begin
+        File.delete path_for(key)
+      rescue Errno::ENOENT
+        # Ignore files already deleted
+      end
+    end
+  end
+
+  def exist?(key)
+    instrument :exist, key do |payload|
+      answer = File.exist? path_for(key)
+      payload[:exist] = answer
+      answer
+    end
+  end
+
+  def url(key, expires_in:, disposition:, filename:)
+    instrument :url, key do |payload|
+      verified_key_with_expiration = ActiveStorage::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in)
+
+      generated_url =
+        if defined?(Rails) && defined?(Rails.application)
+          Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition, filename: filename)
+        else
+          "/rails/active_storage/disk/#{verified_key_with_expiration}/#{filename}?disposition=#{disposition}"
+        end
+
+      payload[:url] = generated_url
+
+      generated_url
+    end
+  end
+
+  private
+    def path_for(key)
+      File.join root, folder_for(key), key
+    end
+
+    def folder_for(key)
+      [ key[0..1], key[2..3] ].join("/")
+    end
+
+    def make_path_for(key)
+      path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) }
+    end
+
+    def ensure_integrity_of(key, checksum)
+      unless Digest::MD5.file(path_for(key)).base64digest == checksum
+        raise ActiveStorage::IntegrityError
+      end
+    end
+end
diff --git a/app/models/active_storage/service/gcs_service.rb b/app/models/active_storage/service/gcs_service.rb
new file mode 100644
index 0000000000..7053a130c0
--- /dev/null
+++ b/app/models/active_storage/service/gcs_service.rb
@@ -0,0 +1,71 @@
+require "google/cloud/storage"
+require "active_support/core_ext/object/to_query"
+
+class ActiveStorage::Service::GCSService < ActiveStorage::Service
+  attr_reader :client, :bucket
+
+  def initialize(project:, keyfile:, bucket:)
+    @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile)
+    @bucket = @client.bucket(bucket)
+  end
+
+  def upload(key, io, checksum: nil)
+    instrument :upload, key, checksum: checksum do
+      begin
+        bucket.create_file(io, key, md5: checksum)
+      rescue Google::Cloud::InvalidArgumentError
+        raise ActiveStorage::IntegrityError
+      end
+    end
+  end
+
+  # FIXME: Add streaming when given a block
+  def download(key)
+    instrument :download, key do
+      io = file_for(key).download
+      io.rewind
+      io.read
+    end
+  end
+
+  def delete(key)
+    instrument :delete, key do
+      file_for(key)&.delete
+    end
+  end
+
+  def exist?(key)
+    instrument :exist, key do |payload|
+      answer = file_for(key).present?
+      payload[:exist] = answer
+      answer
+    end
+  end
+
+  def url(key, expires_in:, disposition:, filename:)
+    instrument :url, key do |payload|
+      query = { "response-content-disposition" => "#{disposition}; filename=\"#{filename}\"" }
+      generated_url = file_for(key).signed_url(expires: expires_in, query: query)
+
+      payload[:url] = generated_url
+
+      generated_url
+    end
+  end
+
+  def url_for_direct_upload(key, expires_in:, content_type:, content_length:)
+    instrument :url, key do |payload|
+      generated_url = bucket.signed_url key, method: "PUT", expires: expires_in,
+        content_type: content_type
+
+      payload[:url] = generated_url
+
+      generated_url
+    end
+  end
+
+  private
+    def file_for(key)
+      bucket.file(key)
+    end
+end
diff --git a/app/models/active_storage/service/mirror_service.rb b/app/models/active_storage/service/mirror_service.rb
new file mode 100644
index 0000000000..54465cad05
--- /dev/null
+++ b/app/models/active_storage/service/mirror_service.rb
@@ -0,0 +1,40 @@
+require "active_support/core_ext/module/delegation"
+
+class ActiveStorage::Service::MirrorService < ActiveStorage::Service
+  attr_reader :primary, :mirrors
+
+  delegate :download, :exist?, :url, to: :primary
+
+  # Stitch together from named services.
+  def self.build(primary:, mirrors:, configurator:, **options) #:nodoc:
+    new \
+      primary: configurator.build(primary),
+      mirrors: mirrors.collect { |name| configurator.build name }
+  end
+
+  def initialize(primary:, mirrors:)
+    @primary, @mirrors = primary, mirrors
+  end
+
+  def upload(key, io, checksum: nil)
+    each_service.collect do |service|
+      service.upload key, io.tap(&:rewind), checksum: checksum
+    end
+  end
+
+  def delete(key)
+    perform_across_services :delete, key
+  end
+
+  private
+    def each_service(&block)
+      [ primary, *mirrors ].each(&block)
+    end
+
+    def perform_across_services(method, *args)
+      # FIXME: Convert to be threaded
+      each_service.collect do |service|
+        service.public_send method, *args
+      end
+    end
+end
diff --git a/app/models/active_storage/service/s3_service.rb b/app/models/active_storage/service/s3_service.rb
new file mode 100644
index 0000000000..efffdec157
--- /dev/null
+++ b/app/models/active_storage/service/s3_service.rb
@@ -0,0 +1,89 @@
+require "aws-sdk"
+require "active_support/core_ext/numeric/bytes"
+
+class ActiveStorage::Service::S3Service < ActiveStorage::Service
+  attr_reader :client, :bucket, :upload_options
+
+  def initialize(access_key_id:, secret_access_key:, region:, bucket:, upload: {}, **options)
+    @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region, **options)
+    @bucket = @client.bucket(bucket)
+
+    @upload_options = upload
+  end
+
+  def upload(key, io, checksum: nil)
+    instrument :upload, key, checksum: checksum do
+      begin
+        object_for(key).put(upload_options.merge(body: io, content_md5: checksum))
+      rescue Aws::S3::Errors::BadDigest
+        raise ActiveStorage::IntegrityError
+      end
+    end
+  end
+
+  def download(key)
+    if block_given?
+      instrument :streaming_download, key do
+        stream(key, &block)
+      end
+    else
+      instrument :download, key do
+        object_for(key).get.body.read.force_encoding(Encoding::BINARY)
+      end
+    end
+  end
+
+  def delete(key)
+    instrument :delete, key do
+      object_for(key).delete
+    end
+  end
+
+  def exist?(key)
+    instrument :exist, key do |payload|
+      answer = object_for(key).exists?
+      payload[:exist] = answer
+      answer
+    end
+  end
+
+  def url(key, expires_in:, disposition:, filename:)
+    instrument :url, key do |payload|
+      generated_url = object_for(key).presigned_url :get, expires_in: expires_in,
+        response_content_disposition: "#{disposition}; filename=\"#{filename}\""
+
+      payload[:url] = generated_url
+
+      generated_url
+    end
+  end
+
+  def url_for_direct_upload(key, expires_in:, content_type:, content_length:)
+    instrument :url, key do |payload|
+      generated_url = object_for(key).presigned_url :put, expires_in: expires_in,
+        content_type: content_type, content_length: content_length
+
+      payload[:url] = generated_url
+
+      generated_url
+    end
+  end
+
+  private
+    def object_for(key)
+      bucket.object(key)
+    end
+
+    # Reads the object for the given key in chunks, yielding each to the block.
+    def stream(key, options = {}, &block)
+      object = object_for(key)
+
+      chunk_size = 5.megabytes
+      offset = 0
+
+      while offset < object.content_length
+        yield object.read(options.merge(range: "bytes=#{offset}-#{offset + chunk_size - 1}"))
+        offset += chunk_size
+      end
+    end
+end
diff --git a/app/models/active_storage/variant.rb b/app/models/active_storage/variant.rb
new file mode 100644
index 0000000000..435033f980
--- /dev/null
+++ b/app/models/active_storage/variant.rb
@@ -0,0 +1,35 @@
+require "active_storage/blob"
+require "mini_magick"
+
+# Image blobs can have variants that are the result of a set of transformations applied to the original.
+class ActiveStorage::Variant
+  attr_reader :blob, :variation
+  delegate :service, to: :blob
+
+  def initialize(blob, variation)
+    @blob, @variation = blob, variation
+  end
+
+  def processed
+    process unless service.exist?(key)
+    self
+  end
+
+  def key
+    "variants/#{blob.key}/#{variation.key}"
+  end
+
+  def url(expires_in: 5.minutes, disposition: :inline)
+    service.url key, expires_in: expires_in, disposition: disposition, filename: blob.filename
+  end
+
+
+  private
+    def process
+      service.upload key, transform(service.download(blob.key))
+    end
+
+    def transform(io)
+      File.open MiniMagick::Image.read(io).tap { |image| variation.transform(image) }.path
+    end
+end
diff --git a/app/models/active_storage/variation.rb b/app/models/active_storage/variation.rb
new file mode 100644
index 0000000000..f7c81bb99a
--- /dev/null
+++ b/app/models/active_storage/variation.rb
@@ -0,0 +1,41 @@
+require "active_support/core_ext/object/inclusion"
+
+# A set of transformations that can be applied to a blob to create a variant.
+class ActiveStorage::Variation
+  class_attribute :verifier
+
+  attr_reader :transformations
+
+  class << self
+    def decode(key)
+      new verifier.verify(key)
+    end
+
+    def encode(transformations)
+      verifier.generate(transformations)
+    end
+  end
+
+  def initialize(transformations)
+    @transformations = transformations
+  end
+
+  def transform(image)
+    transformations.each do |(method, argument)|
+      if eligible_argument?(argument)
+        image.public_send(method, argument)
+      else
+        image.public_send(method)
+      end
+    end
+  end
+
+  def key
+    self.class.encode(transformations)
+  end
+
+  private
+    def eligible_argument?(argument)
+      argument.present? && argument != true
+    end
+end
diff --git a/app/models/active_storage/verified_key_with_expiration.rb b/app/models/active_storage/verified_key_with_expiration.rb
new file mode 100644
index 0000000000..4a46483db5
--- /dev/null
+++ b/app/models/active_storage/verified_key_with_expiration.rb
@@ -0,0 +1,24 @@
+class ActiveStorage::VerifiedKeyWithExpiration
+  class_attribute :verifier
+
+  class << self
+    def encode(key, expires_in: nil)
+      verifier.generate([ key, expires_at(expires_in) ])
+    end
+
+    def decode(encoded_key)
+      key, expires_at = verifier.verified(encoded_key)
+
+      key if key && fresh?(expires_at)
+    end
+
+    private
+      def expires_at(expires_in)
+        expires_in ? Time.now.utc.advance(seconds: expires_in) : nil
+      end
+
+      def fresh?(expires_at)
+        expires_at.nil? || Time.now.utc < expires_at
+      end
+  end
+end