From 35d5bddabcd8f0eccc7de3ddf60431ea196508a1 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Thu, 6 Jul 2017 12:22:44 +0200 Subject: Rename from Site to Service now that we're called Active Storage --- README.md | 10 ++-- lib/active_storage.rb | 2 +- lib/active_storage/blob.rb | 16 +++---- lib/active_storage/config/sites.yml | 12 ++--- lib/active_storage/service.rb | 41 ++++++++++++++++ lib/active_storage/service/disk_service.rb | 71 ++++++++++++++++++++++++++++ lib/active_storage/service/gcs_service.rb | 53 +++++++++++++++++++++ lib/active_storage/service/mirror_service.rb | 51 ++++++++++++++++++++ lib/active_storage/service/s3_service.rb | 63 ++++++++++++++++++++++++ lib/active_storage/site.rb | 41 ---------------- lib/active_storage/site/disk_site.rb | 71 ---------------------------- lib/active_storage/site/gcs_site.rb | 53 --------------------- lib/active_storage/site/mirror_site.rb | 51 -------------------- lib/active_storage/site/s3_site.rb | 63 ------------------------ test/attachments_test.rb | 12 ++--- test/service/.gitignore | 1 + test/service/configurations-example.yml | 11 +++++ test/service/disk_service_test.rb | 8 ++++ test/service/gcs_service_test.rb | 20 ++++++++ test/service/mirror_service_test.rb | 30 ++++++++++++ test/service/s3_service_test.rb | 11 +++++ test/service/shared_service_tests.rb | 63 ++++++++++++++++++++++++ test/site/.gitignore | 1 - test/site/configurations-example.yml | 11 ----- test/site/disk_site_test.rb | 8 ---- test/site/gcs_site_test.rb | 20 -------- test/site/mirror_site_test.rb | 30 ------------ test/site/s3_site_test.rb | 11 ----- test/site/shared_site_tests.rb | 63 ------------------------ test/test_helper.rb | 4 +- 30 files changed, 451 insertions(+), 451 deletions(-) create mode 100644 lib/active_storage/service.rb create mode 100644 lib/active_storage/service/disk_service.rb create mode 100644 lib/active_storage/service/gcs_service.rb create mode 100644 lib/active_storage/service/mirror_service.rb create mode 100644 lib/active_storage/service/s3_service.rb delete mode 100644 lib/active_storage/site.rb delete mode 100644 lib/active_storage/site/disk_site.rb delete mode 100644 lib/active_storage/site/gcs_site.rb delete mode 100644 lib/active_storage/site/mirror_site.rb delete mode 100644 lib/active_storage/site/s3_site.rb create mode 100644 test/service/.gitignore create mode 100644 test/service/configurations-example.yml create mode 100644 test/service/disk_service_test.rb create mode 100644 test/service/gcs_service_test.rb create mode 100644 test/service/mirror_service_test.rb create mode 100644 test/service/s3_service_test.rb create mode 100644 test/service/shared_service_tests.rb delete mode 100644 test/site/.gitignore delete mode 100644 test/site/configurations-example.yml delete mode 100644 test/site/disk_site_test.rb delete mode 100644 test/site/gcs_site_test.rb delete mode 100644 test/site/mirror_site_test.rb delete mode 100644 test/site/s3_site_test.rb delete mode 100644 test/site/shared_site_tests.rb diff --git a/README.md b/README.md index 73602db219..b768520756 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Active Storage -Active Storage makes it simple to upload and reference files in cloud sites, like Amazon S3 or Google Cloud Storage, -and attach those files to Active Records. It also provides a disk site for testing or local deployments, but the +Active Storage makes it simple to upload and reference files in cloud services, like Amazon S3 or Google Cloud Storage, +and attach those files to Active Records. It also provides a disk service for testing or local deployments, but the focus is on cloud storage. ## Example @@ -55,7 +55,7 @@ end ## Configuration -Add `require "active_storage"` to config/application.rb and create a `config/initializers/active_storage_sites.rb` with the following: +Add `require "active_storage"` to config/application.rb and create a `config/initializers/active_storage_services.rb` with the following: ```ruby @@ -65,10 +65,10 @@ Add `require "active_storage"` to config/application.rb and create a `config/ini - Strip Download of its resposibilities and delete class - Proper logging -- Convert MirrorSite to use threading +- Convert MirrorService to use threading - Read metadata via Marcel? - Copy over migration to app via rake task -- Add Migrator to copy/move between sites +- Add Migrator to copy/move between services - Explore direct uploads to cloud - Extract VerifiedKeyWithExpiration into Rails as a feature of MessageVerifier diff --git a/lib/active_storage.rb b/lib/active_storage.rb index e87eb8a506..f72fe0d017 100644 --- a/lib/active_storage.rb +++ b/lib/active_storage.rb @@ -5,5 +5,5 @@ module ActiveStorage extend ActiveSupport::Autoload autoload :Blob - autoload :Site + autoload :Service end diff --git a/lib/active_storage/blob.rb b/lib/active_storage/blob.rb index edf57b5c78..4ce344e2a1 100644 --- a/lib/active_storage/blob.rb +++ b/lib/active_storage/blob.rb @@ -1,4 +1,4 @@ -require "active_storage/site" +require "active_storage/service" require "active_storage/filename" require "active_storage/purge_job" @@ -9,7 +9,7 @@ class ActiveStorage::Blob < ActiveRecord::Base has_secure_token :key store :metadata, coder: JSON - class_attribute :site + class_attribute :service class << self def build_after_upload(io:, filename:, content_type: nil, metadata: nil) @@ -37,24 +37,24 @@ class ActiveStorage::Blob < ActiveRecord::Base end def url(expires_in: 5.minutes, disposition: :inline) - site.url key, expires_in: expires_in, disposition: disposition, filename: filename + service.url key, expires_in: expires_in, disposition: disposition, filename: filename end def upload(io) - site.upload(key, io) + service.upload(key, io) - self.checksum = site.checksum(key) - self.byte_size = site.byte_size(key) + self.checksum = service.checksum(key) + self.byte_size = service.byte_size(key) end def download - site.download key + service.download key end def delete - site.delete key + service.delete key end def purge diff --git a/lib/active_storage/config/sites.yml b/lib/active_storage/config/sites.yml index 43bc77fbf9..317ef2b9b7 100644 --- a/lib/active_storage/config/sites.yml +++ b/lib/active_storage/config/sites.yml @@ -1,25 +1,25 @@ # Configuration should be something like this: # # config/environments/development.rb -# config.active_storage.site = :local +# config.active_storage.service = :local # # config/environments/production.rb -# config.active_storage.site = :amazon +# config.active_storage.service = :amazon local: - site: Disk + service: Disk root: <%%= File.join(Dir.tmpdir, "active_storage") %> amazon: - site: S3 + service: S3 access_key_id: <%%= Rails.application.secrets.aws[:access_key_id] %> secret_access_key: <%%= Rails.application.secrets.aws[:secret_access_key] %> region: us-east-1 bucket: <%= Rails.application.class.name.remove(/::Application$/).underscore %> google: - site: GCS + service: GCS mirror: - site: Mirror + service: Mirror primary: amazon secondaries: google diff --git a/lib/active_storage/service.rb b/lib/active_storage/service.rb new file mode 100644 index 0000000000..038b6ccb53 --- /dev/null +++ b/lib/active_storage/service.rb @@ -0,0 +1,41 @@ +# Abstract class serving as an interface for concrete services. +class ActiveStorage::Service + def self.configure(service, **options) + begin + require "active_storage/service/#{service.to_s.downcase}_service" + ActiveStorage::Service.const_get(:"#{service}Service").new(**options) + rescue LoadError => e + puts "Couldn't configure service: #{service} (#{e.message})" + end + end + + + def upload(key, io) + raise NotImplementedError + end + + def download(key) + raise NotImplementedError + end + + def delete(key) + raise NotImplementedError + end + + def exist?(key) + raise NotImplementedError + end + + + def url(key, expires_in:, disposition:, filename:) + raise NotImplementedError + end + + def bytesize(key) + raise NotImplementedError + end + + def checksum(key) + raise NotImplementedError + end +end diff --git a/lib/active_storage/service/disk_service.rb b/lib/active_storage/service/disk_service.rb new file mode 100644 index 0000000000..6977b5b82e --- /dev/null +++ b/lib/active_storage/service/disk_service.rb @@ -0,0 +1,71 @@ +require "fileutils" +require "pathname" + +class ActiveStorage::Service::DiskService < ActiveStorage::Service + attr_reader :root + + def initialize(root:) + @root = root + end + + def upload(key, io) + File.open(make_path_for(key), "wb") do |file| + while chunk = io.read(65536) + file.write(chunk) + end + end + end + + def download(key) + if block_given? + File.open(path_for(key)) do |file| + while data = file.read(65536) + yield data + end + end + else + File.open path_for(key), &:read + end + end + + def delete(key) + File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted + end + + def exist?(key) + File.exist? path_for(key) + end + + + def url(key, expires_in:, disposition:, filename:) + verified_key_with_expiration = ActiveStorage::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) + + if defined?(Rails) && defined?(Rails.application) + Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) + else + "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" + end + end + + def byte_size(key) + File.size path_for(key) + end + + def checksum(key) + Digest::MD5.file(path_for(key)).hexdigest + end + + + private + def path_for(key) + File.join root, folder_for(key), key + end + + def folder_for(key) + [ key[0..1], key[2..3] ].join("/") + end + + def make_path_for(key) + path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } + end +end diff --git a/lib/active_storage/service/gcs_service.rb b/lib/active_storage/service/gcs_service.rb new file mode 100644 index 0000000000..18ec1de133 --- /dev/null +++ b/lib/active_storage/service/gcs_service.rb @@ -0,0 +1,53 @@ +require "google/cloud/storage" +require "active_support/core_ext/object/to_query" + +class ActiveStorage::Service::GCSService < ActiveStorage::Service + attr_reader :client, :bucket + + def initialize(project:, keyfile:, bucket:) + @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + bucket.create_file(io, key) + end + + def download(key) + io = file_for(key).download + io.rewind + io.read + end + + def delete(key) + file_for(key).try(:delete) + end + + def exist?(key) + file_for(key).present? + end + + + def url(key, expires_in:, disposition:, filename:) + file_for(key).signed_url(expires: expires_in) + "&" + + { "response-content-disposition" => "#{disposition}; filename=\"#{filename}\"" }.to_query + end + + def byte_size(key) + file_for(key).size + end + + def checksum(key) + convert_to_hex base64: file_for(key).md5 + end + + + private + def file_for(key) + bucket.file(key) + end + + def convert_to_hex(base64:) + base64.unpack("m0").first.unpack("H*").first + end +end diff --git a/lib/active_storage/service/mirror_service.rb b/lib/active_storage/service/mirror_service.rb new file mode 100644 index 0000000000..2a3518e59e --- /dev/null +++ b/lib/active_storage/service/mirror_service.rb @@ -0,0 +1,51 @@ +class ActiveStorage::Service::MirrorService < ActiveStorage::Service + attr_reader :services + + def initialize(services:) + @services = services + end + + def upload(key, io) + services.collect do |service| + service.upload key, io + io.rewind + end + end + + def download(key) + services.detect { |service| service.exist?(key) }.download(key) + end + + def delete(key) + perform_across_services :delete, key + end + + def exist?(key) + perform_across_services(:exist?, key).any? + end + + + def url(key, **options) + primary_service.url(key, **options) + end + + def byte_size(key) + primary_service.byte_size(key) + end + + def checksum(key) + primary_service.checksum(key) + end + + private + def primary_service + services.first + end + + def perform_across_services(method, *args) + # FIXME: Convert to be threaded + services.collect do |service| + service.public_send method, *args + end + end +end diff --git a/lib/active_storage/service/s3_service.rb b/lib/active_storage/service/s3_service.rb new file mode 100644 index 0000000000..811321a172 --- /dev/null +++ b/lib/active_storage/service/s3_service.rb @@ -0,0 +1,63 @@ +require "aws-sdk" + +class ActiveStorage::Service::S3Service < ActiveStorage::Service + attr_reader :client, :bucket + + def initialize(access_key_id:, secret_access_key:, region:, bucket:) + @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + object_for(key).put(body: io) + end + + def download(key) + if block_given? + stream(key, &block) + else + object_for(key).get.body.read + end + end + + def delete(key) + object_for(key).delete + end + + def exist?(key) + object_for(key).exists? + end + + + def url(key, expires_in:, disposition:, filename:) + object_for(key).presigned_url :get, expires_in: expires_in, + response_content_disposition: "#{disposition}; filename=\"#{filename}\"" + end + + def byte_size(key) + object_for(key).size + end + + def checksum(key) + object_for(key).etag.remove(/"/) + end + + + private + def object_for(key) + bucket.object(key) + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key, options = {}, &block) + object = object_for(key) + + chunk_size = 5242880 # 5 megabytes + offset = 0 + + while offset < object.content_length + yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) + offset += chunk_size + end + end +end diff --git a/lib/active_storage/site.rb b/lib/active_storage/site.rb deleted file mode 100644 index b3b0221c63..0000000000 --- a/lib/active_storage/site.rb +++ /dev/null @@ -1,41 +0,0 @@ -# Abstract class serving as an interface for concrete sites. -class ActiveStorage::Site - def self.configure(site, **options) - begin - require "active_storage/site/#{site.to_s.downcase}_site" - ActiveStorage::Site.const_get(:"#{site}Site").new(**options) - rescue LoadError => e - puts "Couldn't configure site: #{site} (#{e.message})" - end - end - - - def upload(key, io) - raise NotImplementedError - end - - def download(key) - raise NotImplementedError - end - - def delete(key) - raise NotImplementedError - end - - def exist?(key) - raise NotImplementedError - end - - - def url(key, expires_in:, disposition:, filename:) - raise NotImplementedError - end - - def bytesize(key) - raise NotImplementedError - end - - def checksum(key) - raise NotImplementedError - end -end diff --git a/lib/active_storage/site/disk_site.rb b/lib/active_storage/site/disk_site.rb deleted file mode 100644 index 2ff0b22fae..0000000000 --- a/lib/active_storage/site/disk_site.rb +++ /dev/null @@ -1,71 +0,0 @@ -require "fileutils" -require "pathname" - -class ActiveStorage::Site::DiskSite < ActiveStorage::Site - attr_reader :root - - def initialize(root:) - @root = root - end - - def upload(key, io) - File.open(make_path_for(key), "wb") do |file| - while chunk = io.read(65536) - file.write(chunk) - end - end - end - - def download(key) - if block_given? - File.open(path_for(key)) do |file| - while data = file.read(65536) - yield data - end - end - else - File.open path_for(key), &:read - end - end - - def delete(key) - File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted - end - - def exist?(key) - File.exist? path_for(key) - end - - - def url(key, expires_in:, disposition:, filename:) - verified_key_with_expiration = ActiveStorage::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) - - if defined?(Rails) && defined?(Rails.application) - Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) - else - "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" - end - end - - def byte_size(key) - File.size path_for(key) - end - - def checksum(key) - Digest::MD5.file(path_for(key)).hexdigest - end - - - private - def path_for(key) - File.join root, folder_for(key), key - end - - def folder_for(key) - [ key[0..1], key[2..3] ].join("/") - end - - def make_path_for(key) - path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } - end -end diff --git a/lib/active_storage/site/gcs_site.rb b/lib/active_storage/site/gcs_site.rb deleted file mode 100644 index bf681ca6a2..0000000000 --- a/lib/active_storage/site/gcs_site.rb +++ /dev/null @@ -1,53 +0,0 @@ -require "google/cloud/storage" -require "active_support/core_ext/object/to_query" - -class ActiveStorage::Site::GCSSite < ActiveStorage::Site - attr_reader :client, :bucket - - def initialize(project:, keyfile:, bucket:) - @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) - @bucket = @client.bucket(bucket) - end - - def upload(key, io) - bucket.create_file(io, key) - end - - def download(key) - io = file_for(key).download - io.rewind - io.read - end - - def delete(key) - file_for(key).try(:delete) - end - - def exist?(key) - file_for(key).present? - end - - - def url(key, expires_in:, disposition:, filename:) - file_for(key).signed_url(expires: expires_in) + "&" + - { "response-content-disposition" => "#{disposition}; filename=\"#{filename}\"" }.to_query - end - - def byte_size(key) - file_for(key).size - end - - def checksum(key) - convert_to_hex base64: file_for(key).md5 - end - - - private - def file_for(key) - bucket.file(key) - end - - def convert_to_hex(base64:) - base64.unpack("m0").first.unpack("H*").first - end -end diff --git a/lib/active_storage/site/mirror_site.rb b/lib/active_storage/site/mirror_site.rb deleted file mode 100644 index ba3ef0ef0e..0000000000 --- a/lib/active_storage/site/mirror_site.rb +++ /dev/null @@ -1,51 +0,0 @@ -class ActiveStorage::Site::MirrorSite < ActiveStorage::Site - attr_reader :sites - - def initialize(sites:) - @sites = sites - end - - def upload(key, io) - sites.collect do |site| - site.upload key, io - io.rewind - end - end - - def download(key) - sites.detect { |site| site.exist?(key) }.download(key) - end - - def delete(key) - perform_across_sites :delete, key - end - - def exist?(key) - perform_across_sites(:exist?, key).any? - end - - - def url(key, **options) - primary_site.url(key, **options) - end - - def byte_size(key) - primary_site.byte_size(key) - end - - def checksum(key) - primary_site.checksum(key) - end - - private - def primary_site - sites.first - end - - def perform_across_sites(method, *args) - # FIXME: Convert to be threaded - sites.collect do |site| - site.public_send method, *args - end - end -end diff --git a/lib/active_storage/site/s3_site.rb b/lib/active_storage/site/s3_site.rb deleted file mode 100644 index 65dad37cfe..0000000000 --- a/lib/active_storage/site/s3_site.rb +++ /dev/null @@ -1,63 +0,0 @@ -require "aws-sdk" - -class ActiveStorage::Site::S3Site < ActiveStorage::Site - attr_reader :client, :bucket - - def initialize(access_key_id:, secret_access_key:, region:, bucket:) - @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) - @bucket = @client.bucket(bucket) - end - - def upload(key, io) - object_for(key).put(body: io) - end - - def download(key) - if block_given? - stream(key, &block) - else - object_for(key).get.body.read - end - end - - def delete(key) - object_for(key).delete - end - - def exist?(key) - object_for(key).exists? - end - - - def url(key, expires_in:, disposition:, filename:) - object_for(key).presigned_url :get, expires_in: expires_in, - response_content_disposition: "#{disposition}; filename=\"#{filename}\"" - end - - def byte_size(key) - object_for(key).size - end - - def checksum(key) - object_for(key).etag.remove(/"/) - end - - - private - def object_for(key) - bucket.object(key) - end - - # Reads the object for the given key in chunks, yielding each to the block. - def stream(key, options = {}, &block) - object = object_for(key) - - chunk_size = 5242880 # 5 megabytes - offset = 0 - - while offset < object.content_length - yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) - offset += chunk_size - end - end -end diff --git a/test/attachments_test.rb b/test/attachments_test.rb index 6e25002bb1..33bbff716d 100644 --- a/test/attachments_test.rb +++ b/test/attachments_test.rb @@ -36,7 +36,7 @@ class ActiveStorage::AttachmentsTest < ActiveSupport::TestCase @user.avatar.purge assert_not @user.avatar.attached? - assert_not ActiveStorage::Blob.site.exist?(avatar_key) + assert_not ActiveStorage::Blob.service.exist?(avatar_key) end test "purge attached blob later when the record is destroyed" do @@ -47,7 +47,7 @@ class ActiveStorage::AttachmentsTest < ActiveSupport::TestCase @user.destroy assert_nil ActiveStorage::Blob.find_by(key: avatar_key) - assert_not ActiveStorage::Blob.site.exist?(avatar_key) + assert_not ActiveStorage::Blob.service.exist?(avatar_key) end end @@ -74,8 +74,8 @@ class ActiveStorage::AttachmentsTest < ActiveSupport::TestCase @user.highlights.purge assert_not @user.highlights.attached? - assert_not ActiveStorage::Blob.site.exist?(highlight_keys.first) - assert_not ActiveStorage::Blob.site.exist?(highlight_keys.second) + assert_not ActiveStorage::Blob.service.exist?(highlight_keys.first) + assert_not ActiveStorage::Blob.service.exist?(highlight_keys.second) end test "purge attached blobs later when the record is destroyed" do @@ -86,10 +86,10 @@ class ActiveStorage::AttachmentsTest < ActiveSupport::TestCase @user.destroy assert_nil ActiveStorage::Blob.find_by(key: highlight_keys.first) - assert_not ActiveStorage::Blob.site.exist?(highlight_keys.first) + assert_not ActiveStorage::Blob.service.exist?(highlight_keys.first) assert_nil ActiveStorage::Blob.find_by(key: highlight_keys.second) - assert_not ActiveStorage::Blob.site.exist?(highlight_keys.second) + assert_not ActiveStorage::Blob.service.exist?(highlight_keys.second) end end end diff --git a/test/service/.gitignore b/test/service/.gitignore new file mode 100644 index 0000000000..c102131f3d --- /dev/null +++ b/test/service/.gitignore @@ -0,0 +1 @@ +configurations.yml diff --git a/test/service/configurations-example.yml b/test/service/configurations-example.yml new file mode 100644 index 0000000000..031197342a --- /dev/null +++ b/test/service/configurations-example.yml @@ -0,0 +1,11 @@ +# Copy this file to configurations.yml and edit the credentials to match your IAM test account and bucket +s3: + access_key_id: + secret_access_key: + region: + bucket: + +gcs: + project: + keyfile: + bucket: diff --git a/test/service/disk_service_test.rb b/test/service/disk_service_test.rb new file mode 100644 index 0000000000..5dd7cff303 --- /dev/null +++ b/test/service/disk_service_test.rb @@ -0,0 +1,8 @@ +require "tmpdir" +require "service/shared_service_tests" + +class ActiveStorage::Service::DiskServiceTest < ActiveSupport::TestCase + SERVICE = ActiveStorage::Service.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage")) + + include ActiveStorage::Service::SharedServiceTests +end diff --git a/test/service/gcs_service_test.rb b/test/service/gcs_service_test.rb new file mode 100644 index 0000000000..42f9cd3061 --- /dev/null +++ b/test/service/gcs_service_test.rb @@ -0,0 +1,20 @@ +require "service/shared_service_tests" + +if SERVICE_CONFIGURATIONS[:gcs] + class ActiveStorage::Service::GCSServiceTest < ActiveSupport::TestCase + SERVICE = ActiveStorage::Service.configure(:GCS, SERVICE_CONFIGURATIONS[:gcs]) + + include ActiveStorage::Service::SharedServiceTests + + test "signed URL generation" do + travel_to Time.now do + url = SERVICE.bucket.signed_url(FIXTURE_KEY, expires: 120) + + "&response-content-disposition=inline%3B+filename%3D%22test.txt%22" + + assert_equal url, @service.url(FIXTURE_KEY, expires_in: 2.minutes, disposition: :inline, filename: "test.txt") + end + end + end +else + puts "Skipping GCS Service tests because no GCS configuration was supplied" +end diff --git a/test/service/mirror_service_test.rb b/test/service/mirror_service_test.rb new file mode 100644 index 0000000000..3b22c4f049 --- /dev/null +++ b/test/service/mirror_service_test.rb @@ -0,0 +1,30 @@ +require "tmpdir" +require "service/shared_service_tests" + +class ActiveStorage::Service::MirrorServiceTest < ActiveSupport::TestCase + PRIMARY_DISK_SERVICE = ActiveStorage::Service.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage")) + SECONDARY_DISK_SERVICE = ActiveStorage::Service.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage_mirror")) + + SERVICE = ActiveStorage::Service.configure :Mirror, services: [ PRIMARY_DISK_SERVICE, SECONDARY_DISK_SERVICE ] + + include ActiveStorage::Service::SharedServiceTests + + test "uploading was done to all services" do + begin + key = SecureRandom.base58(24) + data = "Something else entirely!" + io = StringIO.new(data) + @service.upload(key, io) + + assert_equal data, PRIMARY_DISK_SERVICE.download(key) + assert_equal data, SECONDARY_DISK_SERVICE.download(key) + ensure + @service.delete key + end + end + + test "existing in all services" do + assert PRIMARY_DISK_SERVICE.exist?(FIXTURE_KEY) + assert SECONDARY_DISK_SERVICE.exist?(FIXTURE_KEY) + end +end diff --git a/test/service/s3_service_test.rb b/test/service/s3_service_test.rb new file mode 100644 index 0000000000..604dfd6c60 --- /dev/null +++ b/test/service/s3_service_test.rb @@ -0,0 +1,11 @@ +require "service/shared_service_tests" + +if SERVICE_CONFIGURATIONS[:s3] + class ActiveStorage::Service::S3ServiceTest < ActiveSupport::TestCase + SERVICE = ActiveStorage::Service.configure(:S3, SERVICE_CONFIGURATIONS[:s3]) + + include ActiveStorage::Service::SharedServiceTests + end +else + puts "Skipping S3 Service tests because no S3 configuration was supplied" +end diff --git a/test/service/shared_service_tests.rb b/test/service/shared_service_tests.rb new file mode 100644 index 0000000000..16672ab49b --- /dev/null +++ b/test/service/shared_service_tests.rb @@ -0,0 +1,63 @@ +require "test_helper" +require "active_support/core_ext/securerandom" +require "yaml" + +SERVICE_CONFIGURATIONS = begin + YAML.load_file(File.expand_path("../configurations.yml", __FILE__)).deep_symbolize_keys +rescue Errno::ENOENT + puts "Missing service configuration file in test/services/configurations.yml" +end + +module ActiveStorage::Service::SharedServiceTests + extend ActiveSupport::Concern + + FIXTURE_KEY = SecureRandom.base58(24) + FIXTURE_FILE = StringIO.new("Hello world!") + + included do + setup do + @service = self.class.const_get(:SERVICE) + @service.upload FIXTURE_KEY, FIXTURE_FILE + FIXTURE_FILE.rewind + end + + teardown do + @service.delete FIXTURE_KEY + FIXTURE_FILE.rewind + end + + test "uploading" do + begin + key = SecureRandom.base58(24) + data = "Something else entirely!" + @service.upload(key, StringIO.new(data)) + + assert_equal data, @service.download(key) + ensure + @service.delete key + end + end + + test "downloading" do + assert_equal FIXTURE_FILE.read, @service.download(FIXTURE_KEY) + end + + test "existing" do + assert @service.exist?(FIXTURE_KEY) + assert_not @service.exist?(FIXTURE_KEY + "nonsense") + end + + test "deleting" do + @service.delete FIXTURE_KEY + assert_not @service.exist?(FIXTURE_KEY) + end + + test "sizing" do + assert_equal FIXTURE_FILE.size, @service.byte_size(FIXTURE_KEY) + end + + test "checksumming" do + assert_equal Digest::MD5.hexdigest(FIXTURE_FILE.read), @service.checksum(FIXTURE_KEY) + end + end +end diff --git a/test/site/.gitignore b/test/site/.gitignore deleted file mode 100644 index c102131f3d..0000000000 --- a/test/site/.gitignore +++ /dev/null @@ -1 +0,0 @@ -configurations.yml diff --git a/test/site/configurations-example.yml b/test/site/configurations-example.yml deleted file mode 100644 index 031197342a..0000000000 --- a/test/site/configurations-example.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copy this file to configurations.yml and edit the credentials to match your IAM test account and bucket -s3: - access_key_id: - secret_access_key: - region: - bucket: - -gcs: - project: - keyfile: - bucket: diff --git a/test/site/disk_site_test.rb b/test/site/disk_site_test.rb deleted file mode 100644 index a04414ea68..0000000000 --- a/test/site/disk_site_test.rb +++ /dev/null @@ -1,8 +0,0 @@ -require "tmpdir" -require "site/shared_site_tests" - -class ActiveStorage::Site::DiskSiteTest < ActiveSupport::TestCase - SITE = ActiveStorage::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage")) - - include ActiveStorage::Site::SharedSiteTests -end diff --git a/test/site/gcs_site_test.rb b/test/site/gcs_site_test.rb deleted file mode 100644 index 98b1a3d767..0000000000 --- a/test/site/gcs_site_test.rb +++ /dev/null @@ -1,20 +0,0 @@ -require "site/shared_site_tests" - -if SITE_CONFIGURATIONS[:gcs] - class ActiveStorage::Site::GCSSiteTest < ActiveSupport::TestCase - SITE = ActiveStorage::Site.configure(:GCS, SITE_CONFIGURATIONS[:gcs]) - - include ActiveStorage::Site::SharedSiteTests - - test "signed URL generation" do - travel_to Time.now do - url = SITE.bucket.signed_url(FIXTURE_KEY, expires: 120) + - "&response-content-disposition=inline%3B+filename%3D%22test.txt%22" - - assert_equal url, @site.url(FIXTURE_KEY, expires_in: 2.minutes, disposition: :inline, filename: "test.txt") - end - end - end -else - puts "Skipping GCS Site tests because no GCS configuration was supplied" -end diff --git a/test/site/mirror_site_test.rb b/test/site/mirror_site_test.rb deleted file mode 100644 index 7ced377cde..0000000000 --- a/test/site/mirror_site_test.rb +++ /dev/null @@ -1,30 +0,0 @@ -require "tmpdir" -require "site/shared_site_tests" - -class ActiveStorage::Site::MirrorSiteTest < ActiveSupport::TestCase - PRIMARY_DISK_SITE = ActiveStorage::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage")) - SECONDARY_DISK_SITE = ActiveStorage::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage_mirror")) - - SITE = ActiveStorage::Site.configure :Mirror, sites: [ PRIMARY_DISK_SITE, SECONDARY_DISK_SITE ] - - include ActiveStorage::Site::SharedSiteTests - - test "uploading was done to all sites" do - begin - key = SecureRandom.base58(24) - data = "Something else entirely!" - io = StringIO.new(data) - @site.upload(key, io) - - assert_equal data, PRIMARY_DISK_SITE.download(key) - assert_equal data, SECONDARY_DISK_SITE.download(key) - ensure - @site.delete key - end - end - - test "existing in all sites" do - assert PRIMARY_DISK_SITE.exist?(FIXTURE_KEY) - assert SECONDARY_DISK_SITE.exist?(FIXTURE_KEY) - end -end diff --git a/test/site/s3_site_test.rb b/test/site/s3_site_test.rb deleted file mode 100644 index a9cb6ca618..0000000000 --- a/test/site/s3_site_test.rb +++ /dev/null @@ -1,11 +0,0 @@ -require "site/shared_site_tests" - -if SITE_CONFIGURATIONS[:s3] - class ActiveStorage::Site::S3SiteTest < ActiveSupport::TestCase - SITE = ActiveStorage::Site.configure(:S3, SITE_CONFIGURATIONS[:s3]) - - include ActiveStorage::Site::SharedSiteTests - end -else - puts "Skipping S3 Site tests because no S3 configuration was supplied" -end diff --git a/test/site/shared_site_tests.rb b/test/site/shared_site_tests.rb deleted file mode 100644 index 687c35e941..0000000000 --- a/test/site/shared_site_tests.rb +++ /dev/null @@ -1,63 +0,0 @@ -require "test_helper" -require "active_support/core_ext/securerandom" -require "yaml" - -SITE_CONFIGURATIONS = begin - YAML.load_file(File.expand_path("../configurations.yml", __FILE__)).deep_symbolize_keys -rescue Errno::ENOENT - puts "Missing site configuration file in test/sites/configurations.yml" -end - -module ActiveStorage::Site::SharedSiteTests - extend ActiveSupport::Concern - - FIXTURE_KEY = SecureRandom.base58(24) - FIXTURE_FILE = StringIO.new("Hello world!") - - included do - setup do - @site = self.class.const_get(:SITE) - @site.upload FIXTURE_KEY, FIXTURE_FILE - FIXTURE_FILE.rewind - end - - teardown do - @site.delete FIXTURE_KEY - FIXTURE_FILE.rewind - end - - test "uploading" do - begin - key = SecureRandom.base58(24) - data = "Something else entirely!" - @site.upload(key, StringIO.new(data)) - - assert_equal data, @site.download(key) - ensure - @site.delete key - end - end - - test "downloading" do - assert_equal FIXTURE_FILE.read, @site.download(FIXTURE_KEY) - end - - test "existing" do - assert @site.exist?(FIXTURE_KEY) - assert_not @site.exist?(FIXTURE_KEY + "nonsense") - end - - test "deleting" do - @site.delete FIXTURE_KEY - assert_not @site.exist?(FIXTURE_KEY) - end - - test "sizing" do - assert_equal FIXTURE_FILE.size, @site.byte_size(FIXTURE_KEY) - end - - test "checksumming" do - assert_equal Digest::MD5.hexdigest(FIXTURE_FILE.read), @site.checksum(FIXTURE_KEY) - end - end -end diff --git a/test/test_helper.rb b/test/test_helper.rb index f354a995e4..dcabe33c18 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -6,8 +6,8 @@ require "byebug" require "active_storage" -require "active_storage/site" -ActiveStorage::Blob.site = ActiveStorage::Site.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage")) +require "active_storage/service" +ActiveStorage::Blob.service = ActiveStorage::Service.configure(:Disk, root: File.join(Dir.tmpdir, "active_storage")) require "active_storage/verified_key_with_expiration" ActiveStorage::VerifiedKeyWithExpiration.verifier = ActiveSupport::MessageVerifier.new("Testing") -- cgit v1.2.3