From 571509ad12bf3bcb3190efd7494a38c4796302b8 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Wed, 5 Jul 2017 13:06:29 +0200 Subject: Rename from ActiveFile to ActiveVault since activefile gem name was taken --- lib/active_vault/site/disk_site.rb | 71 ++++++++++++++++++++++++++++++++++++ lib/active_vault/site/gcs_site.rb | 47 ++++++++++++++++++++++++ lib/active_vault/site/mirror_site.rb | 44 ++++++++++++++++++++++ lib/active_vault/site/s3_site.rb | 63 ++++++++++++++++++++++++++++++++ 4 files changed, 225 insertions(+) create mode 100644 lib/active_vault/site/disk_site.rb create mode 100644 lib/active_vault/site/gcs_site.rb create mode 100644 lib/active_vault/site/mirror_site.rb create mode 100644 lib/active_vault/site/s3_site.rb (limited to 'lib/active_vault/site') diff --git a/lib/active_vault/site/disk_site.rb b/lib/active_vault/site/disk_site.rb new file mode 100644 index 0000000000..73f86bac6a --- /dev/null +++ b/lib/active_vault/site/disk_site.rb @@ -0,0 +1,71 @@ +require "fileutils" +require "pathname" + +class ActiveVault::Site::DiskSite < ActiveVault::Site + attr_reader :root + + def initialize(root:) + @root = root + end + + def upload(key, io) + File.open(make_path_for(key), "wb") do |file| + while chunk = io.read(65536) + file.write(chunk) + end + end + end + + def download(key) + if block_given? + File.open(path_for(key)) do |file| + while data = file.read(65536) + yield data + end + end + else + File.open path_for(key), &:read + end + end + + def delete(key) + File.delete path_for(key) rescue Errno::ENOENT # Ignore files already deleted + end + + def exist?(key) + File.exist? path_for(key) + end + + + def url(key, expires_in:, disposition:, filename:) + verified_key_with_expiration = ActiveVault::VerifiedKeyWithExpiration.encode(key, expires_in: expires_in) + + if defined?(Rails) && defined?(Rails.application) + Rails.application.routes.url_helpers.rails_disk_blob_path(verified_key_with_expiration, disposition: disposition) + else + "/rails/blobs/#{verified_key_with_expiration}?disposition=#{disposition}" + end + end + + def byte_size(key) + File.size path_for(key) + end + + def checksum(key) + Digest::MD5.file(path_for(key)).hexdigest + end + + + private + def path_for(key) + File.join root, folder_for(key), key + end + + def folder_for(key) + [ key[0..1], key[2..3] ].join("/") + end + + def make_path_for(key) + path_for(key).tap { |path| FileUtils.mkdir_p File.dirname(path) } + end +end diff --git a/lib/active_vault/site/gcs_site.rb b/lib/active_vault/site/gcs_site.rb new file mode 100644 index 0000000000..e509ebbbd2 --- /dev/null +++ b/lib/active_vault/site/gcs_site.rb @@ -0,0 +1,47 @@ +require "google/cloud/storage" + +class ActiveVault::Site::GCSSite < ActiveVault::Site + attr_reader :client, :bucket + + def initialize(project:, keyfile:, bucket:) + @client = Google::Cloud::Storage.new(project: project, keyfile: keyfile) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + bucket.create_file(io, key) + end + + def download(key) + io = file_for(key).download + io.rewind + io.read + end + + def delete(key) + file_for(key).try(:delete) + end + + def exist?(key) + file_for(key).present? + end + + + def byte_size(key) + file_for(key).size + end + + def checksum(key) + convert_to_hex base64: file_for(key).md5 + end + + + private + def file_for(key) + bucket.file(key) + end + + def convert_to_hex(base64:) + base64.unpack("m0").first.unpack("H*").first + end +end diff --git a/lib/active_vault/site/mirror_site.rb b/lib/active_vault/site/mirror_site.rb new file mode 100644 index 0000000000..67d79a2607 --- /dev/null +++ b/lib/active_vault/site/mirror_site.rb @@ -0,0 +1,44 @@ +class ActiveVault::Site::MirrorSite < ActiveVault::Site + attr_reader :sites + + def initialize(sites:) + @sites = sites + end + + def upload(key, io) + perform_across_sites :upload, key, io + end + + def download(key) + sites.detect { |site| site.exist?(key) }.download(key) + end + + def delete(key) + perform_across_sites :delete, key + end + + def exist?(key) + perform_across_sites(:exist?, key).any? + end + + + def byte_size(key) + primary_site.byte_size(key) + end + + def checksum(key) + primary_site.checksum(key) + end + + private + def primary_site + sites.first + end + + def perform_across_sites(method, **args) + # FIXME: Convert to be threaded + sites.collect do |site| + site.send method, **args + end + end +end diff --git a/lib/active_vault/site/s3_site.rb b/lib/active_vault/site/s3_site.rb new file mode 100644 index 0000000000..49a7522170 --- /dev/null +++ b/lib/active_vault/site/s3_site.rb @@ -0,0 +1,63 @@ +require "aws-sdk" + +class ActiveVault::Site::S3Site < ActiveVault::Site + attr_reader :client, :bucket + + def initialize(access_key_id:, secret_access_key:, region:, bucket:) + @client = Aws::S3::Resource.new(access_key_id: access_key_id, secret_access_key: secret_access_key, region: region) + @bucket = @client.bucket(bucket) + end + + def upload(key, io) + object_for(key).put(body: io) + end + + def download(key) + if block_given? + stream(key, &block) + else + object_for(key).get.body.read + end + end + + def delete(key) + object_for(key).delete + end + + def exist?(key) + object_for(key).exists? + end + + + def url(key, expires_in:, disposition:, filename:) + object_for(key).presigned_url :get, expires_in: expires_in, + response_content_disposition: "#{disposition}; filename=\"#{filename}\"" + end + + def byte_size(key) + object_for(key).size + end + + def checksum(key) + object_for(key).etag.remove(/"/) + end + + + private + def object_for(key) + bucket.object(key) + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key, options = {}, &block) + object = object_for(key) + + chunk_size = 5242880 # 5 megabytes + offset = 0 + + while offset < object.content_length + yield object.read(options.merge(:range => "bytes=#{offset}-#{offset + chunk_size - 1}")) + offset += chunk_size + end + end +end -- cgit v1.2.3