From 06ab7b27ea1c1ab357085439abacdb464f6742bf Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Thu, 6 Sep 2018 16:52:52 +0200 Subject: Prevent content type and disposition bypass in storage service URLs * Force content-type to binary on service urls for relevant content types We have a list of content types that must be forcibly served as binary, but in practice this only means to serve them as attachment always. We should also set the Content-Type to the configured binary type. As a bonus: add text/cache-manifest to the list of content types to be served as binary by default. * Store content-disposition and content-type in GCS Forcing these in the service_url when serving the file works fine for S3 and Azure, since these services include params in the signature. However, GCS specifically excludes response-content-disposition and response-content-type from the signature, which means an attacker can modify these and have files that should be served as text/plain attachments served as inline HTML for example. This makes our attempt to force specific files to be served as binary and as attachment can be easily bypassed. The only way this can be forced in GCS is by storing content-disposition and content-type in the object metadata. * Update GCS object metadata after identifying blob In some cases we create the blob and upload the data before identifying the content-type, which means we can't store that in GCS right when uploading. In these, after creating the attachment, we enqueue a job to identify the blob, and set the content-type. In other cases, files are uploaded to the storage service via direct upload link. We create the blob before the direct upload, which happens independently from the blob creation itself. We then mark the blob as identified, but we have already the content-type we need without having put it in the service. In these two cases, then, we need to update the metadata in the GCS service. * Include content-type and disposition in the verified key for disk service This prevents an attacker from modifying these params in the service signed URL, which is particularly important when we want to force them to have specific values for security reasons. * Allow only a list of specific content types to be served inline This is different from the content types that must be served as binary in the sense that any content type not in this list will be always served as attachment but with its original content type. Only types in this list are allowed to be served either inline or as attachment. Apart from forcing this in the service URL, for GCS we need to store the disposition in the metadata. Fix CVE-2018-16477. --- .../test/controllers/disk_controller_test.rb | 11 +++-- activestorage/test/models/blob_test.rb | 27 ++++++++---- activestorage/test/models/variant_test.rb | 2 +- activestorage/test/service/gcs_service_test.rb | 49 ++++++++++++++++++++++ 4 files changed, 78 insertions(+), 11 deletions(-) (limited to 'activestorage/test') diff --git a/activestorage/test/controllers/disk_controller_test.rb b/activestorage/test/controllers/disk_controller_test.rb index 7b5e989699..0aa8648db3 100644 --- a/activestorage/test/controllers/disk_controller_test.rb +++ b/activestorage/test/controllers/disk_controller_test.rb @@ -5,11 +5,12 @@ require "database/setup" class ActiveStorage::DiskControllerTest < ActionDispatch::IntegrationTest test "showing blob inline" do - blob = create_blob + blob = create_blob(filename: "hello.jpg", content_type: "image/jpg") + get blob.service_url assert_response :ok - assert_equal "inline; filename=\"hello.txt\"; filename*=UTF-8''hello.txt", response.headers["Content-Disposition"] - assert_equal "text/plain", response.headers["Content-Type"] + assert_equal "inline; filename=\"hello.jpg\"; filename*=UTF-8''hello.jpg", response.headers["Content-Disposition"] + assert_equal "image/jpg", response.headers["Content-Type"] assert_equal "Hello world!", response.body end @@ -36,6 +37,10 @@ class ActiveStorage::DiskControllerTest < ActionDispatch::IntegrationTest blob.delete get blob.service_url + end + + test "showing blob with invalid key" do + get rails_disk_service_url(encoded_key: "Invalid key", filename: "hello.txt") assert_response :not_found end diff --git a/activestorage/test/models/blob_test.rb b/activestorage/test/models/blob_test.rb index 1a6a89de56..0eac622d72 100644 --- a/activestorage/test/models/blob_test.rb +++ b/activestorage/test/models/blob_test.rb @@ -121,12 +121,21 @@ class ActiveStorage::BlobTest < ActiveSupport::TestCase end end - test "urls force attachment as content disposition for content types served as binary" do + test "urls force content_type to binary and attachment as content disposition for content types served as binary" do blob = create_blob(content_type: "text/html") freeze_time do - assert_equal expected_url_for(blob, disposition: :attachment), blob.service_url - assert_equal expected_url_for(blob, disposition: :attachment), blob.service_url(disposition: :inline) + assert_equal expected_url_for(blob, disposition: :attachment, content_type: "application/octet-stream"), blob.service_url + assert_equal expected_url_for(blob, disposition: :attachment, content_type: "application/octet-stream"), blob.service_url(disposition: :inline) + end + end + + test "urls force attachment as content disposition when the content type is not allowed inline" do + blob = create_blob(content_type: "application/zip") + + freeze_time do + assert_equal expected_url_for(blob, disposition: :attachment, content_type: "application/zip"), blob.service_url + assert_equal expected_url_for(blob, disposition: :attachment, content_type: "application/zip"), blob.service_url(disposition: :inline) end end @@ -148,7 +157,7 @@ class ActiveStorage::BlobTest < ActiveSupport::TestCase arguments = [ blob.key, expires_in: ActiveStorage.service_urls_expire_in, - disposition: :inline, + disposition: :attachment, content_type: blob.content_type, filename: blob.filename, thumb_size: "300x300", @@ -183,9 +192,13 @@ class ActiveStorage::BlobTest < ActiveSupport::TestCase end private - def expected_url_for(blob, disposition: :inline, filename: nil) + def expected_url_for(blob, disposition: :attachment, filename: nil, content_type: nil) filename ||= blob.filename - query_string = { content_type: blob.content_type, disposition: ActionDispatch::Http::ContentDisposition.format(disposition: disposition, filename: filename.sanitized) }.to_param - "https://example.com/rails/active_storage/disk/#{ActiveStorage.verifier.generate(blob.key, expires_in: 5.minutes, purpose: :blob_key)}/#{filename}?#{query_string}" + content_type ||= blob.content_type + + query = { disposition: disposition.to_s + "; #{filename.parameters}", content_type: content_type } + key_params = { key: blob.key }.merge(query) + + "https://example.com/rails/active_storage/disk/#{ActiveStorage.verifier.generate(key_params, expires_in: 5.minutes, purpose: :blob_key)}/#{filename}?#{query.to_param}" end end diff --git a/activestorage/test/models/variant_test.rb b/activestorage/test/models/variant_test.rb index 6577f1cd9f..8552080e7b 100644 --- a/activestorage/test/models/variant_test.rb +++ b/activestorage/test/models/variant_test.rb @@ -156,7 +156,7 @@ class ActiveStorage::VariantTest < ActiveSupport::TestCase test "service_url doesn't grow in length despite long variant options" do blob = create_file_blob(filename: "racecar.jpg") variant = blob.variant(font: "a" * 10_000).processed - assert_operator variant.service_url.length, :<, 525 + assert_operator variant.service_url.length, :<, 726 end test "works for vips processor" do diff --git a/activestorage/test/service/gcs_service_test.rb b/activestorage/test/service/gcs_service_test.rb index 2ba2f8b346..73337ef8e1 100644 --- a/activestorage/test/service/gcs_service_test.rb +++ b/activestorage/test/service/gcs_service_test.rb @@ -31,6 +31,55 @@ if SERVICE_CONFIGURATIONS[:gcs] end end + test "upload with content_type and content_disposition" do + begin + key = SecureRandom.base58(24) + data = "Something else entirely!" + + @service.upload(key, StringIO.new(data), checksum: Digest::MD5.base64digest(data), disposition: :attachment, filename: ActiveStorage::Filename.new("test.txt"), content_type: "text/plain") + + url = @service.url(key, expires_in: 2.minutes, disposition: :inline, content_type: "text/html", filename: ActiveStorage::Filename.new("test.html")) + response = Net::HTTP.get_response(URI(url)) + assert_equal "text/plain", response.content_type + assert_match /attachment;.*test.txt/, response["Content-Disposition"] + ensure + @service.delete key + end + end + + test "upload with content_type" do + begin + key = SecureRandom.base58(24) + data = "Something else entirely!" + + @service.upload(key, StringIO.new(data), checksum: Digest::MD5.base64digest(data), content_type: "text/plain") + + url = @service.url(key, expires_in: 2.minutes, disposition: :inline, content_type: "text/html", filename: ActiveStorage::Filename.new("test.html")) + response = Net::HTTP.get_response(URI(url)) + assert_equal "text/plain", response.content_type + assert_match /inline;.*test.html/, response["Content-Disposition"] + ensure + @service.delete key + end + end + + test "update metadata" do + begin + key = SecureRandom.base58(24) + data = "Something else entirely!" + @service.upload(key, StringIO.new(data), checksum: Digest::MD5.base64digest(data), disposition: :attachment, filename: ActiveStorage::Filename.new("test.html"), content_type: "text/html") + + @service.update_metadata(key, disposition: :inline, filename: ActiveStorage::Filename.new("test.txt"), content_type: "text/plain") + url = @service.url(key, expires_in: 2.minutes, disposition: :attachment, content_type: "text/html", filename: ActiveStorage::Filename.new("test.html")) + + response = Net::HTTP.get_response(URI(url)) + assert_equal "text/plain", response.content_type + assert_match /inline;.*test.txt/, response["Content-Disposition"] + ensure + @service.delete key + end + end + test "signed URL generation" do assert_match(/storage\.googleapis\.com\/.*response-content-disposition=inline.*test\.txt.*response-content-type=text%2Fplain/, @service.url(@key, expires_in: 2.minutes, disposition: :inline, filename: ActiveStorage::Filename.new("test.txt"), content_type: "text/plain")) -- cgit v1.2.3 From 7e7a60bd3fbfac9b02884f404bd31f3456f4022d Mon Sep 17 00:00:00 2001 From: "yuuji.yaginuma" Date: Wed, 28 Nov 2018 09:53:51 +0900 Subject: Remove duplicated test Since 06ab7b27ea1c1ab357085439abacdb464f6742bf, `GCSServiceTest#test_signed_URL_response_headers` is broken. https://travis-ci.org/rails/rails/jobs/460454477#L7084-L7087 This seems to be due to lack of `content_type` at upload. This is solved by specifying `conten_type`. However, since the same content is also tested with `test_upload_with_content_type`, it will be duplicated content, so I think that can remove `test_signed_URL_response_headers`. --- activestorage/test/service/gcs_service_test.rb | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'activestorage/test') diff --git a/activestorage/test/service/gcs_service_test.rb b/activestorage/test/service/gcs_service_test.rb index 73337ef8e1..6d0695ba58 100644 --- a/activestorage/test/service/gcs_service_test.rb +++ b/activestorage/test/service/gcs_service_test.rb @@ -84,20 +84,6 @@ if SERVICE_CONFIGURATIONS[:gcs] assert_match(/storage\.googleapis\.com\/.*response-content-disposition=inline.*test\.txt.*response-content-type=text%2Fplain/, @service.url(@key, expires_in: 2.minutes, disposition: :inline, filename: ActiveStorage::Filename.new("test.txt"), content_type: "text/plain")) end - - test "signed URL response headers" do - begin - key = SecureRandom.base58(24) - data = "Something else entirely!" - @service.upload(key, StringIO.new(data), checksum: Digest::MD5.base64digest(data)) - - url = @service.url(key, expires_in: 2.minutes, disposition: :inline, filename: ActiveStorage::Filename.new("test.txt"), content_type: "text/plain") - response = Net::HTTP.get_response(URI(url)) - assert_equal "text/plain", response.content_type - ensure - @service.delete key - end - end end else puts "Skipping GCS Service tests because no GCS configuration was supplied" -- cgit v1.2.3 From c2ef8bbf522dc69b3f3c270976bbb9779280e605 Mon Sep 17 00:00:00 2001 From: "yuuji.yaginuma" Date: Wed, 28 Nov 2018 10:16:07 +0900 Subject: Fix broken `ActiveStorage::BlobTest` `ActiveStorage::Filename#parameters` was removed by #33829. --- activestorage/test/models/blob_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'activestorage/test') diff --git a/activestorage/test/models/blob_test.rb b/activestorage/test/models/blob_test.rb index 0eac622d72..1503f5fc50 100644 --- a/activestorage/test/models/blob_test.rb +++ b/activestorage/test/models/blob_test.rb @@ -196,7 +196,7 @@ class ActiveStorage::BlobTest < ActiveSupport::TestCase filename ||= blob.filename content_type ||= blob.content_type - query = { disposition: disposition.to_s + "; #{filename.parameters}", content_type: content_type } + query = { disposition: ActionDispatch::Http::ContentDisposition.format(disposition: disposition, filename: filename.sanitized), content_type: content_type } key_params = { key: blob.key }.merge(query) "https://example.com/rails/active_storage/disk/#{ActiveStorage.verifier.generate(key_params, expires_in: 5.minutes, purpose: :blob_key)}/#{filename}?#{query.to_param}" -- cgit v1.2.3 From b7ec129de5120f8ea3b2b3562a8c38c69e508a07 Mon Sep 17 00:00:00 2001 From: "yuuji.yaginuma" Date: Wed, 28 Nov 2018 10:35:57 +0900 Subject: text is treated as `attachment` --- activestorage/test/controllers/disk_controller_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'activestorage/test') diff --git a/activestorage/test/controllers/disk_controller_test.rb b/activestorage/test/controllers/disk_controller_test.rb index 0aa8648db3..a723b4d56a 100644 --- a/activestorage/test/controllers/disk_controller_test.rb +++ b/activestorage/test/controllers/disk_controller_test.rb @@ -23,11 +23,11 @@ class ActiveStorage::DiskControllerTest < ActionDispatch::IntegrationTest assert_equal "Hello world!", response.body end - test "showing blob range inline" do + test "showing blob range" do blob = create_blob get blob.service_url, headers: { "Range" => "bytes=5-9" } assert_response :partial_content - assert_equal "inline; filename=\"hello.txt\"; filename*=UTF-8''hello.txt", response.headers["Content-Disposition"] + assert_equal "attachment; filename=\"hello.txt\"; filename*=UTF-8''hello.txt", response.headers["Content-Disposition"] assert_equal "text/plain", response.headers["Content-Type"] assert_equal " worl", response.body end -- cgit v1.2.3 From ac721c855203ac7570545c0e85fe086f8e94d94a Mon Sep 17 00:00:00 2001 From: "yuuji.yaginuma" Date: Wed, 28 Nov 2018 11:29:03 +0900 Subject: Fix "warning: ambiguous first argument; put parentheses or a space even after `/' operator" --- activestorage/test/service/gcs_service_test.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'activestorage/test') diff --git a/activestorage/test/service/gcs_service_test.rb b/activestorage/test/service/gcs_service_test.rb index 6d0695ba58..af27946357 100644 --- a/activestorage/test/service/gcs_service_test.rb +++ b/activestorage/test/service/gcs_service_test.rb @@ -41,7 +41,7 @@ if SERVICE_CONFIGURATIONS[:gcs] url = @service.url(key, expires_in: 2.minutes, disposition: :inline, content_type: "text/html", filename: ActiveStorage::Filename.new("test.html")) response = Net::HTTP.get_response(URI(url)) assert_equal "text/plain", response.content_type - assert_match /attachment;.*test.txt/, response["Content-Disposition"] + assert_match(/attachment;.*test.txt/, response["Content-Disposition"]) ensure @service.delete key end @@ -57,7 +57,7 @@ if SERVICE_CONFIGURATIONS[:gcs] url = @service.url(key, expires_in: 2.minutes, disposition: :inline, content_type: "text/html", filename: ActiveStorage::Filename.new("test.html")) response = Net::HTTP.get_response(URI(url)) assert_equal "text/plain", response.content_type - assert_match /inline;.*test.html/, response["Content-Disposition"] + assert_match(/inline;.*test.html/, response["Content-Disposition"]) ensure @service.delete key end @@ -74,7 +74,7 @@ if SERVICE_CONFIGURATIONS[:gcs] response = Net::HTTP.get_response(URI(url)) assert_equal "text/plain", response.content_type - assert_match /inline;.*test.txt/, response["Content-Disposition"] + assert_match(/inline;.*test.txt/, response["Content-Disposition"]) ensure @service.delete key end -- cgit v1.2.3 From 7bd21e8575d4d1f5a6315898657b49574de9eb36 Mon Sep 17 00:00:00 2001 From: Yannick Schutz Date: Fri, 30 Nov 2018 15:54:38 +0100 Subject: Add a test with extra keys to active_storage Service#upload --- activestorage/test/service/shared_service_tests.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'activestorage/test') diff --git a/activestorage/test/service/shared_service_tests.rb b/activestorage/test/service/shared_service_tests.rb index ca2490f2bc..26e72029c8 100644 --- a/activestorage/test/service/shared_service_tests.rb +++ b/activestorage/test/service/shared_service_tests.rb @@ -46,6 +46,25 @@ module ActiveStorage::Service::SharedServiceTests end end + test "uploading with integrity and multiple keys" do + begin + key = SecureRandom.base58(24) + data = "Something else entirely!" + @service.upload( + key, + StringIO.new(data), + checksum: Digest::MD5.base64digest(data), + filename: "racecar.jpg", + content_type: "image/jpg", + metadata: { metadata: true } + ) + + assert_equal data, @service.download(key) + ensure + @service.delete key + end + end + test "downloading" do assert_equal FIXTURE_DATA, @service.download(@key) end -- cgit v1.2.3 From 6ca2f2acb2427bf8bbad9c4fb5d0316fb3d19e85 Mon Sep 17 00:00:00 2001 From: "yuuji.yaginuma" Date: Sat, 1 Dec 2018 09:12:55 +0900 Subject: `metadata` is not passed to service Ref: https://github.com/rails/rails/blob/604fac6d7191fca102380b1a5f5eb9c619fb407f/activestorage/app/models/active_storage/blob.rb#L256-L264 This fixes broken `GCSServiceTest`. https://travis-ci.org/rails/rails/jobs/461868394#L6624-L6626 Follow up to #34576. --- activestorage/test/service/shared_service_tests.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'activestorage/test') diff --git a/activestorage/test/service/shared_service_tests.rb b/activestorage/test/service/shared_service_tests.rb index 26e72029c8..b97030b320 100644 --- a/activestorage/test/service/shared_service_tests.rb +++ b/activestorage/test/service/shared_service_tests.rb @@ -55,8 +55,7 @@ module ActiveStorage::Service::SharedServiceTests StringIO.new(data), checksum: Digest::MD5.base64digest(data), filename: "racecar.jpg", - content_type: "image/jpg", - metadata: { metadata: true } + content_type: "image/jpg" ) assert_equal data, @service.download(key) -- cgit v1.2.3 From ce40e851940ffb26d1dec1fdb0f7988bce34f569 Mon Sep 17 00:00:00 2001 From: Marcelo Perini Veloso Date: Sat, 1 Dec 2018 15:47:54 -0200 Subject: add require 'database/setup' in activestorage/test/service/s3_service_test.rb --- activestorage/test/service/s3_service_test.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'activestorage/test') diff --git a/activestorage/test/service/s3_service_test.rb b/activestorage/test/service/s3_service_test.rb index 559aa028f2..0e5b06dd78 100644 --- a/activestorage/test/service/s3_service_test.rb +++ b/activestorage/test/service/s3_service_test.rb @@ -2,6 +2,7 @@ require "service/shared_service_tests" require "net/http" +require "database/setup" if SERVICE_CONFIGURATIONS[:s3] class ActiveStorage::Service::S3ServiceTest < ActiveSupport::TestCase -- cgit v1.2.3