From 84b1feeaff94a598b335c5d3f73c4f74f489a165 Mon Sep 17 00:00:00 2001 From: Godfrey Chan Date: Wed, 11 Apr 2018 21:38:02 -0700 Subject: Add failing test for compression bug On Rails 5.2, when compression is enabled (which it is by default), the actual value being written to the underlying storage is actually _bigger_ than the uncompressed raw value. This is because the `@marshaled_value` instance variable (typically) gets serialized with the entry object, which is then written to the underlying storage, essentially double-storing every value (once uncompressed, once possibly compressed). This regression was introduced in #32254. --- .../test/cache/behaviors/cache_store_behavior.rb | 130 ++++++++++++++++++--- activesupport/test/cache/cache_entry_test.rb | 21 ---- .../test/cache/stores/memory_store_test.rb | 10 +- 3 files changed, 123 insertions(+), 38 deletions(-) (limited to 'activesupport') diff --git a/activesupport/test/cache/behaviors/cache_store_behavior.rb b/activesupport/test/cache/behaviors/cache_store_behavior.rb index efb57d34a2..84a3648ad0 100644 --- a/activesupport/test/cache/behaviors/cache_store_behavior.rb +++ b/activesupport/test/cache/behaviors/cache_store_behavior.rb @@ -141,29 +141,92 @@ module CacheStoreBehavior end end - def test_read_and_write_compressed_small_data - @cache.write("foo", "bar", compress: true) - assert_equal "bar", @cache.read("foo") + # Use strings that are guarenteed to compress well, so we can easily tell if + # the compression kicked in or not. + SMALL_STRING = "0" * 100 + LARGE_STRING = "0" * 2.kilobytes + + SMALL_OBJECT = { data: SMALL_STRING } + LARGE_OBJECT = { data: LARGE_STRING } + + def test_nil_with_default_compression_settings + assert_uncompressed(nil) end - def test_read_and_write_compressed_large_data - @cache.write("foo", "bar", compress: true, compress_threshold: 2) - assert_equal "bar", @cache.read("foo") + def test_nil_with_compress_true + assert_uncompressed(nil, compress: true) end - def test_read_and_write_compressed_nil - @cache.write("foo", nil, compress: true) - assert_nil @cache.read("foo") + def test_nil_with_compress_false + assert_uncompressed(nil, compress: false) end - def test_read_and_write_uncompressed_small_data - @cache.write("foo", "bar", compress: false) - assert_equal "bar", @cache.read("foo") + def test_nil_with_compress_low_compress_threshold + assert_uncompressed(nil, compress: true, compress_threshold: 2) end - def test_read_and_write_uncompressed_nil - @cache.write("foo", nil, compress: false) - assert_nil @cache.read("foo") + def test_small_string_with_default_compression_settings + assert_uncompressed(SMALL_STRING) + end + + def test_small_string_with_compress_true + assert_uncompressed(SMALL_STRING, compress: true) + end + + def test_small_string_with_compress_false + assert_uncompressed(SMALL_STRING, compress: false) + end + + def test_small_string_with_low_compress_threshold + assert_compressed(SMALL_STRING, compress: true, compress_threshold: 2) + end + + def test_small_object_with_default_compression_settings + assert_uncompressed(SMALL_OBJECT) + end + + def test_small_object_with_compress_true + assert_uncompressed(SMALL_OBJECT, compress: true) + end + + def test_small_object_with_compress_false + assert_uncompressed(SMALL_OBJECT, compress: false) + end + + def test_small_object_with_low_compress_threshold + assert_compressed(SMALL_OBJECT, compress: true, compress_threshold: 1) + end + + def test_large_string_with_default_compression_settings + assert_compressed(LARGE_STRING) + end + + def test_large_string_with_compress_true + assert_compressed(LARGE_STRING, compress: true) + end + + def test_large_string_with_compress_false + assert_uncompressed(LARGE_STRING, compress: false) + end + + def test_large_string_with_high_compress_threshold + assert_uncompressed(LARGE_STRING, compress: true, compress_threshold: 1.megabyte) + end + + def test_large_object_with_default_compression_settings + assert_compressed(LARGE_OBJECT) + end + + def test_large_object_with_compress_true + assert_compressed(LARGE_OBJECT, compress: true) + end + + def test_large_object_with_compress_false + assert_uncompressed(LARGE_OBJECT, compress: false) + end + + def test_large_object_with_high_compress_threshold + assert_uncompressed(LARGE_OBJECT, compress: true, compress_threshold: 1.megabyte) end def test_cache_key @@ -359,4 +422,41 @@ module CacheStoreBehavior ensure ActiveSupport::Notifications.unsubscribe "cache_read.active_support" end + + private + + def assert_compressed(value, **options) + assert_compression(true, value, **options) + end + + def assert_uncompressed(value, **options) + assert_compression(false, value, **options) + end + + def assert_compression(should_compress, value, **options) + freeze_time do + @cache.write("actual", value, **options) + @cache.write("uncompressed", value, **options, compress: false) + end + + if value.nil? + assert_nil @cache.read("actual") + assert_nil @cache.read("uncompressed") + else + assert_equal value, @cache.read("actual") + assert_equal value, @cache.read("uncompressed") + end + + actual_entry = @cache.send(:read_entry, @cache.send(:normalize_key, "actual", {}), {}) + uncompressed_entry = @cache.send(:read_entry, @cache.send(:normalize_key, "uncompressed", {}), {}) + + actual_size = Marshal.dump(actual_entry).bytesize + uncompressed_size = Marshal.dump(uncompressed_entry).bytesize + + if should_compress + assert_operator actual_size, :<, uncompressed_size, "value should be compressed" + else + assert_equal uncompressed_size, actual_size, "value should not be compressed" + end + end end diff --git a/activesupport/test/cache/cache_entry_test.rb b/activesupport/test/cache/cache_entry_test.rb index 80ff7ad564..d7baaa5c72 100644 --- a/activesupport/test/cache/cache_entry_test.rb +++ b/activesupport/test/cache/cache_entry_test.rb @@ -13,25 +13,4 @@ class CacheEntryTest < ActiveSupport::TestCase assert entry.expired?, "entry is expired" end end - - def test_compressed_values - value = "value" * 100 - entry = ActiveSupport::Cache::Entry.new(value, compress: true, compress_threshold: 1) - assert_equal value, entry.value - assert(value.bytesize > entry.size, "value is compressed") - end - - def test_compressed_by_default - value = "value" * 100 - entry = ActiveSupport::Cache::Entry.new(value, compress_threshold: 1) - assert_equal value, entry.value - assert(value.bytesize > entry.size, "value is compressed") - end - - def test_uncompressed_values - value = "value" * 100 - entry = ActiveSupport::Cache::Entry.new(value, compress: false) - assert_equal value, entry.value - assert_equal value.bytesize, entry.size - end end diff --git a/activesupport/test/cache/stores/memory_store_test.rb b/activesupport/test/cache/stores/memory_store_test.rb index 72fafc187b..8fe8384fb0 100644 --- a/activesupport/test/cache/stores/memory_store_test.rb +++ b/activesupport/test/cache/stores/memory_store_test.rb @@ -6,8 +6,7 @@ require_relative "../behaviors" class MemoryStoreTest < ActiveSupport::TestCase def setup - @record_size = ActiveSupport::Cache.lookup_store(:memory_store).send(:cached_size, 1, ActiveSupport::Cache::Entry.new("aaaaaaaaaa")) - @cache = ActiveSupport::Cache.lookup_store(:memory_store, expires_in: 60, size: @record_size * 10 + 1) + @cache = ActiveSupport::Cache.lookup_store(:memory_store, expires_in: 60) end include CacheStoreBehavior @@ -15,6 +14,13 @@ class MemoryStoreTest < ActiveSupport::TestCase include CacheDeleteMatchedBehavior include CacheIncrementDecrementBehavior include CacheInstrumentationBehavior +end + +class MemoryStorePruningTest < ActiveSupport::TestCase + def setup + @record_size = ActiveSupport::Cache.lookup_store(:memory_store).send(:cached_size, 1, ActiveSupport::Cache::Entry.new("aaaaaaaaaa")) + @cache = ActiveSupport::Cache.lookup_store(:memory_store, expires_in: 60, size: @record_size * 10 + 1) + end def test_prune_size @cache.write(1, "aaaaaaaaaa") && sleep(0.001) -- cgit v1.2.3 From 66df366268c4e5b17c9235b5a0c3aabcbd578e01 Mon Sep 17 00:00:00 2001 From: Godfrey Chan Date: Wed, 11 Apr 2018 22:54:20 -0700 Subject: Fix `ActiveSupport::Cache` compression (See previous commit for a description of the issue) --- activesupport/CHANGELOG.md | 7 +++ activesupport/lib/active_support/cache.rb | 67 +++++++++++----------- .../test/cache/behaviors/cache_store_behavior.rb | 23 +++++++- 3 files changed, 60 insertions(+), 37 deletions(-) (limited to 'activesupport') diff --git a/activesupport/CHANGELOG.md b/activesupport/CHANGELOG.md index e067c4dfba..82c985fae2 100644 --- a/activesupport/CHANGELOG.md +++ b/activesupport/CHANGELOG.md @@ -1,3 +1,10 @@ +* Fix bug where `ActiveSupport::Cache` will massively inflate the storage + size when compression is enabled (which is true by default). This patch + does not attempt to repair existing data: please manually flush the cache + to clear out the problematic entries. + + *Godfrey Chan* + * Fix bug where `URI.unscape` would fail with mixed Unicode/escaped character input: URI.unescape("\xe3\x83\x90") # => "バ" diff --git a/activesupport/lib/active_support/cache.rb b/activesupport/lib/active_support/cache.rb index 6967c164ab..d06a4971db 100644 --- a/activesupport/lib/active_support/cache.rb +++ b/activesupport/lib/active_support/cache.rb @@ -712,17 +712,14 @@ module ActiveSupport DEFAULT_COMPRESS_LIMIT = 1.kilobyte # Creates a new cache entry for the specified value. Options supported are - # +:compress+, +:compress_threshold+, and +:expires_in+. - def initialize(value, options = {}) - @value = value - if should_compress?(options) - compress! - end - - @version = options[:version] + # +:compress+, +:compress_threshold+, +:version+ and +:expires_in+. + def initialize(value, compress: true, compress_threshold: DEFAULT_COMPRESS_LIMIT, version: nil, expires_in: nil, **) + @value = value + @version = version @created_at = Time.now.to_f - @expires_in = options[:expires_in] - @expires_in = @expires_in.to_f if @expires_in + @expires_in = expires_in && expires_in.to_f + + compress!(compress_threshold) if compress end def value @@ -754,17 +751,13 @@ module ActiveSupport # Returns the size of the cached value. This could be less than # value.size if the data is compressed. def size - if defined?(@s) - @s + case value + when NilClass + 0 + when String + @value.bytesize else - case value - when NilClass - 0 - when String - @value.bytesize - else - @s = Marshal.dump(@value).bytesize - end + @s ||= Marshal.dump(@value).bytesize end end @@ -781,31 +774,35 @@ module ActiveSupport end private - def should_compress?(options) - if @value && options.fetch(:compress, true) - compress_threshold = options.fetch(:compress_threshold, DEFAULT_COMPRESS_LIMIT) - serialized_value_size = (@value.is_a?(String) ? @value : marshaled_value).bytesize + def compress!(compress_threshold) + case @value + when nil, true, false, Numeric + uncompressed_size = 0 + when String + uncompressed_size = @value.bytesize + else + serialized = Marshal.dump(@value) + uncompressed_size = serialized.bytesize + end + + if uncompressed_size >= compress_threshold + serialized ||= Marshal.dump(@value) + compressed = Zlib::Deflate.deflate(serialized) - serialized_value_size >= compress_threshold + if compressed.bytesize < uncompressed_size + @value = compressed + @compressed = true + end end end def compressed? - defined?(@compressed) ? @compressed : false - end - - def compress! - @value = Zlib::Deflate.deflate(marshaled_value) - @compressed = true + defined?(@compressed) end def uncompress(value) Marshal.load(Zlib::Inflate.inflate(value)) end - - def marshaled_value - @marshaled_value ||= Marshal.dump(@value) - end end end end diff --git a/activesupport/test/cache/behaviors/cache_store_behavior.rb b/activesupport/test/cache/behaviors/cache_store_behavior.rb index 84a3648ad0..0806665c27 100644 --- a/activesupport/test/cache/behaviors/cache_store_behavior.rb +++ b/activesupport/test/cache/behaviors/cache_store_behavior.rb @@ -162,7 +162,7 @@ module CacheStoreBehavior end def test_nil_with_compress_low_compress_threshold - assert_uncompressed(nil, compress: true, compress_threshold: 2) + assert_uncompressed(nil, compress: true, compress_threshold: 1) end def test_small_string_with_default_compression_settings @@ -178,7 +178,7 @@ module CacheStoreBehavior end def test_small_string_with_low_compress_threshold - assert_compressed(SMALL_STRING, compress: true, compress_threshold: 2) + assert_compressed(SMALL_STRING, compress: true, compress_threshold: 1) end def test_small_object_with_default_compression_settings @@ -229,6 +229,25 @@ module CacheStoreBehavior assert_uncompressed(LARGE_OBJECT, compress: true, compress_threshold: 1.megabyte) end + def test_incompressable_data + assert_uncompressed(nil, compress: true, compress_threshold: 1) + assert_uncompressed(true, compress: true, compress_threshold: 1) + assert_uncompressed(false, compress: true, compress_threshold: 1) + assert_uncompressed(0, compress: true, compress_threshold: 1) + assert_uncompressed(1.2345, compress: true, compress_threshold: 1) + assert_uncompressed("", compress: true, compress_threshold: 1) + + incompressible = nil + + # generate an incompressible string + loop do + incompressible = SecureRandom.bytes(1.kilobyte) + break if incompressible.bytesize < Zlib::Deflate.deflate(incompressible).bytesize + end + + assert_uncompressed(incompressible, compress: true, compress_threshold: 1) + end + def test_cache_key obj = Object.new def obj.cache_key -- cgit v1.2.3