diff options
Diffstat (limited to 'activerecord')
22 files changed, 278 insertions, 39 deletions
diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index 8d900d9669..786bef7359 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,16 @@ +* Skip query caching when working with batches of records (`find_each`, `find_in_batches`, + `in_batches`). + + Previously, records would be fetched in batches, but all records would be retained in memory + until the end of the request or job. + + *Eugene Kenny* + +* Prevent errors raised by `sql.active_record` notification subscribers from being converted into + `ActiveRecord::StatementInvalid` exceptions. + + *Dennis Taylor* + * Fix eager loading/preloading association with scope including joins. Fixes #28324. diff --git a/activerecord/lib/active_record/attribute_methods/dirty.rb b/activerecord/lib/active_record/attribute_methods/dirty.rb index 948249a6fd..8c58e63931 100644 --- a/activerecord/lib/active_record/attribute_methods/dirty.rb +++ b/activerecord/lib/active_record/attribute_methods/dirty.rb @@ -80,7 +80,7 @@ module ActiveRecord clear_mutation_trackers end - def raw_write_attribute(attr_name, *) + def write_attribute_without_type_cast(attr_name, *) result = super clear_attribute_change(attr_name) result diff --git a/activerecord/lib/active_record/attribute_methods/primary_key.rb b/activerecord/lib/active_record/attribute_methods/primary_key.rb index b9b2acff37..081aad434d 100644 --- a/activerecord/lib/active_record/attribute_methods/primary_key.rb +++ b/activerecord/lib/active_record/attribute_methods/primary_key.rb @@ -21,7 +21,7 @@ module ActiveRecord # Sets the primary key value. def id=(value) sync_with_transaction_state - write_attribute(self.class.primary_key, value) if self.class.primary_key + _write_attribute(self.class.primary_key, value) if self.class.primary_key end # Queries the primary key value. diff --git a/activerecord/lib/active_record/attribute_methods/write.rb b/activerecord/lib/active_record/attribute_methods/write.rb index 75c5a1a600..54b673c72e 100644 --- a/activerecord/lib/active_record/attribute_methods/write.rb +++ b/activerecord/lib/active_record/attribute_methods/write.rb @@ -17,7 +17,7 @@ module ActiveRecord generated_attribute_methods.module_eval <<-STR, __FILE__, __LINE__ + 1 def __temp__#{safe_name}=(value) name = ::ActiveRecord::AttributeMethods::AttrNames::ATTR_#{safe_name} - write_attribute(name, value) + _write_attribute(name, value) end alias_method #{(name + '=').inspect}, :__temp__#{safe_name}= undef_method :__temp__#{safe_name}= @@ -36,20 +36,26 @@ module ActiveRecord end name = self.class.primary_key if name == "id".freeze && self.class.primary_key - @attributes.write_from_user(name, value) - value + _write_attribute(name, value) end - def raw_write_attribute(attr_name, value) # :nodoc: - name = attr_name.to_s - @attributes.write_cast_value(name, value) + # This method exists to avoid the expensive primary_key check internally, without + # breaking compatibility with the write_attribute API + def _write_attribute(attr_name, value) # :nodoc: + @attributes.write_from_user(attr_name.to_s, value) value end private + def write_attribute_without_type_cast(attr_name, value) + name = attr_name.to_s + @attributes.write_cast_value(name, value) + value + end + # Handle *= for method_missing. def attribute=(attribute_name, value) - write_attribute(attribute_name, value) + _write_attribute(attribute_name, value) end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index cfe1892d78..30b29e7007 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -576,12 +576,14 @@ module ActiveRecord type_casted_binds: type_casted_binds, statement_name: statement_name, connection_id: object_id) do + begin @lock.synchronize do yield end + rescue => e + raise translate_exception_class(e, sql) end - rescue => e - raise translate_exception_class(e, sql) + end end def translate_exception(exception, message) diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb index eff96e329f..a46d9f8cbb 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb @@ -53,7 +53,7 @@ module ActiveRecord end def extract_expression_for_virtual_column(column) - if mariadb? + if mariadb? && version < "10.2.5" create_table_info = create_table_info(column.table_name) if %r/#{quote_column_name(column.name)} #{Regexp.quote(column.sql_type)}(?: COLLATE \w+)? AS \((?<expression>.+?)\) #{column.extra}/ =~ create_table_info $~[:expression].inspect diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb index a01fbba201..24f8ff6367 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb @@ -73,8 +73,8 @@ module ActiveRecord def new_column_from_field(table_name, field) type_metadata = fetch_type_metadata(field[:Type], field[:Extra]) - if type_metadata.type == :datetime && field[:Default] == "CURRENT_TIMESTAMP" - default, default_function = nil, field[:Default] + if type_metadata.type == :datetime && /\ACURRENT_TIMESTAMP(?:\(\))?\z/i.match?(field[:Default]) + default, default_function = nil, "CURRENT_TIMESTAMP" else default, default_function = field[:Default], nil end diff --git a/activerecord/lib/active_record/inheritance.rb b/activerecord/lib/active_record/inheritance.rb index 5776807507..1753322274 100644 --- a/activerecord/lib/active_record/inheritance.rb +++ b/activerecord/lib/active_record/inheritance.rb @@ -245,7 +245,7 @@ module ActiveRecord def ensure_proper_type klass = self.class if klass.finder_needs_type_condition? - write_attribute(klass.inheritance_column, klass.sti_name) + _write_attribute(klass.inheritance_column, klass.sti_name) end end end diff --git a/activerecord/lib/active_record/persistence.rb b/activerecord/lib/active_record/persistence.rb index b2dba5516e..a9509e562a 100644 --- a/activerecord/lib/active_record/persistence.rb +++ b/activerecord/lib/active_record/persistence.rb @@ -333,7 +333,7 @@ module ActiveRecord updated_count = self.class.unscoped.where(self.class.primary_key => id).update_all(attributes) attributes.each do |k, v| - raw_write_attribute(k, v) + write_attribute_without_type_cast(k, v) end updated_count == 1 diff --git a/activerecord/lib/active_record/relation.rb b/activerecord/lib/active_record/relation.rb index 52f5d5f3e3..76cf47a3ed 100644 --- a/activerecord/lib/active_record/relation.rb +++ b/activerecord/lib/active_record/relation.rb @@ -6,7 +6,7 @@ module ActiveRecord :extending, :unscope] SINGLE_VALUE_METHODS = [:limit, :offset, :lock, :readonly, :reordering, - :reverse_order, :distinct, :create_with] + :reverse_order, :distinct, :create_with, :skip_query_cache] CLAUSE_METHODS = [:where, :having, :from] INVALID_METHODS_FOR_DELETE_ALL = [:limit, :distinct, :offset, :group, :having] @@ -657,20 +657,32 @@ module ActiveRecord end def exec_queries(&block) - @records = eager_loading? ? find_with_associations.freeze : @klass.find_by_sql(arel, bound_attributes, &block).freeze - - preload = preload_values - preload += includes_values unless eager_loading? - preloader = nil - preload.each do |associations| - preloader ||= build_preloader - preloader.preload @records, associations - end + skip_query_cache_if_necessary do + @records = eager_loading? ? find_with_associations.freeze : @klass.find_by_sql(arel, bound_attributes, &block).freeze + + preload = preload_values + preload += includes_values unless eager_loading? + preloader = nil + preload.each do |associations| + preloader ||= build_preloader + preloader.preload @records, associations + end - @records.each(&:readonly!) if readonly_value + @records.each(&:readonly!) if readonly_value - @loaded = true - @records + @loaded = true + @records + end + end + + def skip_query_cache_if_necessary + if skip_query_cache_value + uncached do + yield + end + else + yield + end end def build_preloader diff --git a/activerecord/lib/active_record/relation/batches.rb b/activerecord/lib/active_record/relation/batches.rb index ee1f25ec84..c7e4f8a88a 100644 --- a/activerecord/lib/active_record/relation/batches.rb +++ b/activerecord/lib/active_record/relation/batches.rb @@ -209,6 +209,7 @@ module ActiveRecord relation = relation.reorder(batch_order).limit(batch_limit) relation = apply_limits(relation, start, finish) + relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching batch_relation = relation loop do diff --git a/activerecord/lib/active_record/relation/calculations.rb b/activerecord/lib/active_record/relation/calculations.rb index 8a54f8f2c3..aaba6c71f2 100644 --- a/activerecord/lib/active_record/relation/calculations.rb +++ b/activerecord/lib/active_record/relation/calculations.rb @@ -184,7 +184,7 @@ module ActiveRecord relation.select_values = column_names.map { |cn| @klass.has_attribute?(cn) || @klass.attribute_alias?(cn) ? arel_attribute(cn) : cn } - result = klass.connection.select_all(relation.arel, nil, bound_attributes) + result = skip_query_cache_if_necessary { klass.connection.select_all(relation.arel, nil, bound_attributes) } result.cast_values(klass.attribute_types) end end @@ -260,7 +260,7 @@ module ActiveRecord query_builder = relation.arel end - result = @klass.connection.select_all(query_builder, nil, bound_attributes) + result = skip_query_cache_if_necessary { @klass.connection.select_all(query_builder, nil, bound_attributes) } row = result.first value = row && row.values.first type = result.column_types.fetch(column_alias) do @@ -311,7 +311,7 @@ module ActiveRecord relation.group_values = group_fields relation.select_values = select_values - calculated_data = @klass.connection.select_all(relation.arel, nil, relation.bound_attributes) + calculated_data = skip_query_cache_if_necessary { @klass.connection.select_all(relation.arel, nil, relation.bound_attributes) } if association key_ids = calculated_data.collect { |row| row[group_aliases.first] } diff --git a/activerecord/lib/active_record/relation/finder_methods.rb b/activerecord/lib/active_record/relation/finder_methods.rb index eee0f36f63..ac0b4f597e 100644 --- a/activerecord/lib/active_record/relation/finder_methods.rb +++ b/activerecord/lib/active_record/relation/finder_methods.rb @@ -315,7 +315,7 @@ module ActiveRecord relation = construct_relation_for_exists(relation, conditions) - connection.select_value(relation.arel, "#{name} Exists", relation.bound_attributes) ? true : false + skip_query_cache_if_necessary { connection.select_value(relation.arel, "#{name} Exists", relation.bound_attributes) } ? true : false rescue ::RangeError false end @@ -376,7 +376,7 @@ module ActiveRecord if ActiveRecord::NullRelation === relation [] else - rows = connection.select_all(relation.arel, "SQL", relation.bound_attributes) + rows = skip_query_cache_if_necessary { connection.select_all(relation.arel, "SQL", relation.bound_attributes) } join_dependency.instantiate(rows, aliases) end end @@ -424,7 +424,7 @@ module ActiveRecord relation = relation.except(:select).select(values).distinct! - id_rows = @klass.connection.select_all(relation.arel, "SQL", relation.bound_attributes) + id_rows = skip_query_cache_if_necessary { @klass.connection.select_all(relation.arel, "SQL", relation.bound_attributes) } id_rows.map { |row| row[primary_key] } end diff --git a/activerecord/lib/active_record/relation/query_methods.rb b/activerecord/lib/active_record/relation/query_methods.rb index 9da8f96337..79495ead91 100644 --- a/activerecord/lib/active_record/relation/query_methods.rb +++ b/activerecord/lib/active_record/relation/query_methods.rb @@ -913,6 +913,11 @@ module ActiveRecord self end + def skip_query_cache! # :nodoc: + self.skip_query_cache_value = true + self + end + # Returns the Arel object associated with the relation. def arel # :nodoc: @arel ||= build_arel diff --git a/activerecord/lib/active_record/timestamp.rb b/activerecord/lib/active_record/timestamp.rb index dc4540eea6..26eea0bc24 100644 --- a/activerecord/lib/active_record/timestamp.rb +++ b/activerecord/lib/active_record/timestamp.rb @@ -86,7 +86,7 @@ module ActiveRecord all_timestamp_attributes_in_model.each do |column| if !attribute_present?(column) - write_attribute(column, current_time) + _write_attribute(column, current_time) end end end @@ -100,7 +100,7 @@ module ActiveRecord timestamp_attributes_for_update_in_model.each do |column| next if will_save_change_to_attribute?(column) - write_attribute(column, current_time) + _write_attribute(column, current_time) end end super(*args) diff --git a/activerecord/test/cases/adapter_test.rb b/activerecord/test/cases/adapter_test.rb index a1fb6427f9..827bcba121 100644 --- a/activerecord/test/cases/adapter_test.rb +++ b/activerecord/test/cases/adapter_test.rb @@ -211,6 +211,28 @@ module ActiveRecord end end + def test_exceptions_from_notifications_are_not_translated + original_error = StandardError.new("This StandardError shouldn't get translated") + subscriber = ActiveSupport::Notifications.subscribe("sql.active_record") { raise original_error } + actual_error = assert_raises(StandardError) do + @connection.execute("SELECT * FROM posts") + end + + assert_equal original_error, actual_error + + ensure + ActiveSupport::Notifications.unsubscribe(subscriber) if subscriber + end + + def test_database_related_exceptions_are_translated_to_statement_invalid + error = assert_raises(ActiveRecord::StatementInvalid) do + @connection.execute("This is a syntax error") + end + + assert_instance_of ActiveRecord::StatementInvalid, error + assert_kind_of Exception, error.cause + end + def test_select_all_always_return_activerecord_result result = @connection.select_all "SELECT * FROM posts" assert result.is_a?(ActiveRecord::Result) diff --git a/activerecord/test/cases/adapters/mysql2/virtual_column_test.rb b/activerecord/test/cases/adapters/mysql2/virtual_column_test.rb index 442a4fb7b5..1c5ef2aa41 100644 --- a/activerecord/test/cases/adapters/mysql2/virtual_column_test.rb +++ b/activerecord/test/cases/adapters/mysql2/virtual_column_test.rb @@ -52,7 +52,7 @@ if ActiveRecord::Base.connection.supports_virtual_columns? def test_schema_dumping output = dump_table_schema("virtual_columns") - assert_match(/t\.virtual\s+"upper_name",\s+type: :string,\s+as: "UPPER\(`name`\)"$/i, output) + assert_match(/t\.virtual\s+"upper_name",\s+type: :string,\s+as: "(?:UPPER|UCASE)\(`name`\)"$/i, output) assert_match(/t\.virtual\s+"name_length",\s+type: :integer,\s+as: "LENGTH\(`name`\)",\s+stored: true$/i, output) end end diff --git a/activerecord/test/cases/batches_test.rb b/activerecord/test/cases/batches_test.rb index 1a66b82b2e..dcd3af487b 100644 --- a/activerecord/test/cases/batches_test.rb +++ b/activerecord/test/cases/batches_test.rb @@ -1,4 +1,5 @@ require "cases/helper" +require "models/comment" require "models/post" require "models/subscriber" @@ -610,4 +611,64 @@ class EachTest < ActiveRecord::TestCase end assert_equal expected, actual end + + test ".find_each bypasses the query cache for its own queries" do + Post.cache do + assert_queries(2) do + Post.find_each {} + Post.find_each {} + end + end + end + + test ".find_each does not disable the query cache inside the given block" do + Post.cache do + Post.find_each(start: 1, finish: 1) do |post| + assert_queries(1) do + post.comments.count + post.comments.count + end + end + end + end + + test ".find_in_batches bypasses the query cache for its own queries" do + Post.cache do + assert_queries(2) do + Post.find_in_batches {} + Post.find_in_batches {} + end + end + end + + test ".find_in_batches does not disable the query cache inside the given block" do + Post.cache do + Post.find_in_batches(start: 1, finish: 1) do |batch| + assert_queries(1) do + batch.first.comments.count + batch.first.comments.count + end + end + end + end + + test ".in_batches bypasses the query cache for its own queries" do + Post.cache do + assert_queries(2) do + Post.in_batches {} + Post.in_batches {} + end + end + end + + test ".in_batches does not disable the query cache inside the given block" do + Post.cache do + Post.in_batches(start: 1, finish: 1) do |relation| + assert_queries(1) do + relation.count + relation.count + end + end + end + end end diff --git a/activerecord/test/cases/calculations_test.rb b/activerecord/test/cases/calculations_test.rb index 80baaac30a..7d6dc21e34 100644 --- a/activerecord/test/cases/calculations_test.rb +++ b/activerecord/test/cases/calculations_test.rb @@ -817,4 +817,46 @@ class CalculationsTest < ActiveRecord::TestCase assert_equal 6, Account.sum(:firm_id) { 1 } end end + + test "#skip_query_cache! for #pluck" do + Account.cache do + assert_queries(1) do + Account.pluck(:credit_limit) + Account.pluck(:credit_limit) + end + + assert_queries(2) do + Account.all.skip_query_cache!.pluck(:credit_limit) + Account.all.skip_query_cache!.pluck(:credit_limit) + end + end + end + + test "#skip_query_cache! for a simple calculation" do + Account.cache do + assert_queries(1) do + Account.calculate(:sum, :credit_limit) + Account.calculate(:sum, :credit_limit) + end + + assert_queries(2) do + Account.all.skip_query_cache!.calculate(:sum, :credit_limit) + Account.all.skip_query_cache!.calculate(:sum, :credit_limit) + end + end + end + + test "#skip_query_cache! for a grouped calculation" do + Account.cache do + assert_queries(1) do + Account.group(:firm_id).calculate(:sum, :credit_limit) + Account.group(:firm_id).calculate(:sum, :credit_limit) + end + + assert_queries(2) do + Account.all.skip_query_cache!.group(:firm_id).calculate(:sum, :credit_limit) + Account.all.skip_query_cache!.group(:firm_id).calculate(:sum, :credit_limit) + end + end + end end diff --git a/activerecord/test/cases/finder_test.rb b/activerecord/test/cases/finder_test.rb index 420f552ef6..af21cd529f 100644 --- a/activerecord/test/cases/finder_test.rb +++ b/activerecord/test/cases/finder_test.rb @@ -1232,6 +1232,34 @@ class FinderTest < ActiveRecord::TestCase assert_equal tyre2, zyke.tyres.custom_find_by(id: tyre2.id) end + test "#skip_query_cache! for #exists?" do + Topic.cache do + assert_queries(1) do + Topic.exists? + Topic.exists? + end + + assert_queries(2) do + Topic.all.skip_query_cache!.exists? + Topic.all.skip_query_cache!.exists? + end + end + end + + test "#skip_query_cache! for #exists? with a limited eager load" do + Topic.cache do + assert_queries(2) do + Topic.eager_load(:replies).limit(1).exists? + Topic.eager_load(:replies).limit(1).exists? + end + + assert_queries(4) do + Topic.eager_load(:replies).limit(1).skip_query_cache!.exists? + Topic.eager_load(:replies).limit(1).skip_query_cache!.exists? + end + end + end + private def table_with_custom_primary_key yield(Class.new(Toy) do diff --git a/activerecord/test/cases/relation/mutation_test.rb b/activerecord/test/cases/relation/mutation_test.rb index dea787c07f..8e73baa70a 100644 --- a/activerecord/test/cases/relation/mutation_test.rb +++ b/activerecord/test/cases/relation/mutation_test.rb @@ -90,7 +90,7 @@ module ActiveRecord assert_equal [], relation.extending_values end - (Relation::SINGLE_VALUE_METHODS - [:lock, :reordering, :reverse_order, :create_with]).each do |method| + (Relation::SINGLE_VALUE_METHODS - [:lock, :reordering, :reverse_order, :create_with, :skip_query_cache]).each do |method| test "##{method}!" do assert relation.public_send("#{method}!", :foo).equal?(relation) assert_equal :foo, relation.public_send("#{method}_value") @@ -162,5 +162,10 @@ module ActiveRecord relation.distinct! :foo assert_equal :foo, relation.distinct_value end + + test "skip_query_cache!" do + relation.skip_query_cache! + assert relation.skip_query_cache_value + end end end diff --git a/activerecord/test/cases/relations_test.rb b/activerecord/test/cases/relations_test.rb index 5767dec315..eb3449b331 100644 --- a/activerecord/test/cases/relations_test.rb +++ b/activerecord/test/cases/relations_test.rb @@ -2034,4 +2034,46 @@ class RelationTest < ActiveRecord::TestCase assert_equal 2, posts.to_a.length end + + test "#skip_query_cache!" do + Post.cache do + assert_queries(1) do + Post.all.load + Post.all.load + end + + assert_queries(2) do + Post.all.skip_query_cache!.load + Post.all.skip_query_cache!.load + end + end + end + + test "#skip_query_cache! with an eager load" do + Post.cache do + assert_queries(1) do + Post.eager_load(:comments).load + Post.eager_load(:comments).load + end + + assert_queries(2) do + Post.eager_load(:comments).skip_query_cache!.load + Post.eager_load(:comments).skip_query_cache!.load + end + end + end + + test "#skip_query_cache! with a preload" do + Post.cache do + assert_queries(2) do + Post.preload(:comments).load + Post.preload(:comments).load + end + + assert_queries(4) do + Post.preload(:comments).skip_query_cache!.load + Post.preload(:comments).skip_query_cache!.load + end + end + end end |