diff options
Diffstat (limited to 'activerecord/lib/active_record')
18 files changed, 159 insertions, 102 deletions
diff --git a/activerecord/lib/active_record/associations/preloader/through_association.rb b/activerecord/lib/active_record/associations/preloader/through_association.rb index 6c83058202..b0203909ce 100644 --- a/activerecord/lib/active_record/associations/preloader/through_association.rb +++ b/activerecord/lib/active_record/associations/preloader/through_association.rb @@ -38,12 +38,7 @@ module ActiveRecord } end - record_offset = {} - @preloaded_records.each_with_index do |record,i| - record_offset[record] = i - end - - through_records.each_with_object({}) { |(lhs,center),records_by_owner| + through_records.each_with_object({}) do |(lhs,center), records_by_owner| pl_to_middle = center.group_by { |record| middle_to_pl[record] } records_by_owner[lhs] = pl_to_middle.flat_map do |pl, middles| @@ -53,13 +48,25 @@ module ActiveRecord target_records_from_association(association) }.compact - rhs_records.sort_by { |rhs| record_offset[rhs] } + # Respect the order on `reflection_scope` if it exists, else use the natural order. + if reflection_scope.values[:order].present? + @id_map ||= id_to_index_map @preloaded_records + rhs_records.sort_by { |rhs| @id_map[rhs] } + else + rhs_records + end end - } + end end private + def id_to_index_map(ids) + id_map = {} + ids.each_with_index { |id, index| id_map[id] = index } + id_map + end + def reset_association(owners, association_name) should_reset = (through_scope != through_reflection.klass.unscoped) || (reflection.options[:source_type] && through_reflection.collection?) diff --git a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb index ccd2899489..e389d818fd 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb @@ -951,24 +951,5 @@ module ActiveRecord owner_to_pool && owner_to_pool[owner.name] end end - - class ConnectionManagement - def initialize(app) - @app = app - end - - def call(env) - testing = env['rack.test'] - - status, headers, body = @app.call(env) - proxy = ::Rack::BodyProxy.new(body) do - ActiveRecord::Base.clear_active_connections! unless testing - end - [status, headers, proxy] - rescue Exception - ActiveRecord::Base.clear_active_connections! unless testing - raise - end - end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb b/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb index 33dbab41cb..0bdfd4f900 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb @@ -65,7 +65,7 @@ module ActiveRecord if @query_cache_enabled && !locked?(arel) arel, binds = binds_from_relation arel, binds sql = to_sql(arel, binds) - cache_sql(sql, binds) { super(sql, name, binds, preparable: visitor.preparable) } + cache_sql(sql, binds) { super(sql, name, binds, preparable: preparable) } else super end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb index b1b6044e72..2209874d0a 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb @@ -7,15 +7,16 @@ module ActiveRecord # Adapter level by over-writing this code inside the database specific adapters module ColumnDumper def column_spec(column) - spec = prepare_column_options(column) - (spec.keys - [:name, :type]).each{ |k| spec[k].insert(0, "#{k}: ")} + spec = Hash[prepare_column_options(column).map { |k, v| [k, "#{k}: #{v}"] }] + spec[:name] = column.name.inspect + spec[:type] = schema_type(column).to_s spec end def column_spec_for_primary_key(column) return if column.type == :integer spec = { id: schema_type(column).inspect } - spec.merge!(prepare_column_options(column).delete_if { |key, _| [:name, :type].include?(key) }) + spec.merge!(prepare_column_options(column)) end # This can be overridden on an Adapter level basis to support other @@ -23,9 +24,6 @@ module ActiveRecord # PostgreSQL::ColumnDumper) def prepare_column_options(column) spec = {} - spec[:name] = column.name.inspect - spec[:type] = schema_type(column).to_s - spec[:null] = 'false' unless column.null if limit = schema_limit(column) spec[:limit] = limit @@ -42,6 +40,8 @@ module ActiveRecord default = schema_default(column) if column.has_default? spec[:default] = default unless default.nil? + spec[:null] = 'false' unless column.null + if collation = schema_collation(column) spec[:collation] = collation end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb b/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb index 6ecdab6eb0..ca795cb1ad 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb @@ -188,7 +188,10 @@ module ActiveRecord transaction = begin_transaction options yield rescue Exception => error - rollback_transaction if transaction + if transaction + rollback_transaction + after_failure_actions(transaction, error) + end raise ensure unless error @@ -214,7 +217,16 @@ module ActiveRecord end private + NULL_TRANSACTION = NullTransaction.new + + # Deallocate invalidated prepared statements outside of the transaction + def after_failure_actions(transaction, error) + return unless transaction.is_a?(RealTransaction) + return unless error.is_a?(ActiveRecord::PreparedStatementCacheExpired) + @connection.clear_cache! + end + end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index 5ef434734a..fcc1ef9d5f 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -27,7 +27,6 @@ module ActiveRecord autoload_at 'active_record/connection_adapters/abstract/connection_pool' do autoload :ConnectionHandler - autoload :ConnectionManagement end autoload_under 'abstract' do diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb index ccf5b6cadc..3cdd46236b 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb @@ -13,7 +13,7 @@ module ActiveRecord return if spec.empty? else spec[:id] = schema_type(column).inspect - spec.merge!(prepare_column_options(column).delete_if { |key, _| [:name, :type, :null].include?(key) }) + spec.merge!(prepare_column_options(column).delete_if { |key, _| key == :null }) end spec end diff --git a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb index 57d8867bb4..ed8cfae657 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb @@ -131,7 +131,7 @@ module ActiveRecord def exec_query(sql, name = 'SQL', binds = [], prepare: false) result = execute(sql, name) @connection.next_result while @connection.more_results? - ActiveRecord::Result.new(result.fields, result.to_a) + ActiveRecord::Result.new(result.fields, result.to_a) if result end def exec_insert(sql, name, binds, pk = nil, sequence_name = nil) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb index b82bdb8b0c..19761618cf 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb @@ -12,7 +12,7 @@ module ActiveRecord spec[:default] = schema_default(column) || 'nil' else spec[:id] = schema_type(column).inspect - spec.merge!(prepare_column_options(column).delete_if { |key, _| [:name, :type, :null].include?(key) }) + spec.merge!(prepare_column_options(column).delete_if { |key, _| key == :null }) end spec end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index beaeef3c78..61c9628de3 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -598,25 +598,41 @@ module ActiveRecord @connection.exec_prepared(stmt_key, type_casted_binds) end rescue ActiveRecord::StatementInvalid => e - pgerror = e.cause + raise unless is_cached_plan_failure?(e) - # Get the PG code for the failure. Annoyingly, the code for - # prepared statements whose return value may have changed is - # FEATURE_NOT_SUPPORTED. Check here for more details: - # http://git.postgresql.org/gitweb/?p=postgresql.git;a=blob;f=src/backend/utils/cache/plancache.c#l573 - begin - code = pgerror.result.result_error_field(PGresult::PG_DIAG_SQLSTATE) - rescue - raise e - end - if FEATURE_NOT_SUPPORTED == code + # Nothing we can do if we are in a transaction because all commands + # will raise InFailedSQLTransaction + if in_transaction? + raise ActiveRecord::PreparedStatementCacheExpired.new(e.cause.message) + else + # outside of transactions we can simply flush this query and retry @statements.delete sql_key(sql) retry - else - raise e end end + # Annoyingly, the code for prepared statements whose return value may + # have changed is FEATURE_NOT_SUPPORTED. + # + # This covers various different error types so we need to do additional + # work to classify the exception definitively as a + # ActiveRecord::PreparedStatementCacheExpired + # + # Check here for more details: + # http://git.postgresql.org/gitweb/?p=postgresql.git;a=blob;f=src/backend/utils/cache/plancache.c#l573 + CACHED_PLAN_HEURISTIC = 'cached plan must not change result type'.freeze + def is_cached_plan_failure?(e) + pgerror = e.cause + code = pgerror.result.result_error_field(PGresult::PG_DIAG_SQLSTATE) + code == FEATURE_NOT_SUPPORTED && pgerror.message.include?(CACHED_PLAN_HEURISTIC) + rescue + false + end + + def in_transaction? + open_transactions > 0 + end + # Returns the statement identifier for the client side cache # of statements def sql_key(sql) diff --git a/activerecord/lib/active_record/core.rb b/activerecord/lib/active_record/core.rb index 24fd0aaecf..86ec8000fb 100644 --- a/activerecord/lib/active_record/core.rb +++ b/activerecord/lib/active_record/core.rb @@ -72,6 +72,14 @@ module ActiveRecord ## # :singleton-method: + # Specifies if an error should be raised on query limit or order being + # ignored when doing batch queries. Useful in applications where the + # limit or scope being ignored is error-worthy, rather than a warning. + mattr_accessor :error_on_ignored_order_or_limit, instance_writer: false + self.error_on_ignored_order_or_limit = false + + ## + # :singleton-method: # Specify whether or not to use timestamps for migration versions mattr_accessor :timestamped_migrations, instance_writer: false self.timestamped_migrations = true diff --git a/activerecord/lib/active_record/errors.rb b/activerecord/lib/active_record/errors.rb index 87f32c042c..2ec9bf3d67 100644 --- a/activerecord/lib/active_record/errors.rb +++ b/activerecord/lib/active_record/errors.rb @@ -139,6 +139,11 @@ module ActiveRecord class NoDatabaseError < StatementInvalid end + # Raised when Postgres returns 'cached plan must not change result type' and + # we cannot retry gracefully (e.g. inside a transaction) + class PreparedStatementCacheExpired < StatementInvalid + end + # Raised on attempt to save stale record. Record is stale when it's being saved in another query after # instantiation, for example, when two users edit the same wiki page and one starts editing and saves # the page before the other. diff --git a/activerecord/lib/active_record/log_subscriber.rb b/activerecord/lib/active_record/log_subscriber.rb index b63caa4473..efa2a4df02 100644 --- a/activerecord/lib/active_record/log_subscriber.rb +++ b/activerecord/lib/active_record/log_subscriber.rb @@ -67,7 +67,7 @@ module ActiveRecord case sql when /\A\s*rollback/mi RED - when /\s*.*?select .*for update/mi, /\A\s*lock/mi + when /select .*for update/mi, /\A\s*lock/mi WHITE when /\A\s*select/i BLUE diff --git a/activerecord/lib/active_record/query_cache.rb b/activerecord/lib/active_record/query_cache.rb index dcb2bd3d84..f451ed1764 100644 --- a/activerecord/lib/active_record/query_cache.rb +++ b/activerecord/lib/active_record/query_cache.rb @@ -23,34 +23,26 @@ module ActiveRecord end end - def initialize(app) - @app = app - end - - def call(env) - connection = ActiveRecord::Base.connection - enabled = connection.query_cache_enabled - connection_id = ActiveRecord::Base.connection_id - connection.enable_query_cache! - - response = @app.call(env) - response[2] = Rack::BodyProxy.new(response[2]) do - restore_query_cache_settings(connection_id, enabled) + def self.install_executor_hooks(executor = ActiveSupport::Executor) + executor.to_run do + connection = ActiveRecord::Base.connection + enabled = connection.query_cache_enabled + connection_id = ActiveRecord::Base.connection_id + connection.enable_query_cache! + + @restore_query_cache_settings = lambda do + ActiveRecord::Base.connection_id = connection_id + ActiveRecord::Base.connection.clear_query_cache + ActiveRecord::Base.connection.disable_query_cache! unless enabled + end end - response - rescue Exception => e - restore_query_cache_settings(connection_id, enabled) - raise e - end - - private + executor.to_complete do + @restore_query_cache_settings.call if defined?(@restore_query_cache_settings) - def restore_query_cache_settings(connection_id, enabled) - ActiveRecord::Base.connection_id = connection_id - ActiveRecord::Base.connection.clear_query_cache - ActiveRecord::Base.connection.disable_query_cache! unless enabled + # FIXME: This should be skipped when env['rack.test'] + ActiveRecord::Base.clear_active_connections! + end end - end end diff --git a/activerecord/lib/active_record/railtie.rb b/activerecord/lib/active_record/railtie.rb index f4200e96b7..4c074c93ed 100644 --- a/activerecord/lib/active_record/railtie.rb +++ b/activerecord/lib/active_record/railtie.rb @@ -16,12 +16,6 @@ module ActiveRecord config.app_generators.orm :active_record, :migration => true, :timestamps => true - config.app_middleware.insert_after ::ActionDispatch::Callbacks, - ActiveRecord::QueryCache - - config.app_middleware.insert_after ::ActionDispatch::Callbacks, - ActiveRecord::ConnectionAdapters::ConnectionManagement - config.action_dispatch.rescue_responses.merge!( 'ActiveRecord::RecordNotFound' => :not_found, 'ActiveRecord::StaleObjectError' => :conflict, @@ -153,11 +147,9 @@ end_warning end end - initializer "active_record.set_reloader_hooks" do |app| - hook = app.config.reload_classes_only_on_change ? :to_prepare : :to_cleanup - + initializer "active_record.set_reloader_hooks" do ActiveSupport.on_load(:active_record) do - ActionDispatch::Reloader.send(hook) do + ActiveSupport::Reloader.before_class_unload do if ActiveRecord::Base.connected? ActiveRecord::Base.clear_cache! ActiveRecord::Base.clear_reloadable_connections! @@ -166,6 +158,12 @@ end_warning end end + initializer "active_record.set_executor_hooks" do + ActiveSupport.on_load(:active_record) do + ActiveRecord::QueryCache.install_executor_hooks + end + end + initializer "active_record.add_watchable_files" do |app| path = app.paths["db"].first config.watchable_files.concat ["#{path}/schema.rb", "#{path}/structure.sql"] diff --git a/activerecord/lib/active_record/reflection.rb b/activerecord/lib/active_record/reflection.rb index 956fe7c51e..43f573f193 100644 --- a/activerecord/lib/active_record/reflection.rb +++ b/activerecord/lib/active_record/reflection.rb @@ -995,7 +995,7 @@ module ActiveRecord end def constraints - [source_type_info] + @reflection.constraints + [source_type_info] end def source_type_info diff --git a/activerecord/lib/active_record/relation/batches.rb b/activerecord/lib/active_record/relation/batches.rb index 243ef0eae9..b99807adf3 100644 --- a/activerecord/lib/active_record/relation/batches.rb +++ b/activerecord/lib/active_record/relation/batches.rb @@ -2,6 +2,8 @@ require "active_record/relation/batches/batch_enumerator" module ActiveRecord module Batches + ORDER_OR_LIMIT_IGNORED_MESSAGE = "Scoped order and limit are ignored, it's forced to be batch order and batch size" + # Looping through a collection of records from the database # (using the Scoping::Named::ClassMethods.all method, for example) # is very inefficient since it will try to instantiate all the objects at once. @@ -31,6 +33,9 @@ module ActiveRecord # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000. # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value. # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value. + # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when + # the order and limit have to be ignored due to batching. + # # This is especially useful if you want multiple workers dealing with # the same processing queue. You can make worker 1 handle all the records # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond @@ -48,13 +53,13 @@ module ActiveRecord # # NOTE: You can't set the limit either, that's used to control # the batch sizes. - def find_each(start: nil, finish: nil, batch_size: 1000) + def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil) if block_given? - find_in_batches(start: start, finish: finish, batch_size: batch_size) do |records| + find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do |records| records.each { |record| yield record } end else - enum_for(:find_each, start: start, finish: finish, batch_size: batch_size) do + enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do relation = self apply_limits(relation, start, finish).size end @@ -83,6 +88,9 @@ module ActiveRecord # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000. # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value. # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value. + # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when + # the order and limit have to be ignored due to batching. + # # This is especially useful if you want multiple workers dealing with # the same processing queue. You can make worker 1 handle all the records # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond @@ -100,16 +108,16 @@ module ActiveRecord # # NOTE: You can't set the limit either, that's used to control # the batch sizes. - def find_in_batches(start: nil, finish: nil, batch_size: 1000) + def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil) relation = self unless block_given? - return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size) do + return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do total = apply_limits(relation, start, finish).size (total - 1).div(batch_size) + 1 end end - in_batches(of: batch_size, start: start, finish: finish, load: true) do |batch| + in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore) do |batch| yield batch.to_a end end @@ -140,6 +148,8 @@ module ActiveRecord # * <tt>:load</tt> - Specifies if the relation should be loaded. Default to false. # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value. # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value. + # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when + # the order and limit have to be ignored due to batching. # # This is especially useful if you want to work with the # ActiveRecord::Relation object instead of the array of records, or if @@ -171,14 +181,14 @@ module ActiveRecord # # NOTE: You can't set the limit either, that's used to control the batch # sizes. - def in_batches(of: 1000, start: nil, finish: nil, load: false) + def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil) relation = self unless block_given? return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self) end - if logger && (arel.orders.present? || arel.taken.present?) - logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size") + if arel.orders.present? || arel.taken.present? + act_on_order_or_limit_ignored(error_on_ignore) end relation = relation.reorder(batch_order).limit(of) @@ -219,5 +229,15 @@ module ActiveRecord def batch_order "#{quoted_table_name}.#{quoted_primary_key} ASC" end + + def act_on_order_or_limit_ignored(error_on_ignore) + raise_error = (error_on_ignore.nil? ? self.klass.error_on_ignored_order_or_limit : error_on_ignore) + + if raise_error + raise ArgumentError.new(ORDER_OR_LIMIT_IGNORED_MESSAGE) + elsif logger + logger.warn(ORDER_OR_LIMIT_IGNORED_MESSAGE) + end + end end end diff --git a/activerecord/lib/active_record/relation/finder_methods.rb b/activerecord/lib/active_record/relation/finder_methods.rb index 0037398554..c3053f0b13 100644 --- a/activerecord/lib/active_record/relation/finder_methods.rb +++ b/activerecord/lib/active_record/relation/finder_methods.rb @@ -255,13 +255,13 @@ module ActiveRecord # Person.offset(3).third_to_last # returns the third-to-last object from OFFSET 3 # Person.where(["user_name = :u", { u: user_name }]).third_to_last def third_to_last - find_nth(-3) + find_nth_from_last 3 end # Same as #third_to_last but raises ActiveRecord::RecordNotFound if no record # is found. def third_to_last! - find_nth!(-3) + find_nth_from_last 3 or raise RecordNotFound.new("Couldn't find #{@klass.name} with [#{arel.where_sql(@klass.arel_engine)}]") end # Find the second-to-last record. @@ -271,13 +271,13 @@ module ActiveRecord # Person.offset(3).second_to_last # returns the second-to-last object from OFFSET 3 # Person.where(["user_name = :u", { u: user_name }]).second_to_last def second_to_last - find_nth(-2) + find_nth_from_last 2 end # Same as #second_to_last but raises ActiveRecord::RecordNotFound if no record # is found. def second_to_last! - find_nth!(-2) + find_nth_from_last 2 or raise RecordNotFound.new("Couldn't find #{@klass.name} with [#{arel.where_sql(@klass.arel_engine)}]") end # Returns true if a record exists in the table that matches the +id+ or @@ -561,6 +561,25 @@ module ActiveRecord relation.limit(limit).to_a end + def find_nth_from_last(index) + if loaded? + @records[-index] + else + relation = if order_values.empty? && primary_key + order(arel_attribute(primary_key).asc) + else + self + end + + relation.to_a[-index] + # TODO: can be made more performant on large result sets by + # for instance, last(index)[-index] (which would require + # refactoring the last(n) finder method to make test suite pass), + # or by using a combination of reverse_order, limit, and offset, + # e.g., reverse_order.offset(index-1).first + end + end + private def find_nth_with_limit_and_offset(index, limit, offset:) # :nodoc: |