diff options
Diffstat (limited to 'activerecord')
24 files changed, 418 insertions, 145 deletions
diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index 8521d3b07b..1d0e4b32a3 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,50 @@ +* Add newline between each migration in `structure.sql`. + + Keeps schema migration inserts as a single commit, but allows for easier + git diffing. + + Fixes #25504. + + *Grey Baker*, *Norberto Lopes* + +* The flag `error_on_ignored_order_or_limit` has been deprecated in favor of + the current `error_on_ignored_order`. + + *Xavier Noria* + +* Batch processing methods support `limit`: + + Post.limit(10_000).find_each do |post| + # ... + end + + It also works in `find_in_batches` and `in_batches`. + + *Xavier Noria* + +* Using `group` with an attribute that has a custom type will properly cast + the hash keys after calling a calculation method like `count`. Fixes #25595. + + *Sean Griffin* + +* Fix the generated `#to_param` method to use `omission: ''` so that + the resulting output is actually up to 20 characters, not + effectively 17 to leave room for the default "...". + Also call `#parameterize` before `#truncate` and make the + `separator: /-/` to maximize the information included in the + output. + + Fixes #23635. + + *Rob Biedenharn* + +* Ensure concurrent invocations of the connection reaper cannot allocate the + same connection to two threads. + + Fixes #25585. + + *Matthew Draper* + * Inspecting an object with an associated array of over 10 elements no longer truncates the array, preventing `inspect` from looping infinitely in some cases. @@ -5,7 +52,7 @@ *Kevin McPhillips* * Removed the unused methods `ActiveRecord::Base.connection_id` and - `ActiveRecord::Base.connection_id=` + `ActiveRecord::Base.connection_id=`. *Sean Griffin* @@ -24,15 +71,15 @@ *Johannes Opper* -* Introduce ActiveRecord::TransactionSerializationError for catching +* Introduce `ActiveRecord::TransactionSerializationError` for catching transaction serialization failures or deadlocks. *Erol Fornoles* -* PostgreSQL: Fix db:structure:load silent failure on SQL error +* PostgreSQL: Fix db:structure:load silent failure on SQL error. - The command line flag "-v ON_ERROR_STOP=1" should be used - when invoking psql to make sure errors are not suppressed. + The command line flag `-v ON_ERROR_STOP=1` should be used + when invoking `psql` to make sure errors are not suppressed. Example: diff --git a/activerecord/lib/active_record/associations/collection_proxy.rb b/activerecord/lib/active_record/associations/collection_proxy.rb index 5d1e7ffb73..17ccf5a86c 100644 --- a/activerecord/lib/active_record/associations/collection_proxy.rb +++ b/activerecord/lib/active_record/associations/collection_proxy.rb @@ -793,7 +793,7 @@ module ActiveRecord # Returns +true+ if the collection is empty. If the collection has been # loaded it is equivalent # to <tt>collection.size.zero?</tt>. If the collection has not been loaded, - # it is equivalent to <tt>collection.exists?</tt>. If the collection has + # it is equivalent to <tt>!collection.exists?</tt>. If the collection has # not already been loaded and you are going to fetch the records anyway it # is better to check <tt>collection.length.zero?</tt>. # diff --git a/activerecord/lib/active_record/associations/has_many_association.rb b/activerecord/lib/active_record/associations/has_many_association.rb index a9f6aaafef..4daafedcfb 100644 --- a/activerecord/lib/active_record/associations/has_many_association.rb +++ b/activerecord/lib/active_record/associations/has_many_association.rb @@ -27,14 +27,12 @@ module ActiveRecord throw(:abort) end + when :destroy + # No point in executing the counter update since we're going to destroy the parent anyway + load_target.each { |t| t.destroyed_by_association = reflection } + destroy_all else - if options[:dependent] == :destroy - # No point in executing the counter update since we're going to destroy the parent anyway - load_target.each { |t| t.destroyed_by_association = reflection } - destroy_all - else - delete_all - end + delete_all end end diff --git a/activerecord/lib/active_record/attribute_methods.rb b/activerecord/lib/active_record/attribute_methods.rb index 78bfcf34a9..eadd73aab0 100644 --- a/activerecord/lib/active_record/attribute_methods.rb +++ b/activerecord/lib/active_record/attribute_methods.rb @@ -209,13 +209,13 @@ module ActiveRecord # end # # person = Person.new - # person.respond_to(:name) # => true - # person.respond_to(:name=) # => true - # person.respond_to(:name?) # => true - # person.respond_to('age') # => true - # person.respond_to('age=') # => true - # person.respond_to('age?') # => true - # person.respond_to(:nothing) # => false + # person.respond_to?(:name) # => true + # person.respond_to?(:name=) # => true + # person.respond_to?(:name?) # => true + # person.respond_to?('age') # => true + # person.respond_to?('age=') # => true + # person.respond_to?('age?') # => true + # person.respond_to?(:nothing) # => false def respond_to?(name, include_private = false) return false unless super diff --git a/activerecord/lib/active_record/attributes.rb b/activerecord/lib/active_record/attributes.rb index 519de271c3..3211b6eaeb 100644 --- a/activerecord/lib/active_record/attributes.rb +++ b/activerecord/lib/active_record/attributes.rb @@ -34,10 +34,10 @@ module ActiveRecord # is not passed, the previous default value (if any) will be used. # Otherwise, the default will be +nil+. # - # +array+ (PG only) specifies that the type should be an array (see the + # +array+ (PostgreSQL only) specifies that the type should be an array (see the # examples below). # - # +range+ (PG only) specifies that the type should be a range (see the + # +range+ (PostgreSQL only) specifies that the type should be a range (see the # examples below). # # ==== Examples diff --git a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb index c341773be1..9b74c3a10f 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb @@ -74,7 +74,7 @@ module ActiveRecord #-- # Synchronization policy: # * all public methods can be called outside +synchronize+ - # * access to these i-vars needs to be in +synchronize+: + # * access to these instance variables needs to be in +synchronize+: # * @connections # * @now_connecting # * private methods that require being called in a +synchronize+ blocks @@ -329,13 +329,13 @@ module ActiveRecord # default max pool size to 5 @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5 - # The cache of threads mapped to reserved connections, the sole purpose - # of the cache is to speed-up +connection+ method, it is not the authoritative - # registry of which thread owns which connection, that is tracked by - # +connection.owner+ attr on each +connection+ instance. + # This variable tracks the cache of threads mapped to reserved connections, with the + # sole purpose of speeding up the +connection+ method. It is not the authoritative + # registry of which thread owns which connection. Connection ownership is tracked by + # the +connection.owner+ attr on each +connection+ instance. # The invariant works like this: if there is mapping of <tt>thread => conn</tt>, - # then that +thread+ does indeed own that +conn+, however an absence of a such - # mapping does not mean that the +thread+ doesn't own the said connection, in + # then that +thread+ does indeed own that +conn+. However, an absence of a such + # mapping does not mean that the +thread+ doesn't own the said connection. In # that case +conn.owner+ attr should be consulted. # Access and modification of +@thread_cached_conns+ does not require # synchronization. @@ -364,10 +364,10 @@ module ActiveRecord @thread_cached_conns[connection_cache_key(Thread.current)] ||= checkout end - # Is there an open connection that is being used for the current thread? + # Returns true if there is an open connection being used for the current thread. # # This method only works for connections that have been obtained through - # #connection or #with_connection methods, connections obtained through + # #connection or #with_connection methods. Connections obtained through # #checkout will not be detected by #active_connection? def active_connection? @thread_cached_conns[connection_cache_key(Thread.current)] @@ -415,7 +415,10 @@ module ActiveRecord with_exclusively_acquired_all_connections(raise_on_acquisition_timeout) do synchronize do @connections.each do |conn| - checkin conn + if conn.in_use? + conn.steal! + checkin conn + end conn.disconnect! end @connections = [] @@ -426,9 +429,9 @@ module ActiveRecord # Disconnects all connections in the pool, and clears the pool. # - # The pool first tries to gain ownership of all connections, if unable to + # The pool first tries to gain ownership of all connections. If unable to # do so within a timeout interval (default duration is - # <tt>spec.config[:checkout_timeout] * 2</tt> seconds), the pool is forcefully + # <tt>spec.config[:checkout_timeout] * 2</tt> seconds), then the pool is forcefully # disconnected without any regard for other connection owning threads. def disconnect! disconnect(false) @@ -447,7 +450,10 @@ module ActiveRecord with_exclusively_acquired_all_connections(raise_on_acquisition_timeout) do synchronize do @connections.each do |conn| - checkin conn + if conn.in_use? + conn.steal! + checkin conn + end conn.disconnect! if conn.requires_reloading? end @connections.delete_if(&:requires_reloading?) @@ -474,9 +480,9 @@ module ActiveRecord # Clears the cache which maps classes and re-connects connections that # require reloading. # - # The pool first tries to gain ownership of all connections, if unable to + # The pool first tries to gain ownership of all connections. If unable to # do so within a timeout interval (default duration is - # <tt>spec.config[:checkout_timeout] * 2</tt> seconds), the pool forcefully + # <tt>spec.config[:checkout_timeout] * 2</tt> seconds), then the pool forcefully # clears the cache and reloads connections without any regard for other # connection owning threads. def clear_reloadable_connections! @@ -530,20 +536,20 @@ module ActiveRecord @available.delete conn # @available.any_waiting? => true means that prior to removing this - # conn, the pool was at its max size (@connections.size == @size) - # this would mean that any threads stuck waiting in the queue wouldn't + # conn, the pool was at its max size (@connections.size == @size). + # This would mean that any threads stuck waiting in the queue wouldn't # know they could checkout_new_connection, so let's do it for them. # Because condition-wait loop is encapsulated in the Queue class # (that in turn is oblivious to ConnectionPool implementation), threads - # that are "stuck" there are helpless, they have no way of creating + # that are "stuck" there are helpless. They have no way of creating # new connections and are completely reliant on us feeding available # connections into the Queue. needs_new_connection = @available.any_waiting? end # This is intentionally done outside of the synchronized section as we - # would like not to hold the main mutex while checking out new connections, - # thus there is some chance that needs_new_connection information is now + # would like not to hold the main mutex while checking out new connections. + # Thus there is some chance that needs_new_connection information is now # stale, we can live with that (bulk_make_new_connections will make # sure not to exceed the pool's @size limit). bulk_make_new_connections(1) if needs_new_connection @@ -556,17 +562,17 @@ module ActiveRecord stale_connections = synchronize do @connections.select do |conn| conn.in_use? && !conn.owner.alive? + end.each do |conn| + conn.steal! end end stale_connections.each do |conn| - synchronize do - if conn.active? - conn.reset! - checkin conn - else - remove conn - end + if conn.active? + conn.reset! + checkin conn + else + remove conn end end end @@ -698,7 +704,7 @@ module ActiveRecord def acquire_connection(checkout_timeout) # NOTE: we rely on +@available.poll+ and +try_to_checkout_new_connection+ to # +conn.lease+ the returned connection (and to do this in a +synchronized+ - # section), this is not the cleanest implementation, as ideally we would + # section). This is not the cleanest implementation, as ideally we would # <tt>synchronize { conn.lease }</tt> in this method, but by leaving it to +@available.poll+ # and +try_to_checkout_new_connection+ we can piggyback on +synchronize+ sections # of the said methods and avoid an additional +synchronize+ overhead. @@ -822,7 +828,7 @@ module ActiveRecord # should use. # # The ConnectionHandler class is not coupled with the Active models, as it has no knowlodge - # about the model. The model, needs to pass a specification name to the handler, + # about the model. The model needs to pass a specification name to the handler, # in order to lookup the correct connection pool. class ConnectionHandler def initialize @@ -890,7 +896,7 @@ module ActiveRecord # Remove the connection for this class. This will close the active # connection and the defined connection (if they exist). The result - # can be used as an argument for establish_connection, for easily + # can be used as an argument for #establish_connection, for easily # re-establishing the connection. def remove_connection(spec_name) if pool = owner_to_pool.delete(spec_name) diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb index eec0bc8518..9e9ace49db 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb @@ -179,7 +179,7 @@ module ActiveRecord # A Symbol can be used to specify the type of the generated primary key column. # [<tt>:primary_key</tt>] # The name of the primary key, if one is to be added automatically. - # Defaults to +id+. If <tt>:id</tt> is false this option is ignored. + # Defaults to +id+. If <tt>:id</tt> is false, then this option is ignored. # # Note that Active Record models will automatically detect their # primary key. This can be avoided by using @@ -305,9 +305,9 @@ module ActiveRecord # # Creates a table called 'assemblies_parts' with no id. # create_join_table(:assemblies, :parts) # - # You can pass a +options+ hash can include the following keys: + # You can pass an +options+ hash which can include the following keys: # [<tt>:table_name</tt>] - # Sets the table name overriding the default + # Sets the table name, overriding the default. # [<tt>:column_options</tt>] # Any extra options you want appended to the columns definition. # [<tt>:options</tt>] @@ -433,7 +433,7 @@ module ActiveRecord # t.remove_index :company_id # end # - # See also Table for details on all of the various column transformation. + # See also Table for details on all of the various column transformations. def change_table(table_name, options = {}) if supports_bulk_alter? && options[:bulk] recorder = ActiveRecord::Migration::CommandRecorder.new(self) @@ -483,10 +483,10 @@ module ActiveRecord # # Available options are (none of these exists by default): # * <tt>:limit</tt> - - # Requests a maximum column length. This is number of characters for a <tt>:string</tt> column + # Requests a maximum column length. This is the number of characters for a <tt>:string</tt> column # and number of bytes for <tt>:text</tt>, <tt>:binary</tt> and <tt>:integer</tt> columns. # * <tt>:default</tt> - - # The column's default value. Use nil for NULL. + # The column's default value. Use +nil+ for +NULL+. # * <tt>:null</tt> - # Allows or disallows +NULL+ values in the column. This option could # have been named <tt>:null_allowed</tt>. @@ -495,7 +495,7 @@ module ActiveRecord # * <tt>:scale</tt> - # Specifies the scale for the <tt>:decimal</tt> and <tt>:numeric</tt> columns. # - # Note: The precision is the total number of significant digits + # Note: The precision is the total number of significant digits, # and the scale is the number of digits that can be stored following # the decimal point. For example, the number 123.45 has a precision of 5 # and a scale of 2. A decimal with a precision of 5 and a scale of 2 can @@ -564,7 +564,7 @@ module ActiveRecord # # The +type+ and +options+ parameters will be ignored if present. It can be helpful # to provide these in a migration's +change+ method so it can be reverted. - # In that case, +type+ and +options+ will be used by add_column. + # In that case, +type+ and +options+ will be used by #add_column. def remove_column(table_name, column_name, type = nil, options = {}) execute "ALTER TABLE #{quote_table_name(table_name)} DROP #{quote_column_name(column_name)}" end @@ -952,13 +952,13 @@ module ActiveRecord # Checks to see if a foreign key exists on a table for a given foreign key definition. # - # # Check a foreign key exists + # # Checks to see if a foreign key exists. # foreign_key_exists?(:accounts, :branches) # - # # Check a foreign key on a specified column exists + # # Checks to see if a foreign key on a specified column exists. # foreign_key_exists?(:accounts, column: :owner_id) # - # # Check a foreign key with a custom name exists + # # Checks to see if a foreign key with a custom name exists. # foreign_key_exists?(:accounts, name: "special_fk_name") # def foreign_key_exists?(from_table, options_or_to_table = {}) @@ -998,8 +998,8 @@ module ActiveRecord sm_table = ActiveRecord::Migrator.schema_migrations_table_name if supports_multi_insert? - sql = "INSERT INTO #{sm_table} (version) VALUES " - sql << versions.map {|v| "('#{v}')" }.join(', ') + sql = "INSERT INTO #{sm_table} (version) VALUES\n" + sql << versions.map {|v| "('#{v}')" }.join(",\n") sql << ";\n\n" sql else @@ -1081,7 +1081,7 @@ module ActiveRecord end # Given a set of columns and an ORDER BY clause, returns the columns for a SELECT DISTINCT. - # PostgreSQL, MySQL, and Oracle overrides this for custom DISTINCT syntax - they + # PostgreSQL, MySQL, and Oracle override this for custom DISTINCT syntax - they # require the order columns appear in the SELECT. # # columns_for_distinct("posts.id", ["posts.created_at desc"]) diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index d4b9e301bc..5747e4d1ee 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -184,7 +184,30 @@ module ActiveRecord # this method must only be called while holding connection pool's mutex def expire - @owner = nil + if in_use? + if @owner != Thread.current + raise ActiveRecordError, "Cannot expire connection, " << + "it is owned by a different thread: #{@owner}. " << + "Current thread: #{Thread.current}." + end + + @owner = nil + else + raise ActiveRecordError, 'Cannot expire connection, it is not currently leased.' + end + end + + # this method must only be called while holding connection pool's mutex (and a desire for segfaults) + def steal! # :nodoc: + if in_use? + if @owner != Thread.current + pool.send :remove_connection_from_thread_cache, self, @owner + + @owner = Thread.current + end + else + raise ActiveRecordError, 'Cannot steal connection, it is not currently leased.' + end end def unprepared_statement diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb index eb2268157b..3384012c49 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb @@ -159,7 +159,7 @@ module ActiveRecord end # Returns 62. SQLite supports index names up to 64 - # characters. The rest is used by rails internally to perform + # characters. The rest is used by Rails internally to perform # temporary rename operations def allowed_index_name_length index_name_length - 2 diff --git a/activerecord/lib/active_record/core.rb b/activerecord/lib/active_record/core.rb index de337b24d6..2c94463acd 100644 --- a/activerecord/lib/active_record/core.rb +++ b/activerecord/lib/active_record/core.rb @@ -72,11 +72,20 @@ module ActiveRecord ## # :singleton-method: - # Specifies if an error should be raised on query limit or order being + # Specifies if an error should be raised if the query has an order being # ignored when doing batch queries. Useful in applications where the - # limit or scope being ignored is error-worthy, rather than a warning. + # scope being ignored is error-worthy, rather than a warning. + mattr_accessor :error_on_ignored_order, instance_writer: false + self.error_on_ignored_order = false + mattr_accessor :error_on_ignored_order_or_limit, instance_writer: false - self.error_on_ignored_order_or_limit = false + def self.error_on_ignored_order_or_limit=(value) + ActiveSupport::Deprecation.warn(<<-MSG.squish) + The flag error_on_ignored_order_or_limit is deprecated. Limits are + now supported. Please use error_on_ignored_order= instead. + MSG + self.error_on_ignored_order = value + end ## # :singleton-method: diff --git a/activerecord/lib/active_record/integration.rb b/activerecord/lib/active_record/integration.rb index 466c8509a4..d729deb07b 100644 --- a/activerecord/lib/active_record/integration.rb +++ b/activerecord/lib/active_record/integration.rb @@ -86,7 +86,7 @@ module ActiveRecord # # user = User.find_by(name: 'David Heinemeier Hansson') # user.id # => 125 - # user_path(user) # => "/users/125-david" + # user_path(user) # => "/users/125-david-heinemeier" # # Because the generated param begins with the record's +id+, it is # suitable for passing to +find+. In a controller, for example: @@ -100,7 +100,7 @@ module ActiveRecord define_method :to_param do if (default = super()) && (result = send(method_name).to_s).present? && - (param = result.squish.truncate(20, separator: /\s/, omission: nil).parameterize).present? + (param = result.squish.parameterize.truncate(20, separator: /-/, omission: '')).present? "#{default}-#{param}" else default diff --git a/activerecord/lib/active_record/model_schema.rb b/activerecord/lib/active_record/model_schema.rb index 7996c32bbc..114686c5d3 100644 --- a/activerecord/lib/active_record/model_schema.rb +++ b/activerecord/lib/active_record/model_schema.rb @@ -282,8 +282,12 @@ module ActiveRecord # # +attr_name+ The name of the attribute to retrieve the type for. Must be # a string - def type_for_attribute(attr_name) - attribute_types[attr_name] + def type_for_attribute(attr_name, &block) + if block + attribute_types.fetch(attr_name, &block) + else + attribute_types[attr_name] + end end # Returns a hash where the keys are column names and the values are diff --git a/activerecord/lib/active_record/persistence.rb b/activerecord/lib/active_record/persistence.rb index afed5e5e85..510e8a6e43 100644 --- a/activerecord/lib/active_record/persistence.rb +++ b/activerecord/lib/active_record/persistence.rb @@ -479,7 +479,12 @@ module ActiveRecord # ball.touch(:updated_at) # => raises ActiveRecordError # def touch(*names, time: nil) - raise ActiveRecordError, "cannot touch on a new record object" unless persisted? + unless persisted? + raise ActiveRecordError, <<-MSG.squish + cannot touch on a new or destroyed record object. Consider using + persisted?, new_record?, or destroyed? before touching + MSG + end time ||= current_time_from_proper_timezone attributes = timestamp_attributes_for_update_in_model diff --git a/activerecord/lib/active_record/railtie.rb b/activerecord/lib/active_record/railtie.rb index 98ea425d16..2c0ca62924 100644 --- a/activerecord/lib/active_record/railtie.rb +++ b/activerecord/lib/active_record/railtie.rb @@ -3,7 +3,7 @@ require "rails" require "active_model/railtie" # For now, action_controller must always be present with -# rails, so let's make sure that it gets required before +# Rails, so let's make sure that it gets required before # here. This is needed for correctly setting up the middleware. # In the future, this might become an optional require. require "action_controller/railtie" diff --git a/activerecord/lib/active_record/relation/batches.rb b/activerecord/lib/active_record/relation/batches.rb index 3639625722..20ed4526b0 100644 --- a/activerecord/lib/active_record/relation/batches.rb +++ b/activerecord/lib/active_record/relation/batches.rb @@ -1,8 +1,8 @@ -require "active_record/relation/batches/batch_enumerator" +require 'active_record/relation/batches/batch_enumerator' module ActiveRecord module Batches - ORDER_OR_LIMIT_IGNORED_MESSAGE = "Scoped order and limit are ignored, it's forced to be batch order and batch size." + ORDER_IGNORE_MESSAGE = "Scoped order is ignored, it's forced to be batch order." # Looping through a collection of records from the database # (using the Scoping::Named::ClassMethods.all method, for example) @@ -34,15 +34,19 @@ module ActiveRecord # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value. # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value. # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when - # the order and limit have to be ignored due to batching. + # an order is present in the relation. # - # This is especially useful if you want multiple workers dealing with - # the same processing queue. You can make worker 1 handle all the records - # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond - # (by setting the +:start+ and +:finish+ option on each worker). + # Limits are honored, and if present there is no requirement for the batch + # size, it can be less than, equal, or greater than the limit. # - # # Let's process for a batch of 2000 records, skipping the first 2000 rows - # Person.find_each(start: 2000, batch_size: 2000) do |person| + # The options +start+ and +finish+ are especially useful if you want + # multiple workers dealing with the same processing queue. You can make + # worker 1 handle all the records between id 1 and 9999 and worker 2 + # handle from 10000 and beyond by setting the +:start+ and +:finish+ + # option on each worker. + # + # # Let's process from record 10_000 on. + # Person.find_each(start: 10_000) do |person| # person.party_all_night! # end # @@ -51,8 +55,8 @@ module ActiveRecord # work. This also means that this method only works when the primary key is # orderable (e.g. an integer or string). # - # NOTE: You can't set the limit either, that's used to control - # the batch sizes. + # NOTE: By its nature, batch processing is subject to race conditions if + # other processes are modifying the database. def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil) if block_given? find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do |records| @@ -89,15 +93,19 @@ module ActiveRecord # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value. # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value. # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when - # the order and limit have to be ignored due to batching. + # an order is present in the relation. + # + # Limits are honored, and if present there is no requirement for the batch + # size, it can be less than, equal, or greater than the limit. # - # This is especially useful if you want multiple workers dealing with - # the same processing queue. You can make worker 1 handle all the records - # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond - # (by setting the +:start+ and +:finish+ option on each worker). + # The options +start+ and +finish+ are especially useful if you want + # multiple workers dealing with the same processing queue. You can make + # worker 1 handle all the records between id 1 and 9999 and worker 2 + # handle from 10000 and beyond by setting the +:start+ and +:finish+ + # option on each worker. # - # # Let's process the next 2000 records - # Person.find_in_batches(start: 2000, batch_size: 2000) do |group| + # # Let's process from record 10_000 on. + # Person.find_in_batches(start: 10_000) do |group| # group.each { |person| person.party_all_night! } # end # @@ -106,8 +114,8 @@ module ActiveRecord # work. This also means that this method only works when the primary key is # orderable (e.g. an integer or string). # - # NOTE: You can't set the limit either, that's used to control - # the batch sizes. + # NOTE: By its nature, batch processing is subject to race conditions if + # other processes are modifying the database. def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil) relation = self unless block_given? @@ -149,17 +157,19 @@ module ActiveRecord # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value. # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value. # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when - # the order and limit have to be ignored due to batching. + # an order is present in the relation. + # + # Limits are honored, and if present there is no requirement for the batch + # size, it can be less than, equal, or greater than the limit. # - # This is especially useful if you want to work with the - # ActiveRecord::Relation object instead of the array of records, or if - # you want multiple workers dealing with the same processing queue. You can - # make worker 1 handle all the records between id 0 and 10,000 and worker 2 - # handle from 10,000 and beyond (by setting the +:start+ and +:finish+ - # option on each worker). + # The options +start+ and +finish+ are especially useful if you want + # multiple workers dealing with the same processing queue. You can make + # worker 1 handle all the records between id 1 and 9999 and worker 2 + # handle from 10000 and beyond by setting the +:start+ and +:finish+ + # option on each worker. # - # # Let's process the next 2000 records - # Person.in_batches(of: 2000, start: 2000).update_all(awesome: true) + # # Let's process from record 10_000 on. + # Person.in_batches(start: 10_000).update_all(awesome: true) # # An example of calling where query method on the relation: # @@ -179,19 +189,25 @@ module ActiveRecord # consistent. Therefore the primary key must be orderable, e.g an integer # or a string. # - # NOTE: You can't set the limit either, that's used to control the batch - # sizes. + # NOTE: By its nature, batch processing is subject to race conditions if + # other processes are modifying the database. def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil) relation = self unless block_given? return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self) end - if arel.orders.present? || arel.taken.present? - act_on_order_or_limit_ignored(error_on_ignore) + if arel.orders.present? + act_on_ignored_order(error_on_ignore) + end + + batch_limit = of + if limit_value + remaining = limit_value + batch_limit = remaining if remaining < batch_limit end - relation = relation.reorder(batch_order).limit(of) + relation = relation.reorder(batch_order).limit(batch_limit) relation = apply_limits(relation, start, finish) batch_relation = relation @@ -199,11 +215,11 @@ module ActiveRecord if load records = batch_relation.records ids = records.map(&:id) - yielded_relation = self.where(primary_key => ids) + yielded_relation = where(primary_key => ids) yielded_relation.load_records(records) else ids = batch_relation.pluck(primary_key) - yielded_relation = self.where(primary_key => ids) + yielded_relation = where(primary_key => ids) end break if ids.empty? @@ -213,7 +229,20 @@ module ActiveRecord yield yielded_relation - break if ids.length < of + break if ids.length < batch_limit + + if limit_value + remaining -= ids.length + + if remaining == 0 + # Saves a useless iteration when the limit is a multiple of the + # batch size. + break + elsif remaining < batch_limit + relation = relation.limit(remaining) + end + end + batch_relation = relation.where(arel_attribute(primary_key).gt(primary_key_offset)) end end @@ -230,13 +259,13 @@ module ActiveRecord "#{quoted_table_name}.#{quoted_primary_key} ASC" end - def act_on_order_or_limit_ignored(error_on_ignore) - raise_error = (error_on_ignore.nil? ? self.klass.error_on_ignored_order_or_limit : error_on_ignore) + def act_on_ignored_order(error_on_ignore) + raise_error = (error_on_ignore.nil? ? self.klass.error_on_ignored_order : error_on_ignore) if raise_error - raise ArgumentError.new(ORDER_OR_LIMIT_IGNORED_MESSAGE) + raise ArgumentError.new(ORDER_IGNORE_MESSAGE) elsif logger - logger.warn(ORDER_OR_LIMIT_IGNORED_MESSAGE) + logger.warn(ORDER_IGNORE_MESSAGE) end end end diff --git a/activerecord/lib/active_record/relation/calculations.rb b/activerecord/lib/active_record/relation/calculations.rb index d6d92b8607..a97b71815a 100644 --- a/activerecord/lib/active_record/relation/calculations.rb +++ b/activerecord/lib/active_record/relation/calculations.rb @@ -312,8 +312,10 @@ module ActiveRecord Hash[calculated_data.map do |row| key = group_columns.map { |aliaz, col_name| - column = calculated_data.column_types.fetch(aliaz) do - type_for(col_name) + column = type_for(col_name) do + calculated_data.column_types.fetch(aliaz) do + Type::Value.new + end end type_cast_calculated_value(row[aliaz], column) } @@ -346,9 +348,9 @@ module ActiveRecord @klass.connection.table_alias_for(table_name) end - def type_for(field) + def type_for(field, &block) field_name = field.respond_to?(:name) ? field.name.to_s : field.to_s.split('.').last - @klass.type_for_attribute(field_name) + @klass.type_for_attribute(field_name, &block) end def type_cast_calculated_value(value, type, operation = nil) diff --git a/activerecord/lib/active_record/touch_later.rb b/activerecord/lib/active_record/touch_later.rb index 9a80a63e28..598873ea3a 100644 --- a/activerecord/lib/active_record/touch_later.rb +++ b/activerecord/lib/active_record/touch_later.rb @@ -8,7 +8,12 @@ module ActiveRecord end def touch_later(*names) # :nodoc: - raise ActiveRecordError, "cannot touch on a new record object" unless persisted? + unless persisted? + raise ActiveRecordError, <<-MSG.squish + cannot touch on a new or destroyed record object. Consider using + persisted?, new_record?, or destroyed? before touching + MSG + end @_defer_touch_attrs ||= timestamp_attributes_for_update_in_model @_defer_touch_attrs |= names diff --git a/activerecord/test/cases/batches_test.rb b/activerecord/test/cases/batches_test.rb index db71840658..2c6011489e 100644 --- a/activerecord/test/cases/batches_test.rb +++ b/activerecord/test/cases/batches_test.rb @@ -68,10 +68,26 @@ class EachTest < ActiveRecord::TestCase end end - def test_warn_if_limit_scope_is_set - assert_called(ActiveRecord::Base.logger, :warn) do - Post.limit(1).find_each { |post| post } + test 'find_each should honor limit if passed a block' do + limit = @total - 1 + total = 0 + + Post.limit(limit).find_each do |post| + total += 1 + end + + assert_equal limit, total + end + + test 'find_each should honor limit if no block is passed' do + limit = @total - 1 + total = 0 + + Post.limit(limit).find_each.each do |post| + total += 1 end + + assert_equal limit, total end def test_warn_if_order_scope_is_set @@ -84,7 +100,7 @@ class EachTest < ActiveRecord::TestCase previous_logger = ActiveRecord::Base.logger ActiveRecord::Base.logger = nil assert_nothing_raised do - Post.limit(1).find_each { |post| post } + Post.order('comments_count DESC').find_each { |post| post } end ensure ActiveRecord::Base.logger = previous_logger @@ -172,26 +188,26 @@ class EachTest < ActiveRecord::TestCase def test_find_in_batches_should_not_error_if_config_overridden # Set the config option which will be overridden - prev = ActiveRecord::Base.error_on_ignored_order_or_limit - ActiveRecord::Base.error_on_ignored_order_or_limit = true + prev = ActiveRecord::Base.error_on_ignored_order + ActiveRecord::Base.error_on_ignored_order = true assert_nothing_raised do PostWithDefaultScope.find_in_batches(error_on_ignore: false){} end ensure # Set back to default - ActiveRecord::Base.error_on_ignored_order_or_limit = prev + ActiveRecord::Base.error_on_ignored_order = prev end def test_find_in_batches_should_error_on_config_specified_to_error # Set the config option - prev = ActiveRecord::Base.error_on_ignored_order_or_limit - ActiveRecord::Base.error_on_ignored_order_or_limit = true + prev = ActiveRecord::Base.error_on_ignored_order + ActiveRecord::Base.error_on_ignored_order = true assert_raise(ArgumentError) do PostWithDefaultScope.find_in_batches(){} end ensure # Set back to default - ActiveRecord::Base.error_on_ignored_order_or_limit = prev + ActiveRecord::Base.error_on_ignored_order = prev end def test_find_in_batches_should_not_error_by_default @@ -249,6 +265,28 @@ class EachTest < ActiveRecord::TestCase end end + test 'find_in_batches should honor limit if passed a block' do + limit = @total - 1 + total = 0 + + Post.limit(limit).find_in_batches do |batch| + total += batch.size + end + + assert_equal limit, total + end + + test 'find_in_batches should honor limit if no block is passed' do + limit = @total - 1 + total = 0 + + Post.limit(limit).find_in_batches.each do |batch| + total += batch.size + end + + assert_equal limit, total + end + def test_in_batches_should_not_execute_any_query assert_no_queries do assert_kind_of ActiveRecord::Batches::BatchEnumerator, Post.in_batches(of: 2) @@ -486,4 +524,78 @@ class EachTest < ActiveRecord::TestCase assert_equal 1, Post.find_in_batches(:batch_size => 10_000).size end end + + [true, false].each do |load| + test "in_batches should return limit records when limit is less than batch size and load is #{load}" do + limit = 3 + batch_size = 5 + total = 0 + + Post.limit(limit).in_batches(of: batch_size, load: load) do |batch| + total += batch.count + end + + assert_equal limit, total + end + + test "in_batches should return limit records when limit is greater than batch size and load is #{load}" do + limit = 5 + batch_size = 3 + total = 0 + + Post.limit(limit).in_batches(of: batch_size, load: load) do |batch| + total += batch.count + end + + assert_equal limit, total + end + + test "in_batches should return limit records when limit is a multiple of the batch size and load is #{load}" do + limit = 6 + batch_size = 3 + total = 0 + + Post.limit(limit).in_batches(of: batch_size, load: load) do |batch| + total += batch.count + end + + assert_equal limit, total + end + + test "in_batches should return no records if the limit is 0 and load is #{load}" do + limit = 0 + batch_size = 1 + total = 0 + + Post.limit(limit).in_batches(of: batch_size, load: load) do |batch| + total += batch.count + end + + assert_equal limit, total + end + + test "in_batches should return all if the limit is greater than the number of records when load is #{load}" do + limit = @total + 1 + batch_size = 1 + total = 0 + + Post.limit(limit).in_batches(of: batch_size, load: load) do |batch| + total += batch.count + end + + assert_equal @total, total + end + end + + test 'error_on_ignored_order_or_limit= is deprecated' do + begin + prev = ActiveRecord::Base.error_on_ignored_order + assert_deprecated 'Please use error_on_ignored_order= instead.' do + ActiveRecord::Base.error_on_ignored_order_or_limit = true + end + assert ActiveRecord::Base.error_on_ignored_order + ensure + ActiveRecord::Base.error_on_ignored_order = prev + end + end end diff --git a/activerecord/test/cases/calculations_test.rb b/activerecord/test/cases/calculations_test.rb index cfae700159..6acfec0621 100644 --- a/activerecord/test/cases/calculations_test.rb +++ b/activerecord/test/cases/calculations_test.rb @@ -1,4 +1,5 @@ require "cases/helper" +require "models/book" require 'models/club' require 'models/company' require "models/contract" @@ -25,7 +26,7 @@ class NumericData < ActiveRecord::Base end class CalculationsTest < ActiveRecord::TestCase - fixtures :companies, :accounts, :topics, :speedometers, :minivans + fixtures :companies, :accounts, :topics, :speedometers, :minivans, :books def test_should_sum_field assert_equal 318, Account.sum(:credit_limit) @@ -793,4 +794,8 @@ class CalculationsTest < ActiveRecord::TestCase assert_equal 50, result[1].credit_limit assert_equal 50, result[2].credit_limit end + + def test_group_by_attribute_with_custom_type + assert_equal({ "proposed" => 2, "published" => 2 }, Book.group(:status).count) + end end diff --git a/activerecord/test/cases/connection_pool_test.rb b/activerecord/test/cases/connection_pool_test.rb index a45ee281c7..09e7848bda 100644 --- a/activerecord/test/cases/connection_pool_test.rb +++ b/activerecord/test/cases/connection_pool_test.rb @@ -151,7 +151,7 @@ module ActiveRecord assert_equal 1, active_connections(@pool).size ensure - @pool.connections.each(&:close) + @pool.connections.each { |conn| conn.close if conn.in_use? } end def test_remove_connection @@ -432,6 +432,9 @@ module ActiveRecord Thread.new { @pool.send(group_action_method) }.join # assert connection has been forcefully taken away from us assert_not @pool.active_connection? + + # make a new connection for with_connection to clean up + @pool.connection end end end diff --git a/activerecord/test/cases/integration_test.rb b/activerecord/test/cases/integration_test.rb index 08a186ae07..97a0cdb5db 100644 --- a/activerecord/test/cases/integration_test.rb +++ b/activerecord/test/cases/integration_test.rb @@ -29,10 +29,30 @@ class IntegrationTest < ActiveRecord::TestCase assert_equal '4-flamboyant-software', firm.to_param end + def test_to_param_class_method_truncates_words_properly + firm = Firm.find(4) + firm.name << ', Inc.' + assert_equal '4-flamboyant-software', firm.to_param + end + + def test_to_param_class_method_truncates_after_parameterize + firm = Firm.find(4) + firm.name = "Huey, Dewey, & Louie LLC" + # 123456789T123456789v + assert_equal '4-huey-dewey-louie-llc', firm.to_param + end + + def test_to_param_class_method_truncates_after_parameterize_with_hyphens + firm = Firm.find(4) + firm.name = "Door-to-Door Wash-n-Fold Service" + # 123456789T123456789v + assert_equal '4-door-to-door-wash-n', firm.to_param + end + def test_to_param_class_method_truncates firm = Firm.find(4) firm.name = 'a ' * 100 - assert_equal '4-a-a-a-a-a-a-a-a-a', firm.to_param + assert_equal '4-a-a-a-a-a-a-a-a-a-a', firm.to_param end def test_to_param_class_method_truncates_edge_case @@ -41,10 +61,16 @@ class IntegrationTest < ActiveRecord::TestCase assert_equal '4-david', firm.to_param end + def test_to_param_class_method_truncates_case_shown_in_doc + firm = Firm.find(4) + firm.name = 'David Heinemeier Hansson' + assert_equal '4-david-heinemeier', firm.to_param + end + def test_to_param_class_method_squishes firm = Firm.find(4) firm.name = "ab \n" * 100 - assert_equal '4-ab-ab-ab-ab-ab-ab', firm.to_param + assert_equal '4-ab-ab-ab-ab-ab-ab-ab', firm.to_param end def test_to_param_class_method_multibyte_character diff --git a/activerecord/test/cases/migration/foreign_key_test.rb b/activerecord/test/cases/migration/foreign_key_test.rb index 49a8fa241f..4237e89731 100644 --- a/activerecord/test/cases/migration/foreign_key_test.rb +++ b/activerecord/test/cases/migration/foreign_key_test.rb @@ -233,7 +233,7 @@ module ActiveRecord end add_foreign_key :houses, :cities, column: "city_id" - # remove and re-add to test that schema is updated and not accidently cached + # remove and re-add to test that schema is updated and not accidentally cached remove_foreign_key :houses, :cities add_foreign_key :houses, :cities, column: "city_id", on_delete: :cascade end diff --git a/activerecord/test/cases/migration/references_statements_test.rb b/activerecord/test/cases/migration/references_statements_test.rb index 70c64f3e71..5dddb35c4c 100644 --- a/activerecord/test/cases/migration/references_statements_test.rb +++ b/activerecord/test/cases/migration/references_statements_test.rb @@ -35,7 +35,7 @@ module ActiveRecord assert_not index_exists?(table_name, :user_id) end - def test_create_reference_id_index_even_if_index_option_is_passed + def test_create_reference_id_index_even_if_index_option_is_not_passed add_reference table_name, :user assert index_exists?(table_name, :user_id) end diff --git a/activerecord/test/cases/mixin_test.rb b/activerecord/test/cases/mixin_test.rb index 7ebdcac711..06af75af0f 100644 --- a/activerecord/test/cases/mixin_test.rb +++ b/activerecord/test/cases/mixin_test.rb @@ -41,13 +41,12 @@ class TouchTest < ActiveRecord::TestCase old_updated_at = stamped.updated_at - travel 5.minutes do - stamped.lft_will_change! - stamped.save + travel 5.minutes + stamped.lft_will_change! + stamped.save - assert_equal Time.now, stamped.updated_at - assert_equal old_updated_at, stamped.created_at - end + assert_equal Time.now, stamped.updated_at + assert_equal old_updated_at, stamped.created_at end def test_create_turned_off |