aboutsummaryrefslogtreecommitdiffstats
path: root/activerecord/lib/active_record/relation/batches.rb
diff options
context:
space:
mode:
Diffstat (limited to 'activerecord/lib/active_record/relation/batches.rb')
-rw-r--r--activerecord/lib/active_record/relation/batches.rb47
1 files changed, 25 insertions, 22 deletions
diff --git a/activerecord/lib/active_record/relation/batches.rb b/activerecord/lib/active_record/relation/batches.rb
index 2fd89882ff..4d14506965 100644
--- a/activerecord/lib/active_record/relation/batches.rb
+++ b/activerecord/lib/active_record/relation/batches.rb
@@ -1,21 +1,26 @@
-require 'active_support/core_ext/object/blank'
module ActiveRecord
module Batches
- # Yields each record that was found by the find +options+. The find is
- # performed by find_in_batches with a batch size of 1000 (or as
- # specified by the <tt>:batch_size</tt> option).
+ # Looping through a collection of records from the database
+ # (using the +all+ method, for example) is very inefficient
+ # since it will try to instantiate all the objects at once.
#
- # Example:
+ # In that case, batch processing methods allow you to work
+ # with the records in batches, thereby greatly reducing memory consumption.
+ #
+ # The #find_each method uses #find_in_batches with a batch size of 1000 (or as
+ # specified by the +:batch_size+ option).
+ #
+ # Person.all.find_each do |person|
+ # person.do_awesome_stuff
+ # end
#
# Person.where("age > 21").find_each do |person|
# person.party_all_night!
# end
#
- # Note: This method is only intended to use for batch processing of
- # large amounts of records that wouldn't fit in memory all at once. If
- # you just need to loop over less than 1000 records, it's probably
- # better just to use the regular find methods.
+ # You can also pass the +:start+ option to specify
+ # an offset to control the starting point.
def find_each(options = {})
find_in_batches(options) do |records|
records.each { |record| yield record }
@@ -23,14 +28,14 @@ module ActiveRecord
end
# Yields each batch of records that was found by the find +options+ as
- # an array. The size of each batch is set by the <tt>:batch_size</tt>
+ # an array. The size of each batch is set by the +:batch_size+
# option; the default is 1000.
#
# You can control the starting point for the batch processing by
- # supplying the <tt>:start</tt> option. This is especially useful if you
+ # supplying the +:start+ option. This is especially useful if you
# want multiple workers dealing with the same processing queue. You can
# make worker 1 handle all the records between id 0 and 10,000 and
- # worker 2 handle from 10,000 and beyond (by setting the <tt>:start</tt>
+ # worker 2 handle from 10,000 and beyond (by setting the +:start+
# option on that worker).
#
# It's not possible to set the order. That is automatically set to
@@ -39,31 +44,29 @@ module ActiveRecord
# primary keys. You can't set the limit either, that's used to control
# the batch sizes.
#
- # Example:
- #
# Person.where("age > 21").find_in_batches do |group|
# sleep(50) # Make sure it doesn't get too crowded in there!
# group.each { |person| person.party_all_night! }
# end
+ #
+ # # Let's process the next 2000 records
+ # Person.all.find_in_batches(start: 2000, batch_size: 2000) do |group|
+ # group.each { |person| person.party_all_night! }
+ # end
def find_in_batches(options = {})
+ options.assert_valid_keys(:start, :batch_size)
+
relation = self
unless arel.orders.blank? && arel.taken.blank?
ActiveRecord::Base.logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
end
- if (finder_options = options.except(:start, :batch_size)).present?
- raise "You can't specify an order, it's forced to be #{batch_order}" if options[:order].present?
- raise "You can't specify a limit, it's forced to be the batch_size" if options[:limit].present?
-
- relation = apply_finder_options(finder_options)
- end
-
start = options.delete(:start).to_i
batch_size = options.delete(:batch_size) || 1000
relation = relation.reorder(batch_order).limit(batch_size)
- records = relation.where(table[primary_key].gteq(start)).all
+ records = relation.where(table[primary_key].gteq(start)).to_a
while records.any?
records_size = records.size