diff options
Diffstat (limited to 'activerecord/lib/active_record/relation/batches.rb')
-rw-r--r-- | activerecord/lib/active_record/relation/batches.rb | 47 |
1 files changed, 25 insertions, 22 deletions
diff --git a/activerecord/lib/active_record/relation/batches.rb b/activerecord/lib/active_record/relation/batches.rb index 2fd89882ff..4d14506965 100644 --- a/activerecord/lib/active_record/relation/batches.rb +++ b/activerecord/lib/active_record/relation/batches.rb @@ -1,21 +1,26 @@ -require 'active_support/core_ext/object/blank' module ActiveRecord module Batches - # Yields each record that was found by the find +options+. The find is - # performed by find_in_batches with a batch size of 1000 (or as - # specified by the <tt>:batch_size</tt> option). + # Looping through a collection of records from the database + # (using the +all+ method, for example) is very inefficient + # since it will try to instantiate all the objects at once. # - # Example: + # In that case, batch processing methods allow you to work + # with the records in batches, thereby greatly reducing memory consumption. + # + # The #find_each method uses #find_in_batches with a batch size of 1000 (or as + # specified by the +:batch_size+ option). + # + # Person.all.find_each do |person| + # person.do_awesome_stuff + # end # # Person.where("age > 21").find_each do |person| # person.party_all_night! # end # - # Note: This method is only intended to use for batch processing of - # large amounts of records that wouldn't fit in memory all at once. If - # you just need to loop over less than 1000 records, it's probably - # better just to use the regular find methods. + # You can also pass the +:start+ option to specify + # an offset to control the starting point. def find_each(options = {}) find_in_batches(options) do |records| records.each { |record| yield record } @@ -23,14 +28,14 @@ module ActiveRecord end # Yields each batch of records that was found by the find +options+ as - # an array. The size of each batch is set by the <tt>:batch_size</tt> + # an array. The size of each batch is set by the +:batch_size+ # option; the default is 1000. # # You can control the starting point for the batch processing by - # supplying the <tt>:start</tt> option. This is especially useful if you + # supplying the +:start+ option. This is especially useful if you # want multiple workers dealing with the same processing queue. You can # make worker 1 handle all the records between id 0 and 10,000 and - # worker 2 handle from 10,000 and beyond (by setting the <tt>:start</tt> + # worker 2 handle from 10,000 and beyond (by setting the +:start+ # option on that worker). # # It's not possible to set the order. That is automatically set to @@ -39,31 +44,29 @@ module ActiveRecord # primary keys. You can't set the limit either, that's used to control # the batch sizes. # - # Example: - # # Person.where("age > 21").find_in_batches do |group| # sleep(50) # Make sure it doesn't get too crowded in there! # group.each { |person| person.party_all_night! } # end + # + # # Let's process the next 2000 records + # Person.all.find_in_batches(start: 2000, batch_size: 2000) do |group| + # group.each { |person| person.party_all_night! } + # end def find_in_batches(options = {}) + options.assert_valid_keys(:start, :batch_size) + relation = self unless arel.orders.blank? && arel.taken.blank? ActiveRecord::Base.logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size") end - if (finder_options = options.except(:start, :batch_size)).present? - raise "You can't specify an order, it's forced to be #{batch_order}" if options[:order].present? - raise "You can't specify a limit, it's forced to be the batch_size" if options[:limit].present? - - relation = apply_finder_options(finder_options) - end - start = options.delete(:start).to_i batch_size = options.delete(:batch_size) || 1000 relation = relation.reorder(batch_order).limit(batch_size) - records = relation.where(table[primary_key].gteq(start)).all + records = relation.where(table[primary_key].gteq(start)).to_a while records.any? records_size = records.size |