blob: 03bd4f9f93a42774654e98c6df027dbe897f25c7 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
|
module ActiveRecord
module Batches # :nodoc:
def self.included(base)
base.extend(ClassMethods)
end
# When processing large numbers of records, it's often a good idea to do so in batches to prevent memory ballooning.
module ClassMethods
# Yields each record that was found by the find +options+. The find is performed by find_in_batches
# with a batch size of 1000 (or as specified by the +batch_size+ option).
#
# Example:
#
# Person.find_each(:conditions => "age > 21") do |person|
# person.party_all_night!
# end
#
# Note: This method is only intended to use for batch processing of large amounts of records that wouldn't fit in
# memory all at once. If you just need to loop over less than 1000 records, it's probably better just to use the
# regular find methods.
def find_each(options = {})
find_in_batches(options) do |records|
records.each { |record| yield record }
end
self
end
# Yields each batch of records that was found by the find +options+ as an array. The size of each batch is
# set by the +batch_size+ option; the default is 1000.
#
# You can control the starting point for the batch processing by supplying the +start+ option. This is especially
# useful if you want multiple workers dealing with the same processing queue. You can make worker 1 handle all the
# records between id 0 and 10,000 and worker 2 handle from 10,000 and beyond (by setting the +start+ option on that
# worker).
#
# It's not possible to set the order. That is automatically set to ascending on the primary key ("id ASC")
# to make the batch ordering work. This also mean that this method only works with integer-based primary keys.
# You can't set the limit either, that's used to control the the batch sizes.
#
# Example:
#
# Person.find_in_batches(:conditions => "age > 21") do |group|
# sleep(50) # Make sure it doesn't get too crowded in there!
# group.each { |person| person.party_all_night! }
# end
def find_in_batches(options = {})
raise "You can't specify an order, it's forced to be #{batch_order}" if options[:order]
raise "You can't specify a limit, it's forced to be the batch_size" if options[:limit]
start = options.delete(:start).to_i
batch_size = options.delete(:batch_size) || 1000
with_scope(:find => options.merge(:order => batch_order, :limit => batch_size)) do
records = find(:all, :conditions => [ "#{table_name}.#{primary_key} >= ?", start ])
while records.any?
yield records
break if records.size < batch_size
records = find(:all, :conditions => [ "#{table_name}.#{primary_key} > ?", records.last.id ])
end
end
end
private
def batch_order
"#{table_name}.#{primary_key} ASC"
end
end
end
end
|
