From 08a748118377f22823e06315f57db16e1ce3a7c3 Mon Sep 17 00:00:00 2001 From: Scott Ringwelski Date: Mon, 1 Feb 2016 20:17:17 -0800 Subject: Add initial support for allowing an error on order or limit of queries being ignored in batches add some documentation and add 4 tests regarding error vs. warning behavior fix a typo when referring to the message go back to default in tests so that ordering is not important. use a constant instead of method. fix assert_nothing_raised call. use self.klass to allow per class configuration remove logger warn assets as that is tested elsewhere. pass error_on_ignore through find_each and find_in_batches also. add blocks to the finds so that the code is actually executed put the setting back to default in an ensure Add a changelog entry --- activerecord/lib/active_record/core.rb | 8 +++++ activerecord/lib/active_record/relation/batches.rb | 38 +++++++++++++++++----- 2 files changed, 37 insertions(+), 9 deletions(-) (limited to 'activerecord/lib') diff --git a/activerecord/lib/active_record/core.rb b/activerecord/lib/active_record/core.rb index 475a298467..bed0cf9eea 100644 --- a/activerecord/lib/active_record/core.rb +++ b/activerecord/lib/active_record/core.rb @@ -70,6 +70,14 @@ module ActiveRecord mattr_accessor :schema_format, instance_writer: false self.schema_format = :ruby + ## + # :singleton-method: + # Specifies if an error should be raised on query limit or order being + # ignored when doing batch queries. Useful in applications where the + # limit or scope being ignored is error-worthy, rather than a warning. + mattr_accessor :error_on_ignored_order_or_limit, instance_writer: false + self.error_on_ignored_order_or_limit = false + ## # :singleton-method: # Specify whether or not to use timestamps for migration versions diff --git a/activerecord/lib/active_record/relation/batches.rb b/activerecord/lib/active_record/relation/batches.rb index 54587ae18e..8f2dae3369 100644 --- a/activerecord/lib/active_record/relation/batches.rb +++ b/activerecord/lib/active_record/relation/batches.rb @@ -2,6 +2,8 @@ require "active_record/relation/batches/batch_enumerator" module ActiveRecord module Batches + ORDER_OR_LIMIT_IGNORED_MESSAGE = "Scoped order and limit are ignored, it's forced to be batch order and batch size" + # Looping through a collection of records from the database # (using the Scoping::Named::ClassMethods.all method, for example) # is very inefficient since it will try to instantiate all the objects at once. @@ -31,6 +33,9 @@ module ActiveRecord # * :batch_size - Specifies the size of the batch. Default to 1000. # * :start - Specifies the primary key value to start from, inclusive of the value. # * :finish - Specifies the primary key value to end at, inclusive of the value. + # * :error_on_ignore - Overrides the application config to specify if an error should be raised when + # the order and limit have to be ignored due to batching. + # # This is especially useful if you want multiple workers dealing with # the same processing queue. You can make worker 1 handle all the records # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond @@ -48,13 +53,13 @@ module ActiveRecord # # NOTE: You can't set the limit either, that's used to control # the batch sizes. - def find_each(start: nil, finish: nil, batch_size: 1000) + def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil) if block_given? - find_in_batches(start: start, finish: finish, batch_size: batch_size) do |records| + find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do |records| records.each { |record| yield record } end else - enum_for(:find_each, start: start, finish: finish, batch_size: batch_size) do + enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do relation = self apply_limits(relation, start, finish).size end @@ -83,6 +88,9 @@ module ActiveRecord # * :batch_size - Specifies the size of the batch. Default to 1000. # * :start - Specifies the primary key value to start from, inclusive of the value. # * :finish - Specifies the primary key value to end at, inclusive of the value. + # * :error_on_ignore - Overrides the application config to specify if an error should be raised when + # the order and limit have to be ignored due to batching. + # # This is especially useful if you want multiple workers dealing with # the same processing queue. You can make worker 1 handle all the records # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond @@ -100,16 +108,16 @@ module ActiveRecord # # NOTE: You can't set the limit either, that's used to control # the batch sizes. - def find_in_batches(start: nil, finish: nil, batch_size: 1000) + def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil) relation = self unless block_given? - return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size) do + return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do total = apply_limits(relation, start, finish).size (total - 1).div(batch_size) + 1 end end - in_batches(of: batch_size, start: start, finish: finish, load: true) do |batch| + in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore) do |batch| yield batch.to_a end end @@ -140,6 +148,8 @@ module ActiveRecord # * :load - Specifies if the relation should be loaded. Default to false. # * :start - Specifies the primary key value to start from, inclusive of the value. # * :finish - Specifies the primary key value to end at, inclusive of the value. + # * :error_on_ignore - Overrides the application config to specify if an error should be raised when + # the order and limit have to be ignored due to batching. # # This is especially useful if you want to work with the # ActiveRecord::Relation object instead of the array of records, or if @@ -171,14 +181,14 @@ module ActiveRecord # # NOTE: You can't set the limit either, that's used to control the batch # sizes. - def in_batches(of: 1000, start: nil, finish: nil, load: false) + def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil) relation = self unless block_given? return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self) end - if logger && (arel.orders.present? || arel.taken.present?) - logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size") + if arel.orders.present? || arel.taken.present? + act_on_order_or_limit_ignored(error_on_ignore) end relation = relation.reorder(batch_order).limit(of) @@ -219,5 +229,15 @@ module ActiveRecord def batch_order "#{quoted_table_name}.#{quoted_primary_key} ASC" end + + def act_on_order_or_limit_ignored(error_on_ignore) + raise_error = (error_on_ignore.nil? ? self.klass.error_on_ignored_order_or_limit : error_on_ignore) + + if raise_error + raise ArgumentError.new(ORDER_OR_LIMIT_IGNORED_MESSAGE) + elsif logger + logger.warn(ORDER_OR_LIMIT_IGNORED_MESSAGE) + end + end end end -- cgit v1.2.3