Adapters can specify maximum number of ids they support in a list of expressions

(default is nil meaning unlimited but Oracle imposes a limit of 1000)
Limit is used to make multiple queries when preloading associated has_many or habtm records

3 files changed, 78 insertions, 5 deletions

diff --git a/activerecord/lib/active_record/association_preload.rb b/activerecord/lib/active_record/association_preload.rb
index 911a5155fd..a205549d60 100644
--- a/activerecord/lib/active_record/association_preload.rb
+++ b/activerecord/lib/active_record/association_preload.rb
@@ -193,13 +193,17 @@ module ActiveRecord
         conditions = "t0.#{reflection.primary_key_name} #{in_or_equals_for_ids(ids)}"
         conditions << append_conditions(reflection, preload_options)
 
-        associated_records = reflection.klass.unscoped.where([conditions, ids]).
+        associated_records_proxy = reflection.klass.unscoped.
             includes(options[:include]).
             joins("INNER JOIN #{connection.quote_table_name options[:join_table]} t0 ON #{reflection.klass.quoted_table_name}.#{reflection.klass.primary_key} = t0.#{reflection.association_foreign_key}").
             select("#{options[:select] || table_name+'.*'}, t0.#{reflection.primary_key_name} as the_parent_record_id").
-            order(options[:order]).to_a
+            order(options[:order])
 
-        set_association_collection_records(id_to_record_map, reflection.name, associated_records, 'the_parent_record_id')
+        all_associated_records = associated_records(ids) do |some_ids|
+          associated_records_proxy.where([conditions, ids]).to_a
+        end
+
+        set_association_collection_records(id_to_record_map, reflection.name, all_associated_records, 'the_parent_record_id')
       end
 
       def preload_has_one_association(records, reflection, preload_options={})
@@ -358,13 +362,14 @@ module ActiveRecord
         find_options = {
           :select => preload_options[:select] || options[:select] || Arel::SqlLiteral.new("#{table_name}.*"),
           :include => preload_options[:include] || options[:include],
-          :conditions => [conditions, ids],
           :joins => options[:joins],
           :group => preload_options[:group] || options[:group],
           :order => preload_options[:order] || options[:order]
         }
 
-        reflection.klass.scoped.apply_finder_options(find_options).to_a
+        associated_records(ids) do |some_ids|
+          reflection.klass.scoped.apply_finder_options(find_options.merge(:conditions => [conditions, some_ids])).to_a
+        end
       end
 
 
@@ -382,6 +387,17 @@ module ActiveRecord
       def in_or_equals_for_ids(ids)
         ids.size > 1 ? "IN (?)" : "= ?"
       end
+      
+      # Some databases impose a limit on the number of ids in a list (in Oracle its 1000)
+      # Make several smaller queries if necessary or make one query if the adapter supports it
+      def associated_records(ids)
+        max_ids_in_a_list = connection.ids_in_list_limit || ids.size
+        records = []
+        ids.each_slice(max_ids_in_a_list) do |some_ids|
+          records += yield(some_ids)
+        end
+        records
+      end
     end
   end
 end
diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb
index f3fba9a3a9..ada1560ce2 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb
@@ -91,6 +91,11 @@ module ActiveRecord
         false
       end
 
+      # Does this adapter restrict the number of ids you can use in a list. Oracle has a limit of 1000.
+      def ids_in_list_limit
+        nil
+      end
+
       # QUOTING ==================================================
 
       # Override to return the quoted table name. Defaults to column quoting.
diff --git a/activerecord/test/cases/associations/eager_test.rb b/activerecord/test/cases/associations/eager_test.rb
index 66fb5ac1e1..c532522e76 100644
--- a/activerecord/test/cases/associations/eager_test.rb
+++ b/activerecord/test/cases/associations/eager_test.rb
@@ -79,6 +79,58 @@ class EagerAssociationTest < ActiveRecord::TestCase
     end
   end
 
+  def test_preloading_has_many_in_multiple_queries_with_more_ids_than_database_can_handle
+    Post.connection.expects(:ids_in_list_limit).at_least_once.returns(5)
+    posts = Post.find(:all, :include=>:comments)
+    assert_equal 7, posts.size
+  end
+
+  def test_preloading_has_many_in_one_queries_when_database_has_no_limit_on_ids_it_can_handle
+    Post.connection.expects(:ids_in_list_limit).at_least_once.returns(nil)
+    posts = Post.find(:all, :include=>:comments)
+    assert_equal 7, posts.size
+  end
+
+  def test_preloading_habtm_in_multiple_queries_with_more_ids_than_database_can_handle
+    Post.connection.expects(:ids_in_list_limit).at_least_once.returns(5)
+    posts = Post.find(:all, :include=>:categories)
+    assert_equal 7, posts.size
+  end
+
+  def test_preloading_habtm_in_one_queries_when_database_has_no_limit_on_ids_it_can_handle
+    Post.connection.expects(:ids_in_list_limit).at_least_once.returns(nil)
+    posts = Post.find(:all, :include=>:categories)
+    assert_equal 7, posts.size
+  end
+  
+  def test_load_associated_records_in_one_query_when_adapter_has_no_limit
+    Post.connection.expects(:ids_in_list_limit).at_least_once.returns(nil)
+    Post.expects(:i_was_called).with([1,2,3,4,5,6,7]).returns([1])
+    associated_records = Post.send(:associated_records, [1,2,3,4,5,6,7]) do |some_ids|
+      Post.i_was_called(some_ids)
+    end
+    assert_equal [1], associated_records
+  end
+
+  def test_load_associated_records_in_several_queries_when_many_ids_passed
+    Post.connection.expects(:ids_in_list_limit).at_least_once.returns(5)
+    Post.expects(:i_was_called).with([1,2,3,4,5]).returns([1])
+    Post.expects(:i_was_called).with([6,7]).returns([6])
+    associated_records = Post.send(:associated_records, [1,2,3,4,5,6,7]) do |some_ids|
+      Post.i_was_called(some_ids)
+    end
+    assert_equal [1,6], associated_records
+  end
+
+  def test_load_associated_records_in_one_query_when_a_few_ids_passed
+    Post.connection.expects(:ids_in_list_limit).at_least_once.returns(5)
+    Post.expects(:i_was_called).with([1,2,3]).returns([1])
+    associated_records = Post.send(:associated_records, [1,2,3]) do |some_ids|
+      Post.i_was_called(some_ids)
+    end
+    assert_equal [1], associated_records
+  end
+
   def test_including_duplicate_objects_from_belongs_to
     popular_post = Post.create!(:title => 'foo', :body => "I like cars!")
     comment = popular_post.comments.create!(:body => "lol")