Use `-=` to change the update the collection on the association

This also mark the association as loaded given we changed it in memory
and avoid the next access to the reader to make a query to the databse.

2 files changed, 2 insertions, 3 deletions

diff --git a/activerecord/lib/active_record/associations/collection_association.rb b/activerecord/lib/active_record/associations/collection_association.rb
index aa668fa4b4..9a90cd2cc7 100644
--- a/activerecord/lib/active_record/associations/collection_association.rb
+++ b/activerecord/lib/active_record/associations/collection_association.rb
@@ -400,8 +400,7 @@ module ActiveRecord
           records.each { |record| callback(:before_remove, record) }
 
           delete_records(existing_records, method) if existing_records.any?
-          hashed_records = records.group_by { |record| record }
-          target.select! { |record| !hashed_records[record] }
+          self.target -= records
 
           records.each { |record| callback(:after_remove, record) }
         end
diff --git a/activerecord/test/cases/associations/has_many_through_associations_test.rb b/activerecord/test/cases/associations/has_many_through_associations_test.rb
index d5573b6d02..259e4eb4b7 100644
--- a/activerecord/test/cases/associations/has_many_through_associations_test.rb
+++ b/activerecord/test/cases/associations/has_many_through_associations_test.rb
@@ -357,7 +357,7 @@ class HasManyThroughAssociationsTest < ActiveRecord::TestCase
       posts(:welcome).people.delete(people(:michael))
     end
 
-    assert_queries(1) do
+    assert_queries(0) do
       assert_empty posts(:welcome).people
     end