Default to sorting user's test cases for now

Goals:

1. Default to :random for newly generated applications
2. Default to :sorted for existing applications with a warning
3. Only show the warning once
4. Only show the warning if the app actually uses AS::TestCase

Fixes #16769

4 files changed, 86 insertions, 7 deletions

diff --git a/activesupport/CHANGELOG.md b/activesupport/CHANGELOG.md
index f995082a15..0e5a28e3fc 100644
--- a/activesupport/CHANGELOG.md
+++ b/activesupport/CHANGELOG.md
@@ -1,3 +1,9 @@
+*   Introduced new configuration option `active_support.test_order` for
+    specifying the order test cases are executed. This option currently defaults
+    to `:sorted` but will be changed to `:random` in Rails 5.0.
+
+    *Akira Matsuda*, *Godfrey Chan*
+
 *   Fixed a bug in Inflector#underscore where acroynms in nested constant names
     are incorrectly parsed as camelCase.
 
@@ -43,12 +49,6 @@
 
     *DHH*
 
-*   Fix ActiveSupport::TestCase not to order users' test cases by default.
-    If this change breaks your tests because your tests are order dependent, you need to explicitly call
-    ActiveSupport::TestCase.my_tests_are_order_dependent! at the top of your tests.
-
-    *Akira Matsuda*
-
 *   Fix DateTime comparison with DateTime::Infinity object.
 
     *Rafael Mendonça França*
diff --git a/activesupport/lib/active_support/test_case.rb b/activesupport/lib/active_support/test_case.rb
index 0df599b692..33139320fa 100644
--- a/activesupport/lib/active_support/test_case.rb
+++ b/activesupport/lib/active_support/test_case.rb
@@ -12,10 +12,42 @@ require 'active_support/core_ext/kernel/reporting'
 require 'active_support/deprecation'
 
 module ActiveSupport
+  class << self
+    delegate :test_order, :test_order=, to: :'ActiveSupport::TestCase'
+  end
+
   class TestCase < ::Minitest::Test
     Assertion = Minitest::Assertion
 
+    @@test_order = nil
+
     class << self
+      def test_order=(new_order)
+        @@test_order = new_order
+      end
+
+      def test_order
+        if @@test_order.nil?
+          ActiveSupport::Deprecation.warn "You did not specify a value for the " \
+            "configuration option 'active_support.test_order'. In Rails 5.0, " \
+            "the default value of this option will change from `:sorted` to " \
+            "`:random`.\n" \
+            "To disable this warning and keep the current behavior, you can add " \
+            "the following line to your `config/environments/test.rb`:\n" \
+            "\n" \
+            "  Rails.application.configure do\n" \
+            "    config.active_support.test_order = :sorted\n" \
+            "  end\n" \
+            "\n" \
+            "Alternatively, you can opt into the future behavior by setting this " \
+            "option to `:random`."
+
+          @@test_order = :sorted
+        end
+
+        @@test_order
+      end
+
       alias :my_tests_are_order_dependent! :i_suck_and_my_tests_are_order_dependent!
     end
 
diff --git a/activesupport/test/abstract_unit.rb b/activesupport/test/abstract_unit.rb
index 52fbaf8a85..f65ec962f9 100644
--- a/activesupport/test/abstract_unit.rb
+++ b/activesupport/test/abstract_unit.rb
@@ -42,4 +42,4 @@ require 'mocha/setup' # FIXME: stop using mocha
 # FIXME: we have tests that depend on run order, we should fix that and
 # remove this method call.
 require 'active_support/test_case'
-ActiveSupport::TestCase.my_tests_are_order_dependent!
+ActiveSupport::TestCase.test_order = :sorted
diff --git a/activesupport/test/test_case_test.rb b/activesupport/test/test_case_test.rb
index c93192f207..5e852c8050 100644
--- a/activesupport/test/test_case_test.rb
+++ b/activesupport/test/test_case_test.rb
@@ -172,3 +172,50 @@ class TestCaseTaggedLoggingTest < ActiveSupport::TestCase
     assert_match "#{self.class}: #{name}\n", @out.string
   end
 end
+
+class TestOrderTest < ActiveSupport::TestCase
+  def setup
+    @original_test_order = ActiveSupport::TestCase.test_order
+  end
+
+  def teardown
+    ActiveSupport::TestCase.test_order = @original_test_order
+  end
+
+  def test_defaults_to_sorted_with_warning
+    ActiveSupport::TestCase.test_order = nil
+
+    assert_equal :sorted, assert_deprecated { ActiveSupport::TestCase.test_order }
+
+    # It should only produce a deprecation warning the first time this is accessed
+    assert_equal :sorted, assert_not_deprecated { ActiveSupport::TestCase.test_order }
+    assert_equal :sorted, assert_not_deprecated { ActiveSupport.test_order }
+  end
+
+  def test_test_order_is_global
+    ActiveSupport::TestCase.test_order = :random
+
+    assert_equal :random, ActiveSupport.test_order
+    assert_equal :random, ActiveSupport::TestCase.test_order
+    assert_equal :random, self.class.test_order
+    assert_equal :random, Class.new(ActiveSupport::TestCase).test_order
+
+    ActiveSupport.test_order = :sorted
+
+    assert_equal :sorted, ActiveSupport.test_order
+    assert_equal :sorted, ActiveSupport::TestCase.test_order
+    assert_equal :sorted, self.class.test_order
+    assert_equal :sorted, Class.new(ActiveSupport::TestCase).test_order
+  end
+
+  def test_i_suck_and_my_tests_are_order_dependent!
+    ActiveSupport::TestCase.test_order = :random
+
+    klass = Class.new(ActiveSupport::TestCase) do
+      i_suck_and_my_tests_are_order_dependent!
+    end
+
+    assert_equal :alpha, klass.test_order
+    assert_equal :random, ActiveSupport::TestCase.test_order
+  end
+end