Default to sorting user's test cases for now

Goals:

1. Default to :random for newly generated applications
2. Default to :sorted for existing applications with a warning
3. Only show the warning once
4. Only show the warning if the app actually uses AS::TestCase

Fixes #16769

14 files changed, 120 insertions, 17 deletions

diff --git a/actionmailer/test/abstract_unit.rb b/actionmailer/test/abstract_unit.rb
index ce426cad1e..7681679dc7 100644
--- a/actionmailer/test/abstract_unit.rb
+++ b/actionmailer/test/abstract_unit.rb
@@ -46,4 +46,4 @@ require 'mocha/setup' # FIXME: stop using mocha
 # FIXME: we have tests that depend on run order, we should fix that and
 # remove this method call.
 require 'active_support/test_case'
-ActiveSupport::TestCase.my_tests_are_order_dependent!
+ActiveSupport::TestCase.test_order = :sorted
diff --git a/actionpack/test/abstract_unit.rb b/actionpack/test/abstract_unit.rb
index 799c17119d..69312e4c22 100644
--- a/actionpack/test/abstract_unit.rb
+++ b/actionpack/test/abstract_unit.rb
@@ -516,4 +516,4 @@ end
 # FIXME: we have tests that depend on run order, we should fix that and
 # remove this method call.
 require 'active_support/test_case'
-ActiveSupport::TestCase.my_tests_are_order_dependent!
+ActiveSupport::TestCase.test_order = :sorted
diff --git a/actionview/test/abstract_unit.rb b/actionview/test/abstract_unit.rb
index 923e637f11..e51461d82c 100644
--- a/actionview/test/abstract_unit.rb
+++ b/actionview/test/abstract_unit.rb
@@ -344,4 +344,4 @@ require 'mocha/setup' # FIXME: stop using mocha
 # FIXME: we have tests that depend on run order, we should fix that and
 # remove this method call.
 require 'active_support/test_case'
-ActiveSupport::TestCase.my_tests_are_order_dependent!
+ActiveSupport::TestCase.test_order = :sorted
diff --git a/activemodel/test/cases/helper.rb b/activemodel/test/cases/helper.rb
index 5e80353836..4ce6103593 100644
--- a/activemodel/test/cases/helper.rb
+++ b/activemodel/test/cases/helper.rb
@@ -17,4 +17,4 @@ require 'mocha/setup' # FIXME: stop using mocha
 # FIXME: we have tests that depend on run order, we should fix that and
 # remove this method call.
 require 'active_support/test_case'
-ActiveSupport::TestCase.my_tests_are_order_dependent!
+ActiveSupport::TestCase.test_order = :sorted
diff --git a/activerecord/test/cases/helper.rb b/activerecord/test/cases/helper.rb
index 3d9328b198..be635aeef9 100644
--- a/activerecord/test/cases/helper.rb
+++ b/activerecord/test/cases/helper.rb
@@ -220,4 +220,4 @@ require 'mocha/setup' # FIXME: stop using mocha
 # FIXME: we have tests that depend on run order, we should fix that and
 # remove this method call.
 require 'active_support/test_case'
-ActiveSupport::TestCase.my_tests_are_order_dependent!
+ActiveSupport::TestCase.test_order = :sorted
diff --git a/activesupport/CHANGELOG.md b/activesupport/CHANGELOG.md
index f995082a15..0e5a28e3fc 100644
--- a/activesupport/CHANGELOG.md
+++ b/activesupport/CHANGELOG.md
@@ -1,3 +1,9 @@
+*   Introduced new configuration option `active_support.test_order` for
+    specifying the order test cases are executed. This option currently defaults
+    to `:sorted` but will be changed to `:random` in Rails 5.0.
+
+    *Akira Matsuda*, *Godfrey Chan*
+
 *   Fixed a bug in Inflector#underscore where acroynms in nested constant names
     are incorrectly parsed as camelCase.
 
@@ -43,12 +49,6 @@
 
     *DHH*
 
-*   Fix ActiveSupport::TestCase not to order users' test cases by default.
-    If this change breaks your tests because your tests are order dependent, you need to explicitly call
-    ActiveSupport::TestCase.my_tests_are_order_dependent! at the top of your tests.
-
-    *Akira Matsuda*
-
 *   Fix DateTime comparison with DateTime::Infinity object.
 
     *Rafael Mendonça França*
diff --git a/activesupport/lib/active_support/test_case.rb b/activesupport/lib/active_support/test_case.rb
index 0df599b692..33139320fa 100644
--- a/activesupport/lib/active_support/test_case.rb
+++ b/activesupport/lib/active_support/test_case.rb
@@ -12,10 +12,42 @@ require 'active_support/core_ext/kernel/reporting'
 require 'active_support/deprecation'
 
 module ActiveSupport
+  class << self
+    delegate :test_order, :test_order=, to: :'ActiveSupport::TestCase'
+  end
+
   class TestCase < ::Minitest::Test
     Assertion = Minitest::Assertion
 
+    @@test_order = nil
+
     class << self
+      def test_order=(new_order)
+        @@test_order = new_order
+      end
+
+      def test_order
+        if @@test_order.nil?
+          ActiveSupport::Deprecation.warn "You did not specify a value for the " \
+            "configuration option 'active_support.test_order'. In Rails 5.0, " \
+            "the default value of this option will change from `:sorted` to " \
+            "`:random`.\n" \
+            "To disable this warning and keep the current behavior, you can add " \
+            "the following line to your `config/environments/test.rb`:\n" \
+            "\n" \
+            "  Rails.application.configure do\n" \
+            "    config.active_support.test_order = :sorted\n" \
+            "  end\n" \
+            "\n" \
+            "Alternatively, you can opt into the future behavior by setting this " \
+            "option to `:random`."
+
+          @@test_order = :sorted
+        end
+
+        @@test_order
+      end
+
       alias :my_tests_are_order_dependent! :i_suck_and_my_tests_are_order_dependent!
     end
 
diff --git a/activesupport/test/abstract_unit.rb b/activesupport/test/abstract_unit.rb
index 52fbaf8a85..f65ec962f9 100644
--- a/activesupport/test/abstract_unit.rb
+++ b/activesupport/test/abstract_unit.rb
@@ -42,4 +42,4 @@ require 'mocha/setup' # FIXME: stop using mocha
 # FIXME: we have tests that depend on run order, we should fix that and
 # remove this method call.
 require 'active_support/test_case'
-ActiveSupport::TestCase.my_tests_are_order_dependent!
+ActiveSupport::TestCase.test_order = :sorted
diff --git a/activesupport/test/test_case_test.rb b/activesupport/test/test_case_test.rb
index c93192f207..5e852c8050 100644
--- a/activesupport/test/test_case_test.rb
+++ b/activesupport/test/test_case_test.rb
@@ -172,3 +172,50 @@ class TestCaseTaggedLoggingTest < ActiveSupport::TestCase
     assert_match "#{self.class}: #{name}\n", @out.string
   end
 end
+
+class TestOrderTest < ActiveSupport::TestCase
+  def setup
+    @original_test_order = ActiveSupport::TestCase.test_order
+  end
+
+  def teardown
+    ActiveSupport::TestCase.test_order = @original_test_order
+  end
+
+  def test_defaults_to_sorted_with_warning
+    ActiveSupport::TestCase.test_order = nil
+
+    assert_equal :sorted, assert_deprecated { ActiveSupport::TestCase.test_order }
+
+    # It should only produce a deprecation warning the first time this is accessed
+    assert_equal :sorted, assert_not_deprecated { ActiveSupport::TestCase.test_order }
+    assert_equal :sorted, assert_not_deprecated { ActiveSupport.test_order }
+  end
+
+  def test_test_order_is_global
+    ActiveSupport::TestCase.test_order = :random
+
+    assert_equal :random, ActiveSupport.test_order
+    assert_equal :random, ActiveSupport::TestCase.test_order
+    assert_equal :random, self.class.test_order
+    assert_equal :random, Class.new(ActiveSupport::TestCase).test_order
+
+    ActiveSupport.test_order = :sorted
+
+    assert_equal :sorted, ActiveSupport.test_order
+    assert_equal :sorted, ActiveSupport::TestCase.test_order
+    assert_equal :sorted, self.class.test_order
+    assert_equal :sorted, Class.new(ActiveSupport::TestCase).test_order
+  end
+
+  def test_i_suck_and_my_tests_are_order_dependent!
+    ActiveSupport::TestCase.test_order = :random
+
+    klass = Class.new(ActiveSupport::TestCase) do
+      i_suck_and_my_tests_are_order_dependent!
+    end
+
+    assert_equal :alpha, klass.test_order
+    assert_equal :random, ActiveSupport::TestCase.test_order
+  end
+end
diff --git a/guides/source/4_2_release_notes.md b/guides/source/4_2_release_notes.md
index dad973cf5e..cb461feff8 100644
--- a/guides/source/4_2_release_notes.md
+++ b/guides/source/4_2_release_notes.md
@@ -688,13 +688,14 @@ Please refer to the [Changelog][active-support] for detailed changes.
 
 ### Notable changes
 
+*   Introduced new configuration option `active_support.test_order` for
+    specifying the order test cases are executed. This option currently defaults
+    to `:sorted` but will be changed to `:random` in Rails 5.0.
+    ([Commit](TODO: fill me in))
+
 *   The `travel_to` test helper now truncates the `usec` component to 0.
     ([Commit](https://github.com/rails/rails/commit/9f6e82ee4783e491c20f5244a613fdeb4024beb5))
 
-*   `ActiveSupport::TestCase` now randomizes the order that test cases are ran
-    by default.
-    ([Commit](https://github.com/rails/rails/commit/6ffb29d24e05abbd9ffe3ea974140d6c70221807))
-
 *   Introduced `Object#itself` as an identity function.
     (Commit [1](https://github.com/rails/rails/commit/702ad710b57bef45b081ebf42e6fa70820fdd810),
     [2](https://github.com/rails/rails/commit/64d91122222c11ad3918cc8e2e3ebc4b0a03448a))
diff --git a/guides/source/configuring.md b/guides/source/configuring.md
index dbbd0c1aea..667281d1aa 100644
--- a/guides/source/configuring.md
+++ b/guides/source/configuring.md
@@ -471,6 +471,8 @@ There are a few configuration options available in Active Support:
 
 * `config.active_support.bare` enables or disables the loading of `active_support/all` when booting Rails. Defaults to `nil`, which means `active_support/all` is loaded.
 
+* `config.active_support.test_order` sets the order that test cases are executed. Possible values are `:sorted` and `:random`. Currently defaults to `:sorted`. In Rails 5.0, the default will be changed to `:random` instead.
+
 * `config.active_support.escape_html_entities_in_json` enables or disables the escaping of HTML entities in JSON serialization. Defaults to `false`.
 
 * `config.active_support.use_standard_json_time_format` enables or disables serializing dates to ISO 8601 format. Defaults to `true`.
diff --git a/guides/source/upgrading_ruby_on_rails.md b/guides/source/upgrading_ruby_on_rails.md
index 989d8400e5..71358ad3f5 100644
--- a/guides/source/upgrading_ruby_on_rails.md
+++ b/guides/source/upgrading_ruby_on_rails.md
@@ -65,6 +65,24 @@ TODO: mention https://github.com/rails/rails/pull/16526
 
 TODO: mention https://github.com/rails/rails/pull/16537
 
+### Ordering of test cases
+
+In Rails 5.0, test cases will be executed in random order by default. In
+anticipation of this change, Rails 4.2 introduced a new configuration option
+`active_support.test_order` for explicitly specifying the test ordering. This
+allows you to either locking down the current behavior by setting the option to
+`:sorted`, or opt into the future behavior by setting the option to `:random`.
+
+If you do not specify a value for this option, a deprecation warning will be
+emitted. To avoid this, add the following line to your test environment:
+
+```ruby
+# config/environments/test.rb
+Rails.application.configure do
+  config.active_support.test_order = :sorted # or `:random` if you prefer
+end
+```
+
 ### Serialized attributes
 
 When using a custom coder (e.g. `serialize :metadata, JSON`),
diff --git a/railties/lib/rails/generators/rails/app/templates/config/environments/test.rb.tt b/railties/lib/rails/generators/rails/app/templates/config/environments/test.rb.tt
index 053f5b66d7..32756eb88b 100644
--- a/railties/lib/rails/generators/rails/app/templates/config/environments/test.rb.tt
+++ b/railties/lib/rails/generators/rails/app/templates/config/environments/test.rb.tt
@@ -31,6 +31,9 @@ Rails.application.configure do
   # ActionMailer::Base.deliveries array.
   config.action_mailer.delivery_method = :test
 
+  # Randomize the order test cases are executed
+  config.active_support.test_order = :random
+
   # Print deprecation notices to the stderr.
   config.active_support.deprecation = :stderr
 
diff --git a/railties/test/abstract_unit.rb b/railties/test/abstract_unit.rb
index d8800eaa0f..3810f0e14f 100644
--- a/railties/test/abstract_unit.rb
+++ b/railties/test/abstract_unit.rb
@@ -30,7 +30,7 @@ end
 class ActiveSupport::TestCase
   # FIXME: we have tests that depend on run order, we should fix that and
   # remove this method call.
-  self.my_tests_are_order_dependent!
+  self.test_order = :sorted
 
   private