Merge pull request #21430 from kaspth/test-runner-fail-fast

Add fail fast reporting to test runner.

5 files changed, 197 insertions, 6 deletions

diff --git a/railties/CHANGELOG.md b/railties/CHANGELOG.md
index f007d11c72..80ef1af7b5 100644
--- a/railties/CHANGELOG.md
+++ b/railties/CHANGELOG.md
@@ -1,3 +1,65 @@
+*   Add fail fast to `bin/rails test`
+
+    Adding `--fail-fast` or `-f` when running tests will interrupt the run on
+    the first failure:
+
+    ```
+    # Running:
+
+    ................................................S......E
+
+    ArgumentError: Wups! Bet you didn't expect this!
+        test/models/bunny_test.rb:19:in `block in <class:BunnyTest>'
+
+    bin/rails test test/models/bunny_test.rb:18
+
+    ....................................F
+
+    This failed
+
+    bin/rails test test/models/bunny_test.rb:14
+
+    Interrupted. Exiting...
+
+
+    Finished in 0.051427s, 1808.3872 runs/s, 1769.4972 assertions/s.
+
+    ```
+
+    Note that any unexpected errors don't abort the run.
+
+    *Kasper Timm Hansen*
+
+*   Add inline output to `bin/rails test`
+
+    Any failures or errors (and skips if running in verbose mode) are output
+    during a test run:
+
+    ```
+    # Running:
+
+    .....S..........................................F
+
+    This failed
+
+    bin/rails test test/models/bunny_test.rb:14
+
+    .................................E
+
+    ArgumentError: Wups! Bet you didn't expect this!
+        test/models/bunny_test.rb:19:in `block in <class:BunnyTest>'
+
+    bin/rails test test/models/bunny_test.rb:18
+
+    ....................
+
+    Finished in 0.069708s, 1477.6019 runs/s, 1448.9106 assertions/s.
+    ```
+
+    Output can be deferred to after a run with the `--defer-output` option.
+
+    *Kasper Timm Hansen*
+
 *   Fix displaying mailer previews on non local requests when config
     `action_mailer.show_previews` is set
 
diff --git a/railties/lib/rails/test_unit/minitest_plugin.rb b/railties/lib/rails/test_unit/minitest_plugin.rb
index 1923433c4a..3a0a58df88 100644
--- a/railties/lib/rails/test_unit/minitest_plugin.rb
+++ b/railties/lib/rails/test_unit/minitest_plugin.rb
@@ -14,6 +14,8 @@ module Minitest
     opts.separator ""
     opts.separator "    bin/rails test test/controllers test/integration/login_test.rb"
     opts.separator ""
+    opts.separator "By default test failures and errors are reported inline during a run."
+    opts.separator ""
 
     opts.separator "Rails options:"
     opts.on("-e", "--environment ENV",
@@ -26,6 +28,16 @@ module Minitest
       options[:full_backtrace] = true
     end
 
+    opts.on("-d", "--defer-output",
+            "Output test failures and errors after the test run") do
+      options[:output_inline] = false
+    end
+
+    opts.on("-f", "--fail-fast",
+            "Abort test run on first failure") do
+      options[:fail_fast] = true
+    end
+
     options[:patterns] = opts.order!
   end
 
diff --git a/railties/lib/rails/test_unit/reporter.rb b/railties/lib/rails/test_unit/reporter.rb
index 09b8675cf8..8f1116b6af 100644
--- a/railties/lib/rails/test_unit/reporter.rb
+++ b/railties/lib/rails/test_unit/reporter.rb
@@ -6,8 +6,25 @@ module Rails
     class_attribute :executable
     self.executable = "bin/rails test"
 
+    def record(result)
+      super
+
+      if output_inline? && result.failure && (!result.skipped? || options[:verbose])
+        io.puts
+        io.puts
+        io.puts result.failures.map(&:message)
+        io.puts
+        io.puts format_rerun_snippet(result)
+        io.puts
+      end
+
+      if fail_fast? && result.failure && !result.error? && !result.skipped?
+        raise Interrupt
+      end
+    end
+
     def report
-      return if filtered_results.empty?
+      return if output_inline? || filtered_results.empty?
       io.puts
       io.puts "Failed tests:"
       io.puts
@@ -15,10 +32,7 @@ module Rails
     end
 
     def aggregated_results # :nodoc:
-      filtered_results.map do |result|
-        location, line = result.method(result.name).source_location
-        "#{self.executable} #{relative_path_for(location)}:#{line}"
-      end.join "\n"
+      filtered_results.map { |result| format_rerun_snippet(result) }.join "\n"
     end
 
     def filtered_results
@@ -32,5 +46,19 @@ module Rails
     def relative_path_for(file)
       file.sub(/^#{Rails.root}\/?/, '')
     end
+
+    private
+      def output_inline?
+        options.fetch(:output_inline, true)
+      end
+
+      def fail_fast?
+        options[:fail_fast]
+      end
+
+      def format_rerun_snippet(result)
+        location, line = result.method(result.name).source_location
+        "#{self.executable} #{relative_path_for(location)}:#{line}"
+      end
   end
 end
diff --git a/railties/test/application/test_runner_test.rb b/railties/test/application/test_runner_test.rb
index 2d47a31826..acfba21f1c 100644
--- a/railties/test/application/test_runner_test.rb
+++ b/railties/test/application/test_runner_test.rb
@@ -340,6 +340,36 @@ module ApplicationTests
       assert_match '0 runs, 0 assertions', run_test_command('')
     end
 
+    def test_output_inline_by_default
+      app_file 'test/models/post_test.rb', <<-RUBY
+        require 'test_helper'
+
+        class PostTest < ActiveSupport::TestCase
+          def test_post
+            assert false, 'wups!'
+          end
+        end
+      RUBY
+
+      output = run_test_command('test/models/post_test.rb')
+      assert_match %r{Running:\n\nF\n\nwups!\n\nbin/rails test test/models/post_test.rb:4}, output
+    end
+
+    def test_fail_fast
+      app_file 'test/models/post_test.rb', <<-RUBY
+        require 'test_helper'
+
+        class PostTest < ActiveSupport::TestCase
+          def test_post
+            assert false, 'wups!'
+          end
+        end
+      RUBY
+
+      assert_match(/Interrupt/,
+        capture(:stderr) { run_test_command('test/models/post_test.rb --fail-fast') })
+    end
+
     def test_raise_error_when_specified_file_does_not_exist
       error = capture(:stderr) { run_test_command('test/not_exists.rb') }
       assert_match(%r{cannot load such file.+test/not_exists\.rb}, error)
diff --git a/railties/test/test_unit/reporter_test.rb b/railties/test/test_unit/reporter_test.rb
index 3066ba82d6..59fdf4bc36 100644
--- a/railties/test/test_unit/reporter_test.rb
+++ b/railties/test/test_unit/reporter_test.rb
@@ -57,6 +57,59 @@ class TestUnitReporterTest < ActiveSupport::TestCase
     end
   end
 
+  test "outputs failures inline" do
+    @reporter.record(failed_test)
+    @reporter.report
+
+    assert_match %r{\A\n\nboo\n\nbin/rails test .*test/test_unit/reporter_test.rb:6\n\n\z}, @output.string
+  end
+
+  test "outputs errors inline" do
+    @reporter.record(errored_test)
+    @reporter.report
+
+    assert_match %r{\A\n\nArgumentError: wups\n    No backtrace\n\nbin/rails test .*test/test_unit/reporter_test.rb:6\n\n\z}, @output.string
+  end
+
+  test "outputs skipped tests inline if verbose" do
+    verbose = Rails::TestUnitReporter.new @output, verbose: true
+    verbose.record(skipped_test)
+    verbose.report
+
+    assert_match %r{\A\n\nskipchurches, misstemples\n\nbin/rails test .*test/test_unit/reporter_test.rb:6\n\n\z}, @output.string
+  end
+
+  test "does not output rerun snippets after run" do
+    @reporter.record(failed_test)
+    @reporter.report
+
+    assert_no_match 'Failed tests:', @output.string
+  end
+
+  test "fail fast interrupts run on failure" do
+    fail_fast = Rails::TestUnitReporter.new @output, fail_fast: true
+    interrupt_raised = false
+
+    # Minitest passes through Interrupt, catch it manually.
+    begin
+      fail_fast.record(failed_test)
+    rescue Interrupt
+      interrupt_raised = true
+    ensure
+      assert interrupt_raised, 'Expected Interrupt to be raised.'
+    end
+  end
+
+  test "fail fast does not interrupt run errors or skips" do
+    fail_fast = Rails::TestUnitReporter.new @output, fail_fast: true
+
+    fail_fast.record(errored_test)
+    assert_no_match 'Failed tests:', @output.string
+
+    fail_fast.record(skipped_test)
+    assert_no_match 'Failed tests:', @output.string
+  end
+
   private
   def assert_rerun_snippet_count(snippet_count)
     assert_equal snippet_count, @output.string.scan(%r{^bin/rails test }).size
@@ -72,6 +125,12 @@ class TestUnitReporterTest < ActiveSupport::TestCase
     ft
   end
 
+  def errored_test
+    et = ExampleTest.new(:woot)
+    et.failures << Minitest::UnexpectedError.new(ArgumentError.new("wups"))
+    et
+  end
+
   def passing_test
     ExampleTest.new(:woot)
   end
@@ -79,7 +138,7 @@ class TestUnitReporterTest < ActiveSupport::TestCase
   def skipped_test
     st = ExampleTest.new(:woot)
     st.failures << begin
-                     raise Minitest::Skip
+                     raise Minitest::Skip, "skipchurches, misstemples"
                    rescue Minitest::Assertion => e
                      e
                    end