Add inline failure reporting to test runner.

Any failures or errors will be reported inline during the run by default.
Skipped tests will be reported if run in verbose mode.

Any result is output with failure messages and a rerun snippet for that test.

Rerun snippets won't be output after a run, unless `--defer-output` is passed.

5 files changed, 113 insertions, 6 deletions

diff --git a/railties/CHANGELOG.md b/railties/CHANGELOG.md
index f007d11c72..36ac0951ca 100644
--- a/railties/CHANGELOG.md
+++ b/railties/CHANGELOG.md
@@ -1,3 +1,33 @@
+*   Add inline output to `bin/rails test`
+
+    Any failures or errors (and skips if running in verbose mode) are output
+    during a test run:
+
+    ```
+    # Running:
+
+    .....S..........................................F
+
+    This failed
+
+    bin/rails test test/models/bunny_test.rb:14
+
+    .................................E
+
+    ArgumentError: Wups! Bet you didn't expect this!
+        test/models/bunny_test.rb:19:in `block in <class:BunnyTest>'
+
+    bin/rails test test/models/bunny_test.rb:18
+
+    ....................
+
+    Finished in 0.069708s, 1477.6019 runs/s, 1448.9106 assertions/s.
+    ```
+
+    Output can be deferred to after a run with the `--defer-output` option.
+
+    *Kasper Timm Hansen*
+
 *   Fix displaying mailer previews on non local requests when config
     `action_mailer.show_previews` is set
 
diff --git a/railties/lib/rails/test_unit/minitest_plugin.rb b/railties/lib/rails/test_unit/minitest_plugin.rb
index 1923433c4a..b65d67b2a5 100644
--- a/railties/lib/rails/test_unit/minitest_plugin.rb
+++ b/railties/lib/rails/test_unit/minitest_plugin.rb
@@ -14,6 +14,8 @@ module Minitest
     opts.separator ""
     opts.separator "    bin/rails test test/controllers test/integration/login_test.rb"
     opts.separator ""
+    opts.separator "By default test failures and errors are reported inline during a run."
+    opts.separator ""
 
     opts.separator "Rails options:"
     opts.on("-e", "--environment ENV",
@@ -26,6 +28,11 @@ module Minitest
       options[:full_backtrace] = true
     end
 
+    opts.on("-d", "--defer-output",
+            "Output test failures and errors after the test run") do
+      options[:output_inline] = false
+    end
+
     options[:patterns] = opts.order!
   end
 
diff --git a/railties/lib/rails/test_unit/reporter.rb b/railties/lib/rails/test_unit/reporter.rb
index 09b8675cf8..c88e9aa60e 100644
--- a/railties/lib/rails/test_unit/reporter.rb
+++ b/railties/lib/rails/test_unit/reporter.rb
@@ -6,8 +6,21 @@ module Rails
     class_attribute :executable
     self.executable = "bin/rails test"
 
+    def record(result)
+      super
+
+      if output_inline? && result.failure && (!result.skipped? || options[:verbose])
+        io.puts
+        io.puts
+        io.puts result.failures.map(&:message)
+        io.puts
+        io.puts format_rerun_snippet(result)
+        io.puts
+      end
+    end
+
     def report
-      return if filtered_results.empty?
+      return if output_inline? || filtered_results.empty?
       io.puts
       io.puts "Failed tests:"
       io.puts
@@ -15,10 +28,7 @@ module Rails
     end
 
     def aggregated_results # :nodoc:
-      filtered_results.map do |result|
-        location, line = result.method(result.name).source_location
-        "#{self.executable} #{relative_path_for(location)}:#{line}"
-      end.join "\n"
+      filtered_results.map { |result| format_rerun_snippet(result) }.join "\n"
     end
 
     def filtered_results
@@ -32,5 +42,15 @@ module Rails
     def relative_path_for(file)
       file.sub(/^#{Rails.root}\/?/, '')
     end
+
+    private
+      def output_inline?
+        options.fetch(:output_inline, true)
+      end
+
+      def format_rerun_snippet(result)
+        location, line = result.method(result.name).source_location
+        "#{self.executable} #{relative_path_for(location)}:#{line}"
+      end
   end
 end
diff --git a/railties/test/application/test_runner_test.rb b/railties/test/application/test_runner_test.rb
index 2d47a31826..4e747b5253 100644
--- a/railties/test/application/test_runner_test.rb
+++ b/railties/test/application/test_runner_test.rb
@@ -340,6 +340,21 @@ module ApplicationTests
       assert_match '0 runs, 0 assertions', run_test_command('')
     end
 
+    def test_output_inline_by_default
+      app_file 'test/models/post_test.rb', <<-RUBY
+        require 'test_helper'
+
+        class PostTest < ActiveSupport::TestCase
+          def test_post
+            assert false, 'wups!'
+          end
+        end
+      RUBY
+
+      output = run_test_command('test/models/post_test.rb')
+      assert_match %r{Running:\n\nF\n\nwups!\n\nbin/rails test test/models/post_test.rb:4}, output
+    end
+
     def test_raise_error_when_specified_file_does_not_exist
       error = capture(:stderr) { run_test_command('test/not_exists.rb') }
       assert_match(%r{cannot load such file.+test/not_exists\.rb}, error)
diff --git a/railties/test/test_unit/reporter_test.rb b/railties/test/test_unit/reporter_test.rb
index 3066ba82d6..b1dd8b5284 100644
--- a/railties/test/test_unit/reporter_test.rb
+++ b/railties/test/test_unit/reporter_test.rb
@@ -57,6 +57,35 @@ class TestUnitReporterTest < ActiveSupport::TestCase
     end
   end
 
+  test "outputs failures inline" do
+    @reporter.record(failed_test)
+    @reporter.report
+
+    assert_match %r{\A\n\nboo\n\nbin/rails test .*test/test_unit/reporter_test.rb:6\n\n\z}, @output.string
+  end
+
+  test "outputs errors inline" do
+    @reporter.record(errored_test)
+    @reporter.report
+
+    assert_match %r{\A\n\nArgumentError: wups\n    No backtrace\n\nbin/rails test .*test/test_unit/reporter_test.rb:6\n\n\z}, @output.string
+  end
+
+  test "outputs skipped tests inline if verbose" do
+    verbose = Rails::TestUnitReporter.new @output, verbose: true
+    verbose.record(skipped_test)
+    verbose.report
+
+    assert_match %r{\A\n\nskipchurches, misstemples\n\nbin/rails test .*test/test_unit/reporter_test.rb:6\n\n\z}, @output.string
+  end
+
+  test "does not output rerun snippets after run" do
+    @reporter.record(failed_test)
+    @reporter.report
+
+    assert_no_match 'Failed tests:', @output.string
+  end
+
   private
   def assert_rerun_snippet_count(snippet_count)
     assert_equal snippet_count, @output.string.scan(%r{^bin/rails test }).size
@@ -72,6 +101,12 @@ class TestUnitReporterTest < ActiveSupport::TestCase
     ft
   end
 
+  def errored_test
+    et = ExampleTest.new(:woot)
+    et.failures << Minitest::UnexpectedError.new(ArgumentError.new("wups"))
+    et
+  end
+
   def passing_test
     ExampleTest.new(:woot)
   end
@@ -79,7 +114,7 @@ class TestUnitReporterTest < ActiveSupport::TestCase
   def skipped_test
     st = ExampleTest.new(:woot)
     st.failures << begin
-                     raise Minitest::Skip
+                     raise Minitest::Skip, "skipchurches, misstemples"
                    rescue Minitest::Assertion => e
                      e
                    end