From 64a3b09b408400c94fd25f22c2ce791a8d90b5cb Mon Sep 17 00:00:00 2001 From: Kasper Timm Hansen Date: Sun, 30 Aug 2015 12:33:37 +0200 Subject: Add inline failure reporting to test runner. Any failures or errors will be reported inline during the run by default. Skipped tests will be reported if run in verbose mode. Any result is output with failure messages and a rerun snippet for that test. Rerun snippets won't be output after a run, unless `--defer-output` is passed. --- railties/CHANGELOG.md | 30 ++++++++++++++++++++ railties/lib/rails/test_unit/minitest_plugin.rb | 7 +++++ railties/lib/rails/test_unit/reporter.rb | 30 ++++++++++++++++---- railties/test/application/test_runner_test.rb | 15 ++++++++++ railties/test/test_unit/reporter_test.rb | 37 ++++++++++++++++++++++++- 5 files changed, 113 insertions(+), 6 deletions(-) (limited to 'railties') diff --git a/railties/CHANGELOG.md b/railties/CHANGELOG.md index f007d11c72..36ac0951ca 100644 --- a/railties/CHANGELOG.md +++ b/railties/CHANGELOG.md @@ -1,3 +1,33 @@ +* Add inline output to `bin/rails test` + + Any failures or errors (and skips if running in verbose mode) are output + during a test run: + + ``` + # Running: + + .....S..........................................F + + This failed + + bin/rails test test/models/bunny_test.rb:14 + + .................................E + + ArgumentError: Wups! Bet you didn't expect this! + test/models/bunny_test.rb:19:in `block in ' + + bin/rails test test/models/bunny_test.rb:18 + + .................... + + Finished in 0.069708s, 1477.6019 runs/s, 1448.9106 assertions/s. + ``` + + Output can be deferred to after a run with the `--defer-output` option. + + *Kasper Timm Hansen* + * Fix displaying mailer previews on non local requests when config `action_mailer.show_previews` is set diff --git a/railties/lib/rails/test_unit/minitest_plugin.rb b/railties/lib/rails/test_unit/minitest_plugin.rb index 1923433c4a..b65d67b2a5 100644 --- a/railties/lib/rails/test_unit/minitest_plugin.rb +++ b/railties/lib/rails/test_unit/minitest_plugin.rb @@ -14,6 +14,8 @@ module Minitest opts.separator "" opts.separator " bin/rails test test/controllers test/integration/login_test.rb" opts.separator "" + opts.separator "By default test failures and errors are reported inline during a run." + opts.separator "" opts.separator "Rails options:" opts.on("-e", "--environment ENV", @@ -26,6 +28,11 @@ module Minitest options[:full_backtrace] = true end + opts.on("-d", "--defer-output", + "Output test failures and errors after the test run") do + options[:output_inline] = false + end + options[:patterns] = opts.order! end diff --git a/railties/lib/rails/test_unit/reporter.rb b/railties/lib/rails/test_unit/reporter.rb index 09b8675cf8..c88e9aa60e 100644 --- a/railties/lib/rails/test_unit/reporter.rb +++ b/railties/lib/rails/test_unit/reporter.rb @@ -6,8 +6,21 @@ module Rails class_attribute :executable self.executable = "bin/rails test" + def record(result) + super + + if output_inline? && result.failure && (!result.skipped? || options[:verbose]) + io.puts + io.puts + io.puts result.failures.map(&:message) + io.puts + io.puts format_rerun_snippet(result) + io.puts + end + end + def report - return if filtered_results.empty? + return if output_inline? || filtered_results.empty? io.puts io.puts "Failed tests:" io.puts @@ -15,10 +28,7 @@ module Rails end def aggregated_results # :nodoc: - filtered_results.map do |result| - location, line = result.method(result.name).source_location - "#{self.executable} #{relative_path_for(location)}:#{line}" - end.join "\n" + filtered_results.map { |result| format_rerun_snippet(result) }.join "\n" end def filtered_results @@ -32,5 +42,15 @@ module Rails def relative_path_for(file) file.sub(/^#{Rails.root}\/?/, '') end + + private + def output_inline? + options.fetch(:output_inline, true) + end + + def format_rerun_snippet(result) + location, line = result.method(result.name).source_location + "#{self.executable} #{relative_path_for(location)}:#{line}" + end end end diff --git a/railties/test/application/test_runner_test.rb b/railties/test/application/test_runner_test.rb index 2d47a31826..4e747b5253 100644 --- a/railties/test/application/test_runner_test.rb +++ b/railties/test/application/test_runner_test.rb @@ -340,6 +340,21 @@ module ApplicationTests assert_match '0 runs, 0 assertions', run_test_command('') end + def test_output_inline_by_default + app_file 'test/models/post_test.rb', <<-RUBY + require 'test_helper' + + class PostTest < ActiveSupport::TestCase + def test_post + assert false, 'wups!' + end + end + RUBY + + output = run_test_command('test/models/post_test.rb') + assert_match %r{Running:\n\nF\n\nwups!\n\nbin/rails test test/models/post_test.rb:4}, output + end + def test_raise_error_when_specified_file_does_not_exist error = capture(:stderr) { run_test_command('test/not_exists.rb') } assert_match(%r{cannot load such file.+test/not_exists\.rb}, error) diff --git a/railties/test/test_unit/reporter_test.rb b/railties/test/test_unit/reporter_test.rb index 3066ba82d6..b1dd8b5284 100644 --- a/railties/test/test_unit/reporter_test.rb +++ b/railties/test/test_unit/reporter_test.rb @@ -57,6 +57,35 @@ class TestUnitReporterTest < ActiveSupport::TestCase end end + test "outputs failures inline" do + @reporter.record(failed_test) + @reporter.report + + assert_match %r{\A\n\nboo\n\nbin/rails test .*test/test_unit/reporter_test.rb:6\n\n\z}, @output.string + end + + test "outputs errors inline" do + @reporter.record(errored_test) + @reporter.report + + assert_match %r{\A\n\nArgumentError: wups\n No backtrace\n\nbin/rails test .*test/test_unit/reporter_test.rb:6\n\n\z}, @output.string + end + + test "outputs skipped tests inline if verbose" do + verbose = Rails::TestUnitReporter.new @output, verbose: true + verbose.record(skipped_test) + verbose.report + + assert_match %r{\A\n\nskipchurches, misstemples\n\nbin/rails test .*test/test_unit/reporter_test.rb:6\n\n\z}, @output.string + end + + test "does not output rerun snippets after run" do + @reporter.record(failed_test) + @reporter.report + + assert_no_match 'Failed tests:', @output.string + end + private def assert_rerun_snippet_count(snippet_count) assert_equal snippet_count, @output.string.scan(%r{^bin/rails test }).size @@ -72,6 +101,12 @@ class TestUnitReporterTest < ActiveSupport::TestCase ft end + def errored_test + et = ExampleTest.new(:woot) + et.failures << Minitest::UnexpectedError.new(ArgumentError.new("wups")) + et + end + def passing_test ExampleTest.new(:woot) end @@ -79,7 +114,7 @@ class TestUnitReporterTest < ActiveSupport::TestCase def skipped_test st = ExampleTest.new(:woot) st.failures << begin - raise Minitest::Skip + raise Minitest::Skip, "skipchurches, misstemples" rescue Minitest::Assertion => e e end -- cgit v1.2.3 From 2310fb9d810f11681c1eecdb88518c23c8b379cf Mon Sep 17 00:00:00 2001 From: Kasper Timm Hansen Date: Mon, 28 Sep 2015 20:27:30 +0200 Subject: Add fail fast to test runner. Passing `--fail-fast` to the test runner will now abort the test run on the first failure. The run continues on any unexpected errors. --- railties/CHANGELOG.md | 32 +++++++++++++++++++++++++ railties/lib/rails/test_unit/minitest_plugin.rb | 5 ++++ railties/lib/rails/test_unit/reporter.rb | 8 +++++++ railties/test/application/test_runner_test.rb | 15 ++++++++++++ railties/test/test_unit/reporter_test.rb | 24 +++++++++++++++++++ 5 files changed, 84 insertions(+) (limited to 'railties') diff --git a/railties/CHANGELOG.md b/railties/CHANGELOG.md index 36ac0951ca..80ef1af7b5 100644 --- a/railties/CHANGELOG.md +++ b/railties/CHANGELOG.md @@ -1,3 +1,35 @@ +* Add fail fast to `bin/rails test` + + Adding `--fail-fast` or `-f` when running tests will interrupt the run on + the first failure: + + ``` + # Running: + + ................................................S......E + + ArgumentError: Wups! Bet you didn't expect this! + test/models/bunny_test.rb:19:in `block in ' + + bin/rails test test/models/bunny_test.rb:18 + + ....................................F + + This failed + + bin/rails test test/models/bunny_test.rb:14 + + Interrupted. Exiting... + + + Finished in 0.051427s, 1808.3872 runs/s, 1769.4972 assertions/s. + + ``` + + Note that any unexpected errors don't abort the run. + + *Kasper Timm Hansen* + * Add inline output to `bin/rails test` Any failures or errors (and skips if running in verbose mode) are output diff --git a/railties/lib/rails/test_unit/minitest_plugin.rb b/railties/lib/rails/test_unit/minitest_plugin.rb index b65d67b2a5..3a0a58df88 100644 --- a/railties/lib/rails/test_unit/minitest_plugin.rb +++ b/railties/lib/rails/test_unit/minitest_plugin.rb @@ -33,6 +33,11 @@ module Minitest options[:output_inline] = false end + opts.on("-f", "--fail-fast", + "Abort test run on first failure") do + options[:fail_fast] = true + end + options[:patterns] = opts.order! end diff --git a/railties/lib/rails/test_unit/reporter.rb b/railties/lib/rails/test_unit/reporter.rb index c88e9aa60e..8f1116b6af 100644 --- a/railties/lib/rails/test_unit/reporter.rb +++ b/railties/lib/rails/test_unit/reporter.rb @@ -17,6 +17,10 @@ module Rails io.puts format_rerun_snippet(result) io.puts end + + if fail_fast? && result.failure && !result.error? && !result.skipped? + raise Interrupt + end end def report @@ -48,6 +52,10 @@ module Rails options.fetch(:output_inline, true) end + def fail_fast? + options[:fail_fast] + end + def format_rerun_snippet(result) location, line = result.method(result.name).source_location "#{self.executable} #{relative_path_for(location)}:#{line}" diff --git a/railties/test/application/test_runner_test.rb b/railties/test/application/test_runner_test.rb index 4e747b5253..acfba21f1c 100644 --- a/railties/test/application/test_runner_test.rb +++ b/railties/test/application/test_runner_test.rb @@ -355,6 +355,21 @@ module ApplicationTests assert_match %r{Running:\n\nF\n\nwups!\n\nbin/rails test test/models/post_test.rb:4}, output end + def test_fail_fast + app_file 'test/models/post_test.rb', <<-RUBY + require 'test_helper' + + class PostTest < ActiveSupport::TestCase + def test_post + assert false, 'wups!' + end + end + RUBY + + assert_match(/Interrupt/, + capture(:stderr) { run_test_command('test/models/post_test.rb --fail-fast') }) + end + def test_raise_error_when_specified_file_does_not_exist error = capture(:stderr) { run_test_command('test/not_exists.rb') } assert_match(%r{cannot load such file.+test/not_exists\.rb}, error) diff --git a/railties/test/test_unit/reporter_test.rb b/railties/test/test_unit/reporter_test.rb index b1dd8b5284..59fdf4bc36 100644 --- a/railties/test/test_unit/reporter_test.rb +++ b/railties/test/test_unit/reporter_test.rb @@ -86,6 +86,30 @@ class TestUnitReporterTest < ActiveSupport::TestCase assert_no_match 'Failed tests:', @output.string end + test "fail fast interrupts run on failure" do + fail_fast = Rails::TestUnitReporter.new @output, fail_fast: true + interrupt_raised = false + + # Minitest passes through Interrupt, catch it manually. + begin + fail_fast.record(failed_test) + rescue Interrupt + interrupt_raised = true + ensure + assert interrupt_raised, 'Expected Interrupt to be raised.' + end + end + + test "fail fast does not interrupt run errors or skips" do + fail_fast = Rails::TestUnitReporter.new @output, fail_fast: true + + fail_fast.record(errored_test) + assert_no_match 'Failed tests:', @output.string + + fail_fast.record(skipped_test) + assert_no_match 'Failed tests:', @output.string + end + private def assert_rerun_snippet_count(snippet_count) assert_equal snippet_count, @output.string.scan(%r{^bin/rails test }).size -- cgit v1.2.3