Merge pull request #19216 from senny/bin_test_runner

`bin/rails test` runner (rerun snippets, run tests by line, option documentation)

4 files changed, 203 insertions, 41 deletions

diff --git a/railties/test/application/test_runner_test.rb b/railties/test/application/test_runner_test.rb
index a12f3cfc24..c122b315c0 100644
--- a/railties/test/application/test_runner_test.rb
+++ b/railties/test/application/test_runner_test.rb
@@ -7,7 +7,6 @@ module ApplicationTests
 
     def setup
       build_app
-      ENV['RAILS_ENV'] = nil
       create_schema
     end
 
@@ -55,7 +54,7 @@ module ApplicationTests
       create_test_file :models, 'foo'
       create_test_file :models, 'bar'
       create_test_file :controllers, 'foobar_controller'
-      run_test_models_command.tap do |output|
+      run_test_command("test/models").tap do |output|
         assert_match "FooTest", output
         assert_match "BarTest", output
         assert_match "2 runs, 2 assertions, 0 failures", output
@@ -66,7 +65,7 @@ module ApplicationTests
       create_test_file :helpers, 'foo_helper'
       create_test_file :helpers, 'bar_helper'
       create_test_file :controllers, 'foobar_controller'
-      run_test_helpers_command.tap do |output|
+      run_test_command("test/helpers").tap do |output|
         assert_match "FooHelperTest", output
         assert_match "BarHelperTest", output
         assert_match "2 runs, 2 assertions, 0 failures", output
@@ -74,6 +73,7 @@ module ApplicationTests
     end
 
     def test_run_units
+      skip "we no longer have the concept of unit tests. Just different directories..."
       create_test_file :models, 'foo'
       create_test_file :helpers, 'bar_helper'
       create_test_file :unit, 'baz_unit'
@@ -90,7 +90,7 @@ module ApplicationTests
       create_test_file :controllers, 'foo_controller'
       create_test_file :controllers, 'bar_controller'
       create_test_file :models, 'foo'
-      run_test_controllers_command.tap do |output|
+      run_test_command("test/controllers").tap do |output|
         assert_match "FooControllerTest", output
         assert_match "BarControllerTest", output
         assert_match "2 runs, 2 assertions, 0 failures", output
@@ -101,7 +101,7 @@ module ApplicationTests
       create_test_file :mailers, 'foo_mailer'
       create_test_file :mailers, 'bar_mailer'
       create_test_file :models, 'foo'
-      run_test_mailers_command.tap do |output|
+      run_test_command("test/mailers").tap do |output|
         assert_match "FooMailerTest", output
         assert_match "BarMailerTest", output
         assert_match "2 runs, 2 assertions, 0 failures", output
@@ -112,7 +112,7 @@ module ApplicationTests
       create_test_file :jobs, 'foo_job'
       create_test_file :jobs, 'bar_job'
       create_test_file :models, 'foo'
-      run_test_jobs_command.tap do |output|
+      run_test_command("test/jobs").tap do |output|
         assert_match "FooJobTest", output
         assert_match "BarJobTest", output
         assert_match "2 runs, 2 assertions, 0 failures", output
@@ -120,6 +120,7 @@ module ApplicationTests
     end
 
     def test_run_functionals
+      skip "we no longer have the concept of functional tests. Just different directories..."
       create_test_file :mailers, 'foo_mailer'
       create_test_file :controllers, 'bar_controller'
       create_test_file :functional, 'baz_functional'
@@ -135,7 +136,7 @@ module ApplicationTests
     def test_run_integration
       create_test_file :integration, 'foo_integration'
       create_test_file :models, 'foo'
-      run_test_integration_command.tap do |output|
+      run_test_command("test/integration").tap do |output|
         assert_match "FooIntegration", output
         assert_match "1 runs, 1 assertions, 0 failures", output
       end
@@ -165,7 +166,7 @@ module ApplicationTests
         end
       RUBY
 
-      run_test_command('test/unit/chu_2_koi_test.rb test_rikka').tap do |output|
+      run_test_command('-n test_rikka test/unit/chu_2_koi_test.rb').tap do |output|
         assert_match "Rikka", output
         assert_no_match "Sanae", output
       end
@@ -186,7 +187,7 @@ module ApplicationTests
         end
       RUBY
 
-      run_test_command('test/unit/chu_2_koi_test.rb /rikka/').tap do |output|
+      run_test_command('-p rikka test/unit/chu_2_koi_test.rb').tap do |output|
         assert_match "Rikka", output
         assert_no_match "Sanae", output
       end
@@ -194,18 +195,18 @@ module ApplicationTests
 
     def test_load_fixtures_when_running_test_suites
       create_model_with_fixture
-      suites = [:models, :helpers, [:units, :unit], :controllers, :mailers,
-        [:functionals, :functional], :integration]
+      suites = [:models, :helpers, :controllers, :mailers, :integration]
 
       suites.each do |suite, directory|
         directory ||= suite
         create_fixture_test directory
-        assert_match "3 users", run_task(["test:#{suite}"])
+        assert_match "3 users", run_test_command("test/#{suite}")
         Dir.chdir(app_path) { FileUtils.rm_f "test/#{directory}" }
       end
     end
 
     def test_run_with_model
+      skip "These feel a bit odd. Not sure we should keep supporting them."
       create_model_with_fixture
       create_fixture_test 'models', 'user'
       assert_match "3 users", run_task(["test models/user"])
@@ -213,6 +214,7 @@ module ApplicationTests
     end
 
     def test_run_different_environment_using_env_var
+      skip "no longer possible. Running tests in a different environment should be explicit"
       app_file 'test/unit/env_test.rb', <<-RUBY
         require 'test_helper'
 
@@ -227,7 +229,7 @@ module ApplicationTests
       assert_match "development", run_test_command('test/unit/env_test.rb')
     end
 
-    def test_run_different_environment_using_e_tag
+    def test_run_different_environment
       env = "development"
       app_file 'test/unit/env_test.rb', <<-RUBY
         require 'test_helper'
@@ -239,7 +241,7 @@ module ApplicationTests
         end
       RUBY
 
-      assert_match env, run_test_command("test/unit/env_test.rb RAILS_ENV=#{env}")
+      assert_match env, run_test_command("-e #{env} test/unit/env_test.rb")
     end
 
     def test_generated_scaffold_works_with_rails_test
@@ -248,17 +250,8 @@ module ApplicationTests
     end
 
     private
-      def run_task(tasks)
-        Dir.chdir(app_path) { `bundle exec rake #{tasks.join ' '}` }
-      end
-
       def run_test_command(arguments = 'test/unit/test_test.rb')
-        run_task ['test', arguments]
-      end
-      %w{ mailers models helpers units controllers functionals integration jobs }.each do |type|
-        define_method("run_test_#{type}_command") do
-          run_task ["test:#{type}"]
-        end
+        Dir.chdir(app_path) { `bin/rails t #{arguments}` }
       end
 
       def create_model_with_fixture
diff --git a/railties/test/application/test_test.rb b/railties/test/application/test_test.rb
index c7132837b1..61652e5052 100644
--- a/railties/test/application/test_test.rb
+++ b/railties/test/application/test_test.rb
@@ -65,6 +65,7 @@ module ApplicationTests
 
       output = run_test_file('unit/failing_test.rb', env: { "BACKTRACE" => "1" })
       assert_match %r{/app/test/unit/failing_test\.rb}, output
+      assert_match %r{/app/test/unit/failing_test\.rb:4}, output
     end
 
     test "ruby schema migrations" do
@@ -300,23 +301,7 @@ Expected: ["id", "name"]
       end
 
       def run_test_file(name, options = {})
-        ruby '-Itest', "#{app_path}/test/#{name}", options.deep_merge(env: {"RAILS_ENV" => "test"})
-      end
-
-      def ruby(*args)
-        options = args.extract_options!
-        env = options.fetch(:env, {})
-        env["RUBYLIB"] = $:.join(':')
-
-        Dir.chdir(app_path) do
-          `#{env_string(env)} #{Gem.ruby} #{args.join(' ')} 2>&1`
-        end
-      end
-
-      def env_string(variables)
-        variables.map do |key, value|
-          "#{key}='#{value}'"
-        end.join " "
+        Dir.chdir(app_path) { `bin/rails test "#{app_path}/test/#{name}" 2>&1` }
       end
   end
 end
diff --git a/railties/test/test_unit/reporter_test.rb b/railties/test/test_unit/reporter_test.rb
new file mode 100644
index 0000000000..77883612f5
--- /dev/null
+++ b/railties/test/test_unit/reporter_test.rb
@@ -0,0 +1,74 @@
+require 'abstract_unit'
+require 'rails/test_unit/reporter'
+
+class TestUnitReporterTest < ActiveSupport::TestCase
+  class ExampleTest < Minitest::Test
+    def woot; end
+  end
+
+  setup do
+    @output = StringIO.new
+    @reporter = Rails::TestUnitReporter.new @output
+  end
+
+  test "prints rerun snippet to run a single failed test" do
+    @reporter.record(failed_test)
+    @reporter.report
+
+    assert_match %r{^bin/rails test .*test/test_unit/reporter_test.rb:6$}, @output.string
+    assert_rerun_snippet_count 1
+  end
+
+  test "prints rerun snippet for every failed test" do
+    @reporter.record(failed_test)
+    @reporter.record(failed_test)
+    @reporter.record(failed_test)
+    @reporter.report
+
+    assert_rerun_snippet_count 3
+  end
+
+  test "does not print snippet for successful and skipped tests" do
+    @reporter.record(passing_test)
+    @reporter.record(skipped_test)
+    @reporter.report
+    assert_rerun_snippet_count 0
+  end
+
+  test "prints rerun snippet for skipped tests if run in verbose mode" do
+    verbose = Rails::TestUnitReporter.new @output, verbose: true
+    verbose.record(skipped_test)
+    verbose.report
+
+    assert_rerun_snippet_count 1
+  end
+
+  private
+  def assert_rerun_snippet_count(snippet_count)
+    assert_equal snippet_count, @output.string.scan(%r{^bin/rails test }).size
+  end
+
+  def failed_test
+    ft = ExampleTest.new(:woot)
+    ft.failures << begin
+                     raise Minitest::Assertion, "boo"
+                   rescue Minitest::Assertion => e
+                     e
+                   end
+    ft
+  end
+
+  def passing_test
+    ExampleTest.new(:woot)
+  end
+
+  def skipped_test
+    st = ExampleTest.new(:woot)
+    st.failures << begin
+                     raise Minitest::Skip
+                   rescue Minitest::Assertion => e
+                     e
+                   end
+    st
+  end
+end
diff --git a/railties/test/test_unit/runner_test.rb b/railties/test/test_unit/runner_test.rb
new file mode 100644
index 0000000000..c040c71635
--- /dev/null
+++ b/railties/test/test_unit/runner_test.rb
@@ -0,0 +1,110 @@
+require 'abstract_unit'
+require 'env_helpers'
+require 'rails/test_unit/runner'
+
+class TestUnitTestRunnerTest < ActiveSupport::TestCase
+  include EnvHelpers
+
+  setup do
+    @options = Rails::TestRunner::Options
+  end
+
+  test "shows the filtered backtrace by default" do
+    options = @options.parse([])
+    assert_not options[:backtrace]
+  end
+
+  test "has --backtrace (-b) option to show the full backtrace" do
+    options = @options.parse(["-b"])
+    assert options[:backtrace]
+
+    options = @options.parse(["--backtrace"])
+    assert options[:backtrace]
+  end
+
+  test "show full backtrace using BACKTRACE environment variable" do
+    switch_env "BACKTRACE", "true" do
+      options = @options.parse([])
+      assert options[:backtrace]
+    end
+  end
+
+  test "tests run in the test environment by default" do
+    options = @options.parse([])
+    assert_equal "test", options[:environment]
+  end
+
+  test "can run in a specific environment" do
+    options = @options.parse(["-e development"])
+    assert_equal "development", options[:environment]
+  end
+
+  test "parse the filename and line" do
+    file = "test/test_unit/runner_test.rb"
+    absolute_file = __FILE__
+    options = @options.parse(["#{file}:20"])
+    assert_equal absolute_file, options[:filename]
+    assert_equal 20, options[:line]
+
+    options = @options.parse(["#{file}:"])
+    assert_equal [absolute_file], options[:patterns]
+    assert_nil options[:line]
+
+    options = @options.parse([file])
+    assert_equal [absolute_file], options[:patterns]
+    assert_nil options[:line]
+  end
+
+  test "find_method on same file" do
+    options = @options.parse(["#{__FILE__}:#{__LINE__}"])
+    runner = Rails::TestRunner.new(options)
+    assert_equal "test_find_method_on_same_file", runner.find_method
+  end
+
+  test "find_method on a different file" do
+    options = @options.parse(["foobar.rb:#{__LINE__}"])
+    runner = Rails::TestRunner.new(options)
+    assert_nil runner.find_method
+  end
+
+  test "run all tests in a directory" do
+    options = @options.parse([__dir__])
+
+    assert_equal ["#{__dir__}/**/*_test.rb"], options[:patterns]
+    assert_nil options[:filename]
+    assert_nil options[:line]
+  end
+
+  test "run multiple folders" do
+    application_dir = File.expand_path("#{__dir__}/../application")
+
+    options = @options.parse([__dir__, application_dir])
+
+    assert_equal ["#{__dir__}/**/*_test.rb", "#{application_dir}/**/*_test.rb"], options[:patterns]
+    assert_nil options[:filename]
+    assert_nil options[:line]
+
+    runner = Rails::TestRunner.new(options)
+    assert runner.test_files.size > 0
+  end
+
+  test "run multiple files and run one file by line" do
+    line = __LINE__
+    options = @options.parse([__dir__, "#{__FILE__}:#{line}"])
+
+    assert_equal ["#{__dir__}/**/*_test.rb"], options[:patterns]
+    assert_equal __FILE__, options[:filename]
+    assert_equal line, options[:line]
+
+    runner = Rails::TestRunner.new(options)
+    assert_equal [__FILE__], runner.test_files, 'Only returns the file that running by line'
+  end
+
+  test "running multiple files passing line number" do
+    line = __LINE__
+    options = @options.parse(["foobar.rb:8", "#{__FILE__}:#{line}"])
+
+    assert_equal __FILE__, options[:filename], 'Returns the last file'
+    assert_equal line, options[:line]
+  end
+end