Significantly improved internal encoding heuristics and support.

* Default Encoding.default_internal to UTF-8
* Eliminated the use of file-wide magic comments to coerce code evaluated inside the file
* Read templates as BINARY, use default_external or template-wide magic comments
  inside the Template to set the initial encoding
  * This means that template handlers in Ruby 1.9 will receive Strings encoded
    in default_internal (UTF-8 by default)
* Create a better Exception for encoding issues, and use it when the template
  source has bytes that are not compatible with the specified encoding
* Allow template handlers to opt-into handling BINARY. If they do so, they
  need to do some of their own manual encoding work
* Added a "Configuration Gotchas" section to the intro Rails Guide instructing
  users to use UTF-8 for everything
* Use config.encoding= in Ruby 1.8, and raise if a value that is an invalid
  $KCODE value is used

Also:
* Fixed a few tests that were assert() rather than assert_equal() and
  were caught by Minitest requiring a String for the message
* Fixed a test where an assert_select was misformed, also caught by
  Minitest being more restrictive
* Fixed a test where a Rack response was returning a String rather
  than an Enumerable

10 files changed, 146 insertions, 15 deletions

diff --git a/actionpack/test/abstract_unit.rb b/actionpack/test/abstract_unit.rb
index 89ba0990f1..479e62b23d 100644
--- a/actionpack/test/abstract_unit.rb
+++ b/actionpack/test/abstract_unit.rb
@@ -12,6 +12,10 @@ $:.unshift(File.dirname(__FILE__) + '/fixtures/alternate_helpers')
 
 ENV['TMPDIR'] = File.join(File.dirname(__FILE__), 'tmp')
 
+if defined?(Encoding.default_internal)
+  Encoding.default_internal = "UTF-8"
+end
+
 require 'test/unit'
 require 'abstract_controller'
 require 'action_controller'
diff --git a/actionpack/test/controller/assert_select_test.rb b/actionpack/test/controller/assert_select_test.rb
index 7012c4c9b0..4f8ad23174 100644
--- a/actionpack/test/controller/assert_select_test.rb
+++ b/actionpack/test/controller/assert_select_test.rb
@@ -212,12 +212,12 @@ class AssertSelectTest < ActionController::TestCase
       assert_nothing_raised    { assert_select "div", "bar" }
       assert_nothing_raised    { assert_select "div", /\w*/ }
       assert_nothing_raised    { assert_select "div", :text => /\w*/, :count=>2 }
-      assert_raise(Assertion) { assert_select "div", :text=>"foo", :count=>2 }
+      assert_raise(Assertion)  { assert_select "div", :text=>"foo", :count=>2 }
       assert_nothing_raised    { assert_select "div", :html=>"<span>bar</span>" }
       assert_nothing_raised    { assert_select "div", :html=>"<span>bar</span>" }
       assert_nothing_raised    { assert_select "div", :html=>/\w*/ }
       assert_nothing_raised    { assert_select "div", :html=>/\w*/, :count=>2 }
-      assert_raise(Assertion) { assert_select "div", :html=>"<span>foo</span>", :count=>2 }
+      assert_raise(Assertion)  { assert_select "div", :html=>"<span>foo</span>", :count=>2 }
     end
   end
 
diff --git a/actionpack/test/controller/capture_test.rb b/actionpack/test/controller/capture_test.rb
index d1dbd535c4..47253f22b8 100644
--- a/actionpack/test/controller/capture_test.rb
+++ b/actionpack/test/controller/capture_test.rb
@@ -68,6 +68,6 @@ class CaptureTest < ActionController::TestCase
 
   private
     def expected_content_for_output
-      "<title>Putting stuff in the title!</title>\n\nGreat stuff!"
+      "<title>Putting stuff in the title!</title>\nGreat stuff!"
     end
 end
diff --git a/actionpack/test/controller/render_test.rb b/actionpack/test/controller/render_test.rb
index 52049f2a8a..2b1f2a27df 100644
--- a/actionpack/test/controller/render_test.rb
+++ b/actionpack/test/controller/render_test.rb
@@ -1079,7 +1079,7 @@ class RenderTest < ActionController::TestCase
 
   def test_action_talk_to_layout
     get :action_talk_to_layout
-    assert_equal "<title>Talking to the layout</title>\n\nAction was here!", @response.body
+    assert_equal "<title>Talking to the layout</title>\nAction was here!", @response.body
   end
 
   # :addressed:
@@ -1096,7 +1096,7 @@ class RenderTest < ActionController::TestCase
 
   def test_yield_content_for
     assert_not_deprecated { get :yield_content_for }
-    assert_equal "<title>Putting stuff in the title!</title>\n\nGreat stuff!\n", @response.body
+    assert_equal "<title>Putting stuff in the title!</title>\nGreat stuff!\n", @response.body
   end
 
   def test_overwritting_rendering_relative_file_with_extension
diff --git a/actionpack/test/fixtures/test/content_for.erb b/actionpack/test/fixtures/test/content_for.erb
index 0e47ca8c3d..1fb829f54c 100644
--- a/actionpack/test/fixtures/test/content_for.erb
+++ b/actionpack/test/fixtures/test/content_for.erb
@@ -1,2 +1 @@
-<% content_for :title do %>Putting stuff in the title!<% end %>
-Great stuff!
\ No newline at end of file
+<% content_for :title do -%>Putting stuff in the title!<% end -%>Great stuff!
\ No newline at end of file
diff --git a/actionpack/test/fixtures/test/content_for_concatenated.erb b/actionpack/test/fixtures/test/content_for_concatenated.erb
index fb6b4b05d7..e65f629574 100644
--- a/actionpack/test/fixtures/test/content_for_concatenated.erb
+++ b/actionpack/test/fixtures/test/content_for_concatenated.erb
@@ -1,3 +1,3 @@
 <% content_for :title, "Putting stuff "
-   content_for :title, "in the title!" %>
+   content_for :title, "in the title!" -%>
 Great stuff!
\ No newline at end of file
diff --git a/actionpack/test/fixtures/test/content_for_with_parameter.erb b/actionpack/test/fixtures/test/content_for_with_parameter.erb
index 57aecbac05..aeb6f73ce0 100644
--- a/actionpack/test/fixtures/test/content_for_with_parameter.erb
+++ b/actionpack/test/fixtures/test/content_for_with_parameter.erb
@@ -1,2 +1,2 @@
-<% content_for :title, "Putting stuff in the title!" %>
+<% content_for :title, "Putting stuff in the title!" -%>
 Great stuff!
\ No newline at end of file
diff --git a/actionpack/test/fixtures/test/non_erb_block_content_for.builder b/actionpack/test/fixtures/test/non_erb_block_content_for.builder
index a94643561c..d539a425a4 100644
--- a/actionpack/test/fixtures/test/non_erb_block_content_for.builder
+++ b/actionpack/test/fixtures/test/non_erb_block_content_for.builder
@@ -1,4 +1,4 @@
 content_for :title do
   'Putting stuff in the title!'
 end
-xml << "\nGreat stuff!"
+xml << "Great stuff!"
diff --git a/actionpack/test/template/render_test.rb b/actionpack/test/template/render_test.rb
index d0212024ae..aca96e0a24 100644
--- a/actionpack/test/template/render_test.rb
+++ b/actionpack/test/template/render_test.rb
@@ -232,13 +232,13 @@ module RenderTestCases
   # TODO: Move to deprecated_tests.rb
   def test_render_with_nested_layout_deprecated
     assert_deprecated do
-      assert_equal %(<title>title</title>\n\n\n<div id="column">column</div>\n<div id="content">content</div>\n),
+      assert_equal %(<title>title</title>\n\n<div id="column">column</div>\n<div id="content">content</div>\n),
         @view.render(:file => "test/deprecated_nested_layout.erb", :layout => "layouts/yield")
     end
   end
 
   def test_render_with_nested_layout
-    assert_equal %(<title>title</title>\n\n\n<div id="column">column</div>\n<div id="content">content</div>\n),
+    assert_equal %(<title>title</title>\n\n<div id="column">column</div>\n<div id="content">content</div>\n),
       @view.render(:file => "test/nested_layout.erb", :layout => "layouts/yield")
   end
 
@@ -284,7 +284,7 @@ class LazyViewRenderTest < ActiveSupport::TestCase
       with_external_encoding Encoding::ASCII_8BIT do
         result = @view.render(:file => "test/utf8_magic.html.erb", :layouts => "layouts/yield")
         assert_equal Encoding::UTF_8, result.encoding
-        assert_equal "Русский текст\n\nUTF-8\nUTF-8\nUTF-8\n", result
+        assert_equal "\nРусский \nтекст\n\nUTF-8\nUTF-8\nUTF-8\n", result
       end
     end
 
@@ -302,7 +302,7 @@ class LazyViewRenderTest < ActiveSupport::TestCase
           result = @view.render(:file => "test/utf8.html.erb", :layouts => "layouts/yield")
           flunk 'Should have raised incompatible encoding error'
         rescue ActionView::Template::Error => error
-          assert_match 'invalid byte sequence in Shift_JIS', error.original_exception.message
+          assert_match 'Your template was not saved as valid Shift_JIS', error.original_exception.message
         end
       end
     end
@@ -313,7 +313,7 @@ class LazyViewRenderTest < ActiveSupport::TestCase
           result = @view.render(:file => "test/utf8_magic_with_bare_partial.html.erb", :layouts => "layouts/yield")
           flunk 'Should have raised incompatible encoding error'
         rescue ActionView::Template::Error => error
-          assert_match 'invalid byte sequence in Shift_JIS', error.original_exception.message
+          assert_match 'Your template was not saved as valid Shift_JIS', error.original_exception.message
         end
       end
     end
diff --git a/actionpack/test/template/template_test.rb b/actionpack/test/template/template_test.rb
new file mode 100644
index 0000000000..c4a65d84fc
--- /dev/null
+++ b/actionpack/test/template/template_test.rb
@@ -0,0 +1,128 @@
+require "abstract_unit"
+
+# These are the normal settings that will be set up by Railties
+# TODO: Have these tests support other combinations of these values
+Encoding.default_internal = "UTF-8"
+Encoding.default_external = "UTF-8"
+
+class TestERBTemplate < ActiveSupport::TestCase
+  ERBHandler = ActionView::Template::Handlers::ERB
+
+  class Context
+    def initialize
+      @output_buffer = "original"
+    end
+
+    def hello
+      "Hello"
+    end
+
+    def partial
+      ActionView::Template.new(
+        "<%= @_virtual_path %>",
+        "partial",
+        ERBHandler,
+        :virtual_path => "partial"
+      )
+    end
+
+    def logger
+      require "logger"
+      Logger.new(STDERR)
+    end
+
+    def my_buffer
+      @output_buffer
+    end
+  end
+
+  def new_template(body = "<%= hello %>", handler = ERBHandler, details = {})
+    ActionView::Template.new(body, "hello template", ERBHandler, {:virtual_path => "hello"})
+  end
+
+  def render(locals = {})
+    @template.render(@obj, locals)
+  end
+
+  def setup
+    @obj = Context.new
+  end
+
+  def test_basic_template
+    @template = new_template
+    assert_equal "Hello", render
+  end
+
+  def test_locals
+    @template = new_template("<%= my_local %>")
+    assert_equal "I'm a local", render(:my_local => "I'm a local")
+  end
+
+  def test_restores_buffer
+    @template = new_template
+    assert_equal "Hello", render
+    assert_equal "original", @obj.my_buffer
+  end
+
+  def test_virtual_path
+    @template = new_template("<%= @_virtual_path %>" \
+                             "<%= partial.render(self, {}) %>" \
+                             "<%= @_virtual_path %>")
+    assert_equal "hellopartialhello", render
+  end
+
+  if "ruby".encoding_aware?
+    def test_resulting_string_is_utf8
+      @template = new_template
+      assert_equal Encoding::UTF_8, render.encoding
+    end
+
+    def test_no_magic_comment_word_with_utf_8
+      @template = new_template("hello \u{fc}mlat")
+      assert_equal Encoding::UTF_8, render.encoding
+      assert_equal "hello \u{fc}mlat", render
+    end
+
+    # This test ensures that if the default_external
+    # is set to something other than UTF-8, we don't
+    # get any errors and get back a UTF-8 String.
+    def test_default_external_works
+      Encoding.default_external = "ISO-8859-1"
+      @template = new_template("hello \xFCmlat")
+      assert_equal Encoding::UTF_8, render.encoding
+      assert_equal "hello \u{fc}mlat", render
+    ensure
+      Encoding.default_external = "UTF-8"
+    end
+
+    def test_encoding_can_be_specified_with_magic_comment
+      @template = new_template("# encoding: ISO-8859-1\nhello \xFCmlat")
+      assert_equal Encoding::UTF_8, render.encoding
+      assert_equal "\nhello \u{fc}mlat", render
+    end
+
+    # TODO: This is currently handled inside ERB. The case of explicitly
+    # lying about encodings via the normal Rails API should be handled
+    # inside Rails.
+    def test_lying_with_magic_comment
+      assert_raises(ActionView::Template::Error) do
+        @template = new_template("# encoding: UTF-8\nhello \xFCmlat")
+        render
+      end
+    end
+
+    def test_encoding_can_be_specified_with_magic_comment_in_erb
+      @template = new_template("<%# encoding: ISO-8859-1 %>hello \xFCmlat")
+      result = render
+      assert_equal Encoding::UTF_8, render.encoding
+      assert_equal "hello \u{fc}mlat", render
+    end
+
+    def test_error_when_template_isnt_valid_utf8
+      assert_raises(ActionView::Template::Error, /\xFC/) do
+        @template = new_template("hello \xFCmlat")
+        render
+      end
+    end
+  end
+end
\ No newline at end of file