From 64d109e3539ad600f58536d3ecabd2f87b67fd1c Mon Sep 17 00:00:00 2001 From: wycats Date: Sun, 16 May 2010 10:25:55 +0400 Subject: Significantly improved internal encoding heuristics and support. * Default Encoding.default_internal to UTF-8 * Eliminated the use of file-wide magic comments to coerce code evaluated inside the file * Read templates as BINARY, use default_external or template-wide magic comments inside the Template to set the initial encoding * This means that template handlers in Ruby 1.9 will receive Strings encoded in default_internal (UTF-8 by default) * Create a better Exception for encoding issues, and use it when the template source has bytes that are not compatible with the specified encoding * Allow template handlers to opt-into handling BINARY. If they do so, they need to do some of their own manual encoding work * Added a "Configuration Gotchas" section to the intro Rails Guide instructing users to use UTF-8 for everything * Use config.encoding= in Ruby 1.8, and raise if a value that is an invalid $KCODE value is used Also: * Fixed a few tests that were assert() rather than assert_equal() and were caught by Minitest requiring a String for the message * Fixed a test where an assert_select was misformed, also caught by Minitest being more restrictive * Fixed a test where a Rack response was returning a String rather than an Enumerable --- railties/guides/source/getting_started.textile | 21 +++++++++++++++++++++ railties/lib/rails.rb | 1 + railties/lib/rails/application/configuration.rb | 10 +++++++++- railties/test/application/configuration_test.rb | 3 ++- 4 files changed, 33 insertions(+), 2 deletions(-) (limited to 'railties') diff --git a/railties/guides/source/getting_started.textile b/railties/guides/source/getting_started.textile index 5da7ff7daa..b7301bff20 100644 --- a/railties/guides/source/getting_started.textile +++ b/railties/guides/source/getting_started.textile @@ -1462,11 +1462,32 @@ Rails also comes with built-in help that you can generate using the rake command * Running +rake doc:guides+ will put a full copy of the Rails Guides in the +doc/guides+ folder of your application. Open +doc/guides/index.html+ in your web browser to explore the Guides. * Running +rake doc:rails+ will put a full copy of the API documentation for Rails in the +doc/api+ folder of your application. Open +doc/api/index.html+ in your web browser to explore the API documentation. +h3. Configuration Gotchas + +The easiest way to work with Rails is to store all external data as UTF-8. If you don't, Ruby libraries and Rails will often be able to convert your native data into UTF-8, but this doesn't always work reliably, so you're better off ensuring that all external data is UTF-8. + +If you have made a mistake in this area, the most common symptom is a black diamond with a question mark inside appearing in the browser. Another common symptom is characters like "ü" appearing instead of "ü". Rails takes a number of internal steps to mitigate common causes of these problems that can be automatically detected and corrected. However, if you have external data that is not stored as UTF-8, it can occasionally result in these kinds of issues that cannot be automatically detected by Rails and corrected. + +Two very common sources of data that are not UTF-8: +* Your text editor: Most text editors (such as Textmate), default to saving files as + UTF-8. If your text editor does not, this can result in special characters that you + enter in your templates (such as é) to appear as a diamond with a question mark inside + in the browser. This also applies to your I18N translation files. + Most editors that do not already default to UTF-8 (such as some versions of + Dreamweaver) offer a way to change the default to UTF-8. Do so. +* Your database. Rails defaults to converting data from your database into UTF-8 at + the boundary. However, if your database is not using UTF-8 internally, it may not + be able to store all characters that your users enter. For instance, if your database + is using Latin-1 internally, and your user enters a Russian, Hebrew, or Japanese + character, the data will be lost forever once it enters the database. If possible, + use UTF-8 as the internal storage of your database. h3. Changelog "Lighthouse ticket":http://rails.lighthouseapp.com/projects/16213-rails-guides/tickets/2 +* May 16, 2010: Added a section on configuration gotchas to address common encoding + problems that people might have * April 30, 2010: Fixes, editing and updating of code samples by "Rohit Arondekar":http://rohitarondekar.com * April 25, 2010: Couple of more minor fixups "Mikel Lindsaar":credits:html#raasdnil * April 1, 2010: Fixed document to validate XHTML 1.0 Strict. "Jaime Iniesta":http://jaimeiniesta.com diff --git a/railties/lib/rails.rb b/railties/lib/rails.rb index 0611b2a9f5..be486ef2ac 100644 --- a/railties/lib/rails.rb +++ b/railties/lib/rails.rb @@ -23,6 +23,7 @@ if RUBY_VERSION < '1.9' $KCODE='u' else Encoding.default_external = Encoding::UTF_8 + Encoding.default_internal = Encoding::UTF_8 end module Rails diff --git a/railties/lib/rails/application/configuration.rb b/railties/lib/rails/application/configuration.rb index 9353fbefef..8afe423973 100644 --- a/railties/lib/rails/application/configuration.rb +++ b/railties/lib/rails/application/configuration.rb @@ -1,4 +1,5 @@ require 'active_support/deprecation' +require 'active_support/core_ext/string/encoding' require 'rails/engine/configuration' module Rails @@ -27,8 +28,15 @@ module Rails def encoding=(value) @encoding = value - if defined?(Encoding) && Encoding.respond_to?(:default_external=) + if "ruby".encoding_aware? Encoding.default_external = value + Encoding.default_internal = value + else + $KCODE = value + if $KCODE == "NONE" + raise "The value you specified for config.encoding is " \ + "invalid. The possible values are UTF8, SJIS, or EUC" + end end end diff --git a/railties/test/application/configuration_test.rb b/railties/test/application/configuration_test.rb index dfc4e2359b..c08bd2ef22 100644 --- a/railties/test/application/configuration_test.rb +++ b/railties/test/application/configuration_test.rb @@ -180,7 +180,8 @@ module ApplicationTests require "#{app_path}/config/application" unless RUBY_VERSION < '1.9' - assert_equal Encoding.find("utf-8"), Encoding.default_external + assert_equal Encoding::UTF_8, Encoding.default_external + assert_equal Encoding::UTF_8, Encoding.default_internal end end -- cgit v1.2.3