diff options
25 files changed, 94 insertions, 384 deletions
diff --git a/actionpack/README.rdoc b/actionpack/README.rdoc index fc36423f83..076a93bbcd 100644 --- a/actionpack/README.rdoc +++ b/actionpack/README.rdoc @@ -206,7 +206,7 @@ A short rundown of some of the major features: You specify a logger through a class method, such as: - ActionController::Base.logger = Logger.new("Application Log") + ActionController::Base.logger = ActiveSupport::Logger.new("Application Log") ActionController::Base.logger = Log4r::Logger.new("Application Log") diff --git a/actionpack/lib/action_view/helpers/tag_helper.rb b/actionpack/lib/action_view/helpers/tag_helper.rb index 8c33ef09fa..93a3c40683 100644 --- a/actionpack/lib/action_view/helpers/tag_helper.rb +++ b/actionpack/lib/action_view/helpers/tag_helper.rb @@ -118,7 +118,7 @@ module ActionView # escape_once("<< Accept & Checkout") # # => "<< Accept & Checkout" def escape_once(html) - ActiveSupport::Multibyte.clean(html.to_s).gsub(/[\"><]|&(?!([a-zA-Z]+|(#\d+));)/) { |special| ERB::Util::HTML_ESCAPE[special] } + html.to_s.gsub(/[\"><]|&(?!([a-zA-Z]+|(#\d+));)/) { |special| ERB::Util::HTML_ESCAPE[special] } end private diff --git a/actionpack/lib/action_view/helpers/text_helper.rb b/actionpack/lib/action_view/helpers/text_helper.rb index 209360ee82..ce79a3da48 100644 --- a/actionpack/lib/action_view/helpers/text_helper.rb +++ b/actionpack/lib/action_view/helpers/text_helper.rb @@ -162,15 +162,15 @@ module ActionView options.reverse_merge!(:radius => 100, :omission => "...") phrase = Regexp.escape(phrase) - return unless found_pos = text.mb_chars =~ /(#{phrase})/i + return unless found_pos = text =~ /(#{phrase})/i start_pos = [ found_pos - options[:radius], 0 ].max - end_pos = [ [ found_pos + phrase.mb_chars.length + options[:radius] - 1, 0].max, text.mb_chars.length ].min + end_pos = [ [ found_pos + phrase.length + options[:radius] - 1, 0].max, text.length ].min prefix = start_pos > 0 ? options[:omission] : "" - postfix = end_pos < text.mb_chars.length - 1 ? options[:omission] : "" + postfix = end_pos < text.length - 1 ? options[:omission] : "" - prefix + text.mb_chars[start_pos..end_pos].strip + postfix + prefix + text[start_pos..end_pos].strip + postfix end # Attempts to pluralize the +singular+ word unless +count+ is 1. If diff --git a/actionpack/test/controller/content_type_test.rb b/actionpack/test/controller/content_type_test.rb index d51882066d..a312b7c32a 100644 --- a/actionpack/test/controller/content_type_test.rb +++ b/actionpack/test/controller/content_type_test.rb @@ -59,7 +59,7 @@ class ContentTypeTest < ActionController::TestCase super # enable a logger so that (e.g.) the benchmarking stuff runs, so we can get # a more accurate simulation of what happens in "real life". - @controller.logger = Logger.new(nil) + @controller.logger = ActiveSupport::Logger.new(nil) end # :ported: diff --git a/actionpack/test/controller/new_base/render_streaming_test.rb b/actionpack/test/controller/new_base/render_streaming_test.rb index 1532bd5c98..61ea68e3f7 100644 --- a/actionpack/test/controller/new_base/render_streaming_test.rb +++ b/actionpack/test/controller/new_base/render_streaming_test.rb @@ -85,7 +85,7 @@ module RenderStreaming test "rendering with template exception logs the exception" do io = StringIO.new - _old, ActionController::Base.logger = ActionController::Base.logger, Logger.new(io) + _old, ActionController::Base.logger = ActionController::Base.logger, ActiveSupport::Logger.new(io) begin get "/render_streaming/basic/template_exception" diff --git a/actionpack/test/controller/render_test.rb b/actionpack/test/controller/render_test.rb index 768cfb34ac..ec26315dc7 100644 --- a/actionpack/test/controller/render_test.rb +++ b/actionpack/test/controller/render_test.rb @@ -696,7 +696,7 @@ class RenderTest < ActionController::TestCase # enable a logger so that (e.g.) the benchmarking stuff runs, so we can get # a more accurate simulation of what happens in "real life". super - @controller.logger = Logger.new(nil) + @controller.logger = ActiveSupport::Logger.new(nil) @request.host = "www.nextangle.com" end @@ -907,7 +907,7 @@ class RenderTest < ActionController::TestCase def test_access_to_logger_in_view get :accessing_logger_in_template - assert_equal "Logger", @response.body + assert_equal "ActiveSupport::Logger", @response.body end # :ported: diff --git a/actionpack/test/controller/render_xml_test.rb b/actionpack/test/controller/render_xml_test.rb index ec4dc848ff..8b4f2f5349 100644 --- a/actionpack/test/controller/render_xml_test.rb +++ b/actionpack/test/controller/render_xml_test.rb @@ -48,7 +48,7 @@ class RenderXmlTest < ActionController::TestCase # enable a logger so that (e.g.) the benchmarking stuff runs, so we can get # a more accurate simulation of what happens in "real life". super - @controller.logger = Logger.new(nil) + @controller.logger = ActiveSupport::Logger.new(nil) @request.host = "www.nextangle.com" end diff --git a/actionpack/test/dispatch/request/json_params_parsing_test.rb b/actionpack/test/dispatch/request/json_params_parsing_test.rb index ad44b4b16a..ae425dd406 100644 --- a/actionpack/test/dispatch/request/json_params_parsing_test.rb +++ b/actionpack/test/dispatch/request/json_params_parsing_test.rb @@ -34,7 +34,7 @@ class JsonParamsParsingTest < ActionDispatch::IntegrationTest with_test_routing do output = StringIO.new json = "[\"person]\": {\"name\": \"David\"}}" - post "/parse", json, {'CONTENT_TYPE' => 'application/json', 'action_dispatch.show_exceptions' => true, 'action_dispatch.logger' => Logger.new(output)} + post "/parse", json, {'CONTENT_TYPE' => 'application/json', 'action_dispatch.show_exceptions' => true, 'action_dispatch.logger' => ActiveSupport::Logger.new(output)} assert_response :error output.rewind && err = output.read assert err =~ /Error occurred while parsing request parameters/ diff --git a/actionpack/test/dispatch/request/xml_params_parsing_test.rb b/actionpack/test/dispatch/request/xml_params_parsing_test.rb index 0984f00066..afd400c2a9 100644 --- a/actionpack/test/dispatch/request/xml_params_parsing_test.rb +++ b/actionpack/test/dispatch/request/xml_params_parsing_test.rb @@ -56,7 +56,7 @@ class XmlParamsParsingTest < ActionDispatch::IntegrationTest with_test_routing do output = StringIO.new xml = "<person><name>David</name><avatar type='file' name='me.jpg' content_type='image/jpg'>#{::Base64.encode64('ABC')}</avatar></pineapple>" - post "/parse", xml, default_headers.merge('action_dispatch.show_exceptions' => true, 'action_dispatch.logger' => Logger.new(output)) + post "/parse", xml, default_headers.merge('action_dispatch.show_exceptions' => true, 'action_dispatch.logger' => ActiveSupport::Logger.new(output)) assert_response :error output.rewind && err = output.read assert err =~ /Error occurred while parsing request parameters/ diff --git a/actionpack/test/template/template_test.rb b/actionpack/test/template/template_test.rb index f9c228f0c3..3084527f70 100644 --- a/actionpack/test/template/template_test.rb +++ b/actionpack/test/template/template_test.rb @@ -37,7 +37,7 @@ class TestERBTemplate < ActiveSupport::TestCase end def logger - Logger.new(STDERR) + ActiveSupport::Logger.new(STDERR) end def my_buffer diff --git a/actionpack/test/template/text_helper_test.rb b/actionpack/test/template/text_helper_test.rb index 839bf900dc..aa185d9cb0 100644 --- a/actionpack/test/template/text_helper_test.rb +++ b/actionpack/test/template/text_helper_test.rb @@ -81,10 +81,6 @@ class TextHelperTest < ActionView::TestCase end def test_truncate_multibyte - # .mb_chars always returns a UTF-8 String. - # assert_equal "\354\225\210\353\205\225\355...", - # truncate("\354\225\210\353\205\225\355\225\230\354\204\270\354\232\224", :length => 10) - assert_equal "\354\225\204\353\246\254\353\236\221 \354\225\204\353\246\254 ...".force_encoding('UTF-8'), truncate("\354\225\204\353\246\254\353\236\221 \354\225\204\353\246\254 \354\225\204\353\235\274\353\246\254\354\230\244".force_encoding('UTF-8'), :length => 10) end @@ -231,8 +227,6 @@ class TextHelperTest < ActionView::TestCase def test_excerpt_with_utf8 assert_equal("...\357\254\203ciency could not be...".force_encoding('UTF-8'), excerpt("That's why e\357\254\203ciency could not be helped".force_encoding('UTF-8'), 'could', 8)) - # .mb_chars always returns UTF-8, even in 1.9. This is not great, but it's how it works. Let's work this out. - # assert_equal("...\203ciency could not be...", excerpt("That's why e\357\254\203ciency could not be helped".force_encoding("BINARY"), 'could', 8)) end def test_word_wrap diff --git a/activerecord/README.rdoc b/activerecord/README.rdoc index 6f9f63ceee..30a66ff5f0 100644 --- a/activerecord/README.rdoc +++ b/activerecord/README.rdoc @@ -143,7 +143,7 @@ A short rundown of some of the major features: * Logging support for Log4r[http://log4r.sourceforge.net] and Logger[http://www.ruby-doc.org/stdlib/libdoc/logger/rdoc]. - ActiveRecord::Base.logger = Logger.new(STDOUT) + ActiveRecord::Base.logger = ActiveSupport::Logger.new(STDOUT) ActiveRecord::Base.logger = Log4r::Logger.new("Application Log") diff --git a/activerecord/lib/active_record/railtie.rb b/activerecord/lib/active_record/railtie.rb index 165d0750dd..b1c8ae5b77 100644 --- a/activerecord/lib/active_record/railtie.rb +++ b/activerecord/lib/active_record/railtie.rb @@ -38,7 +38,7 @@ module ActiveRecord # first time. Also, make it output to STDERR. console do |app| require "active_record/railties/console_sandbox" if app.sandbox? - ActiveRecord::Base.logger = Logger.new(STDERR) + ActiveRecord::Base.logger = ActiveSupport::Logger.new(STDERR) end initializer "active_record.initialize_timezone" do diff --git a/activerecord/lib/active_record/validations/uniqueness.rb b/activerecord/lib/active_record/validations/uniqueness.rb index 2e2ea8c42b..d56248b928 100644 --- a/activerecord/lib/active_record/validations/uniqueness.rb +++ b/activerecord/lib/active_record/validations/uniqueness.rb @@ -54,7 +54,7 @@ module ActiveRecord def build_relation(klass, table, attribute, value) #:nodoc: column = klass.columns_hash[attribute.to_s] - value = column.limit ? value.to_s.mb_chars[0, column.limit] : value.to_s if column.text? + value = column.limit ? value.to_s[0, column.limit] : value.to_s if column.text? if !options[:case_sensitive] && value && column.text? # will use SQL LOWER function before comparison, unless it detects a case insensitive collation diff --git a/activeresource/test/abstract_unit.rb b/activeresource/test/abstract_unit.rb index 9c1e9a526d..38df583990 100644 --- a/activeresource/test/abstract_unit.rb +++ b/activeresource/test/abstract_unit.rb @@ -7,11 +7,10 @@ require 'test/unit' require 'active_resource' require 'active_support' require 'active_support/test_case' - require 'setter_trap' +require 'active_support/logger' -require 'logger' -ActiveResource::Base.logger = Logger.new("#{File.dirname(__FILE__)}/debug.log") +ActiveResource::Base.logger = ActiveSupport::Logger.new("#{File.dirname(__FILE__)}/debug.log") def setup_response matz_hash = { 'person' => { :id => 1, :name => 'Matz' } } diff --git a/activesupport/lib/active_support/multibyte.rb b/activesupport/lib/active_support/multibyte.rb index 57e8e24bf4..5efe13c537 100644 --- a/activesupport/lib/active_support/multibyte.rb +++ b/activesupport/lib/active_support/multibyte.rb @@ -1,9 +1,5 @@ -# encoding: utf-8 -require 'active_support/core_ext/module/attribute_accessors' - module ActiveSupport #:nodoc: module Multibyte - autoload :EncodingError, 'active_support/multibyte/exceptions' autoload :Chars, 'active_support/multibyte/chars' autoload :Unicode, 'active_support/multibyte/unicode' @@ -21,24 +17,5 @@ module ActiveSupport #:nodoc: def self.proxy_class @proxy_class ||= ActiveSupport::Multibyte::Chars end - - # Regular expressions that describe valid byte sequences for a character - VALID_CHARACTER = { - # Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site) - 'UTF-8' => /\A(?: - [\x00-\x7f] | - [\xc2-\xdf] [\x80-\xbf] | - \xe0 [\xa0-\xbf] [\x80-\xbf] | - [\xe1-\xef] [\x80-\xbf] [\x80-\xbf] | - \xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] | - [\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] | - \xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf])\z /xn, - # Quick check for valid Shift-JIS characters, disregards the odd-even pairing - 'Shift_JIS' => /\A(?: - [\x00-\x7e\xa1-\xdf] | - [\x81-\x9f\xe0-\xef] [\x40-\x7e\x80-\x9e\x9f-\xfc])\z /xn - } end -end - -require 'active_support/multibyte/utils'
\ No newline at end of file +end
\ No newline at end of file diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index dcc176e93f..99b974e4a7 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -1,6 +1,7 @@ # encoding: utf-8 require 'active_support/core_ext/string/access' require 'active_support/core_ext/string/behavior' +require 'active_support/core_ext/module/delegation' module ActiveSupport #:nodoc: module Multibyte #:nodoc: @@ -34,10 +35,13 @@ module ActiveSupport #:nodoc: # # ActiveSupport::Multibyte.proxy_class = CharsForUTF32 class Chars + include Comparable attr_reader :wrapped_string alias to_s wrapped_string alias to_str wrapped_string + delegate :<=>, :=~, :acts_like_string?, :to => :wrapped_string + # Creates a new Chars instance by wrapping _string_. def initialize(string) @wrapped_string = string @@ -47,8 +51,8 @@ module ActiveSupport #:nodoc: # Forward all undefined methods to the wrapped string. def method_missing(method, *args, &block) if method.to_s =~ /!$/ - @wrapped_string.__send__(method, *args, &block) - self + result = @wrapped_string.__send__(method, *args, &block) + self if result else result = @wrapped_string.__send__(method, *args, &block) result.kind_of?(String) ? chars(result) : result @@ -61,35 +65,9 @@ module ActiveSupport #:nodoc: super || @wrapped_string.respond_to?(method, include_private) end - # Enable more predictable duck-typing on String-like classes. See Object#acts_like?. - def acts_like_string? - true - end - # Returns +true+ when the proxy class can handle the string. Returns +false+ otherwise. def self.consumes?(string) - # Unpack is a little bit faster than regular expressions. - string.unpack('U*') - true - rescue ArgumentError - false - end - - include Comparable - - # Returns -1, 0, or 1, depending on whether the Chars object is to be sorted before, - # equal or after the object on the right side of the operation. It accepts any object - # that implements +to_s+: - # - # 'é'.mb_chars <=> 'ü'.mb_chars # => -1 - # - # See <tt>String#<=></tt> for more details. - def <=>(other) - @wrapped_string <=> other.to_s - end - - def =~(other) - @wrapped_string =~ other + string.encoding == Encoding::UTF_8 end # Works just like <tt>String#split</tt>, with the exception that the items in the resulting list are Chars @@ -101,45 +79,10 @@ module ActiveSupport #:nodoc: @wrapped_string.split(*args).map { |i| i.mb_chars } end - # Like <tt>String#[]=</tt>, except instead of byte offsets you specify character offsets. - # - # Example: - # - # s = "Müller" - # s.mb_chars[2] = "e" # Replace character with offset 2 - # s - # # => "Müeler" - # - # s = "Müller" - # s.mb_chars[1, 2] = "ö" # Replace 2 characters at character offset 1 - # s - # # => "Möler" - def []=(*args) - replace_by = args.pop - # Indexed replace with regular expressions already works - if args.first.is_a?(Regexp) - @wrapped_string[*args] = replace_by - else - result = Unicode.u_unpack(@wrapped_string) - case args.first - when Fixnum - raise IndexError, "index #{args[0]} out of string" if args[0] >= result.length - min = args[0] - max = args[1].nil? ? min : (min + args[1] - 1) - range = Range.new(min, max) - replace_by = [replace_by].pack('U') if replace_by.is_a?(Fixnum) - when Range - raise RangeError, "#{args[0]} out of range" if args[0].min >= result.length - range = args[0] - else - needle = args[0].to_s - min = index(needle) - max = min + Unicode.u_unpack(needle).length - 1 - range = Range.new(min, max) - end - result[range] = Unicode.u_unpack(replace_by) - @wrapped_string.replace(result.pack('U*')) - end + # Works like like <tt>String#slice!</tt>, but returns an instance of Chars, or nil if the string was not + # modified. + def slice!(*args) + chars(@wrapped_string.slice!(*args)) end # Reverses all characters in the string. @@ -147,36 +90,8 @@ module ActiveSupport #:nodoc: # Example: # 'Café'.mb_chars.reverse.to_s # => 'éfaC' def reverse - chars(Unicode.g_unpack(@wrapped_string).reverse.flatten.pack('U*')) - end - - # Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that - # character. - # - # Example: - # 'こんにちは'.mb_chars.slice(2..3).to_s # => "にち" - def slice(*args) - if args.size > 2 - raise ArgumentError, "wrong number of arguments (#{args.size} for 1)" # Do as if we were native - elsif (args.size == 2 && !(args.first.is_a?(Numeric) || args.first.is_a?(Regexp))) - raise TypeError, "cannot convert #{args.first.class} into Integer" # Do as if we were native - elsif (args.size == 2 && !args[1].is_a?(Numeric)) - raise TypeError, "cannot convert #{args[1].class} into Integer" # Do as if we were native - elsif args[0].kind_of? Range - cps = Unicode.u_unpack(@wrapped_string).slice(*args) - result = cps.nil? ? nil : cps.pack('U*') - elsif args[0].kind_of? Regexp - result = @wrapped_string.slice(*args) - elsif args.size == 1 && args[0].kind_of?(Numeric) - character = Unicode.u_unpack(@wrapped_string)[args[0]] - result = character && [character].pack('U') - else - cps = Unicode.u_unpack(@wrapped_string).slice(*args) - result = cps && cps.pack('U*') - end - result && chars(result) + chars(Unicode.unpack_graphemes(@wrapped_string).reverse.flatten.pack('U*')) end - alias_method :[], :slice # Limit the byte size of the string to a number of bytes without breaking characters. Usable # when the storage for a string is limited for some reason. @@ -192,7 +107,7 @@ module ActiveSupport #:nodoc: # Example: # 'Laurent, où sont les tests ?'.mb_chars.upcase.to_s # => "LAURENT, OÙ SONT LES TESTS ?" def upcase - chars(Unicode.apply_mapping @wrapped_string, :uppercase_mapping) + chars Unicode.upcase(@wrapped_string) end # Convert characters in the string to lowercase. @@ -200,7 +115,7 @@ module ActiveSupport #:nodoc: # Example: # 'VĚDA A VÝZKUM'.mb_chars.downcase.to_s # => "věda a výzkum" def downcase - chars(Unicode.apply_mapping @wrapped_string, :lowercase_mapping) + chars Unicode.downcase(@wrapped_string) end # Converts the first character to uppercase and the remainder to lowercase. @@ -217,7 +132,7 @@ module ActiveSupport #:nodoc: # "ÉL QUE SE ENTERÓ".mb_chars.titleize # => "Él Que Se Enteró" # "日本語".mb_chars.titleize # => "日本語" def titleize - chars(downcase.to_s.gsub(/\b('?[\S])/u) { Unicode.apply_mapping $1, :uppercase_mapping }) + chars(downcase.to_s.gsub(/\b('?[\S])/u) { Unicode.upcase($1)}) end alias_method :titlecase, :titleize @@ -237,7 +152,7 @@ module ActiveSupport #:nodoc: # 'é'.length # => 2 # 'é'.mb_chars.decompose.to_s.length # => 3 def decompose - chars(Unicode.decompose_codepoints(:canonical, Unicode.u_unpack(@wrapped_string)).pack('U*')) + chars(Unicode.decompose(:canonical, @wrapped_string.codepoints.to_a).pack('U*')) end # Performs composition on all the characters. @@ -246,16 +161,16 @@ module ActiveSupport #:nodoc: # 'é'.length # => 3 # 'é'.mb_chars.compose.to_s.length # => 2 def compose - chars(Unicode.compose_codepoints(Unicode.u_unpack(@wrapped_string)).pack('U*')) + chars(Unicode.compose(@wrapped_string.codepoints.to_a).pack('U*')) end # Returns the number of grapheme clusters in the string. # # Example: # 'क्षि'.mb_chars.length # => 4 - # 'क्षि'.mb_chars.g_length # => 3 - def g_length - Unicode.g_unpack(@wrapped_string).length + # 'क्षि'.mb_chars.grapheme_length # => 3 + def grapheme_length + Unicode.unpack_graphemes(@wrapped_string).length end # Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string. @@ -265,14 +180,10 @@ module ActiveSupport #:nodoc: chars(Unicode.tidy_bytes(@wrapped_string, force)) end - %w(capitalize downcase lstrip reverse rstrip slice strip tidy_bytes upcase).each do |method| - # Only define a corresponding bang method for methods defined in the proxy; On 1.9 the proxy will - # exclude lstrip!, rstrip! and strip! because they are already work as expected on multibyte strings. - if public_method_defined?(method) - define_method("#{method}!") do |*args| - @wrapped_string = send(args.nil? ? method : method, *args).to_s - self - end + %w(capitalize downcase reverse tidy_bytes upcase).each do |method| + define_method("#{method}!") do |*args| + @wrapped_string = send(method, *args).to_s + self end end @@ -282,41 +193,14 @@ module ActiveSupport #:nodoc: return nil if byte_offset.nil? return 0 if @wrapped_string == '' - @wrapped_string = @wrapped_string.dup.force_encoding(Encoding::ASCII_8BIT) - begin - @wrapped_string[0...byte_offset].unpack('U*').length + @wrapped_string.byteslice(0...byte_offset).unpack('U*').length rescue ArgumentError byte_offset -= 1 retry end end - def justify(integer, way, padstr=' ') #:nodoc: - raise ArgumentError, "zero width padding" if padstr.length == 0 - padsize = integer - size - padsize = padsize > 0 ? padsize : 0 - case way - when :right - result = @wrapped_string.dup.insert(0, padding(padsize, padstr)) - when :left - result = @wrapped_string.dup.insert(-1, padding(padsize, padstr)) - when :center - lpad = padding((padsize / 2.0).floor, padstr) - rpad = padding((padsize / 2.0).ceil, padstr) - result = @wrapped_string.dup.insert(0, lpad).insert(-1, rpad) - end - chars(result) - end - - def padding(padsize, padstr=' ') #:nodoc: - if padsize != 0 - chars(padstr * ((padsize / Unicode.u_unpack(padstr).size) + 1)).slice(0, padsize) - else - '' - end - end - def chars(string) #:nodoc: self.class.new(string) end diff --git a/activesupport/lib/active_support/multibyte/exceptions.rb b/activesupport/lib/active_support/multibyte/exceptions.rb deleted file mode 100644 index 62066e3c71..0000000000 --- a/activesupport/lib/active_support/multibyte/exceptions.rb +++ /dev/null @@ -1,8 +0,0 @@ -# encoding: utf-8 - -module ActiveSupport #:nodoc: - module Multibyte #:nodoc: - # Raised when a problem with the encoding was found. - class EncodingError < StandardError; end - end -end
\ No newline at end of file diff --git a/activesupport/lib/active_support/multibyte/unicode.rb b/activesupport/lib/active_support/multibyte/unicode.rb index 754ca9290b..94fb0a48aa 100644 --- a/activesupport/lib/active_support/multibyte/unicode.rb +++ b/activesupport/lib/active_support/multibyte/unicode.rb @@ -10,7 +10,7 @@ module ActiveSupport NORMALIZATION_FORMS = [:c, :kc, :d, :kd] # The Unicode version that is supported by the implementation - UNICODE_VERSION = '5.2.0' + UNICODE_VERSION = '6.0.0' # The default normalization used for operations that require normalization. It can be set to any of the # normalizations in NORMALIZATION_FORMS. @@ -61,19 +61,6 @@ module ActiveSupport TRAILERS_PAT = /(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+\Z/u LEADERS_PAT = /\A(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+/u - # Unpack the string at codepoints boundaries. Raises an EncodingError when the encoding of the string isn't - # valid UTF-8. - # - # Example: - # Unicode.u_unpack('Café') # => [67, 97, 102, 233] - def u_unpack(string) - begin - string.unpack 'U*' - rescue ArgumentError - raise EncodingError, 'malformed UTF-8 character' - end - end - # Detect whether the codepoint is in a certain character class. Returns +true+ when it's in the specified # character class and +false+ otherwise. Valid character classes are: <tt>:cr</tt>, <tt>:lf</tt>, <tt>:l</tt>, # <tt>:v</tt>, <tt>:lv</tt>, <tt>:lvt</tt> and <tt>:t</tt>. @@ -86,10 +73,10 @@ module ActiveSupport # Unpack the string at grapheme boundaries. Returns a list of character lists. # # Example: - # Unicode.g_unpack('क्षि') # => [[2325, 2381], [2359], [2367]] - # Unicode.g_unpack('Café') # => [[67], [97], [102], [233]] - def g_unpack(string) - codepoints = u_unpack(string) + # Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]] + # Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]] + def unpack_graphemes(string) + codepoints = string.codepoints.to_a unpacked = [] pos = 0 marker = 0 @@ -118,12 +105,12 @@ module ActiveSupport unpacked end - # Reverse operation of g_unpack. + # Reverse operation of unpack_graphemes. # # Example: - # Unicode.g_pack(Unicode.g_unpack('क्षि')) # => 'क्षि' - def g_pack(unpacked) - (unpacked.flatten).pack('U*') + # Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि' + def pack_graphemes(unpacked) + unpacked.flatten.pack('U*') end # Re-order codepoints so the string becomes canonical. @@ -143,7 +130,7 @@ module ActiveSupport end # Decompose composed characters to the decomposed form. - def decompose_codepoints(type, codepoints) + def decompose(type, codepoints) codepoints.inject([]) do |decomposed, cp| # if it's a hangul syllable starter character if HANGUL_SBASE <= cp and cp < HANGUL_SLAST @@ -156,7 +143,7 @@ module ActiveSupport decomposed.concat ncp # if the codepoint is decomposable in with the current decomposition type elsif (ncp = database.codepoints[cp].decomp_mapping) and (!database.codepoints[cp].decomp_type || type == :compatability) - decomposed.concat decompose_codepoints(type, ncp.dup) + decomposed.concat decompose(type, ncp.dup) else decomposed << cp end @@ -164,7 +151,7 @@ module ActiveSupport end # Compose decomposed characters to the composed form. - def compose_codepoints(codepoints) + def compose(codepoints) pos = 0 eoa = codepoints.length - 1 starter_pos = 0 @@ -283,30 +270,27 @@ module ActiveSupport def normalize(string, form=nil) form ||= @default_normalization_form # See http://www.unicode.org/reports/tr15, Table 1 - codepoints = u_unpack(string) + codepoints = string.codepoints.to_a case form when :d - reorder_characters(decompose_codepoints(:canonical, codepoints)) + reorder_characters(decompose(:canonical, codepoints)) when :c - compose_codepoints(reorder_characters(decompose_codepoints(:canonical, codepoints))) + compose(reorder_characters(decompose(:canonical, codepoints))) when :kd - reorder_characters(decompose_codepoints(:compatability, codepoints)) + reorder_characters(decompose(:compatability, codepoints)) when :kc - compose_codepoints(reorder_characters(decompose_codepoints(:compatability, codepoints))) + compose(reorder_characters(decompose(:compatability, codepoints))) else raise ArgumentError, "#{form} is not a valid normalization variant", caller end.pack('U*') end - def apply_mapping(string, mapping) #:nodoc: - u_unpack(string).map do |codepoint| - cp = database.codepoints[codepoint] - if cp and (ncp = cp.send(mapping)) and ncp > 0 - ncp - else - codepoint - end - end.pack('U*') + def downcase(string) + apply_mapping string, :lowercase_mapping + end + + def upcase(string) + apply_mapping string, :uppercase_mapping end # Holds data about a codepoint in the Unicode database @@ -374,6 +358,17 @@ module ActiveSupport private + def apply_mapping(string, mapping) #:nodoc: + string.each_codepoint.map do |codepoint| + cp = database.codepoints[codepoint] + if cp and (ncp = cp.send(mapping)) and ncp > 0 + ncp + else + codepoint + end + end.pack('U*') + end + def tidy_byte(byte) if byte < 160 [database.cp1252[byte] || byte].pack("U").unpack("C*") diff --git a/activesupport/lib/active_support/multibyte/utils.rb b/activesupport/lib/active_support/multibyte/utils.rb deleted file mode 100644 index bd6d4bad41..0000000000 --- a/activesupport/lib/active_support/multibyte/utils.rb +++ /dev/null @@ -1,27 +0,0 @@ -# encoding: utf-8 - -module ActiveSupport #:nodoc: - module Multibyte #:nodoc: - # Returns a regular expression that matches valid characters in the current encoding - def self.valid_character - VALID_CHARACTER[Encoding.default_external.to_s] - end - - # Verifies the encoding of a string - def self.verify(string) - string.valid_encoding? - end - - # Verifies the encoding of the string and raises an exception when it's not valid - def self.verify!(string) - raise EncodingError.new("Found characters with invalid encoding") unless verify(string) - end - - # Removes all invalid characters from the string. - # - # Note: this method is a no-op in Ruby 1.9 - def self.clean(string) - string - end - end -end diff --git a/activesupport/lib/active_support/values/unicode_tables.dat b/activesupport/lib/active_support/values/unicode_tables.dat Binary files differindex 4fe0268cca..7edc4663e8 100644 --- a/activesupport/lib/active_support/values/unicode_tables.dat +++ b/activesupport/lib/active_support/values/unicode_tables.dat diff --git a/activesupport/test/buffered_logger_test.rb b/activesupport/test/buffered_logger_test.rb index b9891c74c8..4ed425732a 100644 --- a/activesupport/test/buffered_logger_test.rb +++ b/activesupport/test/buffered_logger_test.rb @@ -3,7 +3,6 @@ require 'multibyte_test_helpers' require 'stringio' require 'fileutils' require 'tempfile' -require 'active_support/buffered_logger' require 'active_support/testing/deprecation' class BufferedLoggerTest < Test::Unit::TestCase diff --git a/activesupport/test/caching_test.rb b/activesupport/test/caching_test.rb index aa6fb14e7b..3454c378d3 100644 --- a/activesupport/test/caching_test.rb +++ b/activesupport/test/caching_test.rb @@ -681,7 +681,7 @@ uses_memcached 'memcached backed store' do @data = @cache.instance_variable_get(:@data) @cache.clear @cache.silence! - @cache.logger = Logger.new("/dev/null") + @cache.logger = ActiveSupport::Logger.new("/dev/null") end include CacheStoreBehavior @@ -786,7 +786,7 @@ class CacheStoreLoggerTest < ActiveSupport::TestCase @cache = ActiveSupport::Cache.lookup_store(:memory_store) @buffer = StringIO.new - @cache.logger = Logger.new(@buffer) + @cache.logger = ActiveSupport::Logger.new(@buffer) end def test_logging diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb index 20e56e2c81..e36bd22081 100644 --- a/activesupport/test/multibyte_chars_test.rb +++ b/activesupport/test/multibyte_chars_test.rb @@ -7,6 +7,7 @@ class String def __method_for_multibyte_testing_with_integer_result; 1; end def __method_for_multibyte_testing; 'result'; end def __method_for_multibyte_testing!; 'result'; end + def __method_for_multibyte_testing_that_returns_nil!; end end class MultibyteCharsTest < Test::Unit::TestCase @@ -36,11 +37,15 @@ class MultibyteCharsTest < Test::Unit::TestCase assert_not_equal @chars.object_id, @chars.__method_for_multibyte_testing.object_id end - def test_forwarded_bang_method_calls_should_return_the_original_chars_instance + def test_forwarded_bang_method_calls_should_return_the_original_chars_instance_when_result_is_not_nil assert_kind_of @proxy_class, @chars.__method_for_multibyte_testing! assert_equal @chars.object_id, @chars.__method_for_multibyte_testing!.object_id end + def test_forwarded_bang_method_calls_should_return_nil_when_result_is_nil + assert_nil @chars.__method_for_multibyte_testing_that_returns_nil! + end + def test_methods_are_forwarded_to_wrapped_string_for_byte_strings assert_equal BYTE_STRING.class, BYTE_STRING.mb_chars.class end @@ -67,17 +72,6 @@ class MultibyteCharsTest < Test::Unit::TestCase assert !@proxy_class.consumes?(BYTE_STRING) end - def test_unpack_utf8_strings - assert_equal 4, ActiveSupport::Multibyte::Unicode.u_unpack(UNICODE_STRING).length - assert_equal 5, ActiveSupport::Multibyte::Unicode.u_unpack(ASCII_STRING).length - end - - def test_unpack_raises_encoding_error_on_broken_strings - assert_raise(ActiveSupport::Multibyte::EncodingError) do - ActiveSupport::Multibyte::Unicode.u_unpack(BYTE_STRING) - end - end - def test_concatenation_should_return_a_proxy_class_instance assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars + 'b').class assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars << 'b').class @@ -112,15 +106,11 @@ class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase end end - def test_indexed_insert_accepts_fixnums - @chars[2] = 32 - assert_equal 'こに わ', @chars - end - - %w{capitalize downcase lstrip reverse rstrip strip upcase}.each do |method| + %w{capitalize downcase lstrip reverse rstrip upcase}.each do |method| class_eval(<<-EOTESTS) - def test_#{method}_bang_should_return_self - assert_equal @chars.object_id, @chars.send("#{method}!").object_id + def test_#{method}_bang_should_return_self_when_modifying_wrapped_string + chars = ' él piDió Un bUen café ' + assert_equal chars.object_id, chars.send("#{method}!").object_id end def test_#{method}_bang_should_change_wrapped_string @@ -419,7 +409,7 @@ class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase def test_slice_bang_removes_the_slice_from_the_receiver chars = 'úüù'.mb_chars chars.slice!(0,2) - assert_equal 'úü', chars + assert_equal 'ù', chars end def test_slice_should_throw_exceptions_on_invalid_arguments @@ -506,7 +496,7 @@ class MultibyteCharsExtrasTest < Test::Unit::TestCase def test_limit_should_work_on_a_multibyte_string example = chars(UNICODE_STRING) - bytesize = UNICODE_STRING.respond_to?(:bytesize) ? UNICODE_STRING.bytesize : UNICODE_STRING.size + bytesize = UNICODE_STRING.bytesize assert_equal UNICODE_STRING, example.limit(bytesize) assert_equal '', example.limit(0) @@ -605,7 +595,7 @@ class MultibyteCharsExtrasTest < Test::Unit::TestCase else str = input end - assert_equal expected_length, chars(str).g_length + assert_equal expected_length, chars(str).grapheme_length end end diff --git a/activesupport/test/multibyte_utils_test.rb b/activesupport/test/multibyte_utils_test.rb deleted file mode 100644 index f807492be0..0000000000 --- a/activesupport/test/multibyte_utils_test.rb +++ /dev/null @@ -1,93 +0,0 @@ -# encoding: utf-8 - -require 'abstract_unit' -require 'multibyte_test_helpers' - -class MultibyteUtilsTest < ActiveSupport::TestCase - include MultibyteTestHelpers - - test "valid_character returns an expression for the current encoding" do - with_encoding('None') do - assert_nil ActiveSupport::Multibyte.valid_character - end - with_encoding('UTF8') do - assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['UTF-8'], ActiveSupport::Multibyte.valid_character - end - with_encoding('SJIS') do - assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['Shift_JIS'], ActiveSupport::Multibyte.valid_character - end - end - - test "verify verifies ASCII strings are properly encoded" do - with_encoding('None') do - examples.each do |example| - assert ActiveSupport::Multibyte.verify(example) - end - end - end - - test "verify verifies UTF-8 strings are properly encoded" do - with_encoding('UTF8') do - assert ActiveSupport::Multibyte.verify(example('valid UTF-8')) - assert !ActiveSupport::Multibyte.verify(example('invalid UTF-8')) - end - end - - test "verify verifies Shift-JIS strings are properly encoded" do - with_encoding('SJIS') do - assert ActiveSupport::Multibyte.verify(example('valid Shift-JIS')) - assert !ActiveSupport::Multibyte.verify(example('invalid Shift-JIS')) - end - end - - test "verify! raises an exception when it finds an invalid character" do - with_encoding('UTF8') do - assert_raises(ActiveSupport::Multibyte::EncodingError) do - ActiveSupport::Multibyte.verify!(example('invalid UTF-8')) - end - end - end - - test "verify! doesn't raise an exception when the encoding is valid" do - with_encoding('UTF8') do - assert_nothing_raised do - ActiveSupport::Multibyte.verify!(example('valid UTF-8')) - end - end - end - - test "clean is a no-op" do - with_encoding('UTF8') do - assert_equal example('invalid Shift-JIS'), ActiveSupport::Multibyte.clean(example('invalid Shift-JIS')) - end - end - - private - - STRINGS = { - 'valid ASCII' => [65, 83, 67, 73, 73].pack('C*'), - 'invalid ASCII' => [128].pack('C*'), - 'valid UTF-8' => [227, 129, 147, 227, 129, 171, 227, 129, 161, 227, 130, 143].pack('C*'), - 'invalid UTF-8' => [184, 158, 8, 136, 165].pack('C*'), - 'valid Shift-JIS' => [131, 122, 129, 91, 131, 128].pack('C*'), - 'invalid Shift-JIS' => [184, 158, 8, 0, 255, 136, 165].pack('C*') - } - - def example(key) - STRINGS[key].force_encoding(Encoding.default_external) - end - - def examples - STRINGS.values.map { |s| s.force_encoding(Encoding.default_external) } - end - - KCODE_TO_ENCODING = Hash.new(Encoding::BINARY). - update('UTF8' => Encoding::UTF_8, 'SJIS' => Encoding::Shift_JIS) - - def with_encoding(enc) - before = Encoding.default_external - silence_warnings { Encoding.default_external = KCODE_TO_ENCODING[enc] } - yield - silence_warnings { Encoding.default_external = before } - end -end |