diff options
Diffstat (limited to 'activesupport/lib/active_support/multibyte')
4 files changed, 87 insertions, 406 deletions
diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index b78d92f599..9a748dfa60 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -1,6 +1,8 @@ # encoding: utf-8 +require 'active_support/json' require 'active_support/core_ext/string/access' require 'active_support/core_ext/string/behavior' +require 'active_support/core_ext/module/delegation' module ActiveSupport #:nodoc: module Multibyte #:nodoc: @@ -34,27 +36,24 @@ module ActiveSupport #:nodoc: # # ActiveSupport::Multibyte.proxy_class = CharsForUTF32 class Chars + include Comparable attr_reader :wrapped_string alias to_s wrapped_string alias to_str wrapped_string - if RUBY_VERSION >= "1.9" - # Creates a new Chars instance by wrapping _string_. - def initialize(string) - @wrapped_string = string - @wrapped_string.force_encoding(Encoding::UTF_8) unless @wrapped_string.frozen? - end - else - def initialize(string) #:nodoc: - @wrapped_string = string - end + delegate :<=>, :=~, :acts_like_string?, :to => :wrapped_string + + # Creates a new Chars instance by wrapping _string_. + def initialize(string) + @wrapped_string = string + @wrapped_string.force_encoding(Encoding::UTF_8) unless @wrapped_string.frozen? end # Forward all undefined methods to the wrapped string. def method_missing(method, *args, &block) if method.to_s =~ /!$/ - @wrapped_string.__send__(method, *args, &block) - self + result = @wrapped_string.__send__(method, *args, &block) + self if result else result = @wrapped_string.__send__(method, *args, &block) result.kind_of?(String) ? chars(result) : result @@ -67,178 +66,9 @@ module ActiveSupport #:nodoc: super || @wrapped_string.respond_to?(method, include_private) end - # Enable more predictable duck-typing on String-like classes. See Object#acts_like?. - def acts_like_string? - true - end - # Returns +true+ when the proxy class can handle the string. Returns +false+ otherwise. def self.consumes?(string) - # Unpack is a little bit faster than regular expressions. - string.unpack('U*') - true - rescue ArgumentError - false - end - - include Comparable - - # Returns -1, 0, or 1, depending on whether the Chars object is to be sorted before, - # equal or after the object on the right side of the operation. It accepts any object - # that implements +to_s+: - # - # 'é'.mb_chars <=> 'ü'.mb_chars # => -1 - # - # See <tt>String#<=></tt> for more details. - def <=>(other) - @wrapped_string <=> other.to_s - end - - if RUBY_VERSION < "1.9" - # Returns +true+ if the Chars class can and should act as a proxy for the string _string_. Returns - # +false+ otherwise. - def self.wants?(string) - $KCODE == 'UTF8' && consumes?(string) - end - - # Returns a new Chars object containing the _other_ object concatenated to the string. - # - # Example: - # ('Café'.mb_chars + ' périferôl').to_s # => "Café périferôl" - def +(other) - chars(@wrapped_string + other) - end - - # Like <tt>String#=~</tt> only it returns the character offset (in codepoints) instead of the byte offset. - # - # Example: - # 'Café périferôl'.mb_chars =~ /ô/ # => 12 - def =~(other) - translate_offset(@wrapped_string =~ other) - end - - # Inserts the passed string at specified codepoint offsets. - # - # Example: - # 'Café'.mb_chars.insert(4, ' périferôl').to_s # => "Café périferôl" - def insert(offset, fragment) - unpacked = Unicode.u_unpack(@wrapped_string) - unless offset > unpacked.length - @wrapped_string.replace( - Unicode.u_unpack(@wrapped_string).insert(offset, *Unicode.u_unpack(fragment)).pack('U*') - ) - else - raise IndexError, "index #{offset} out of string" - end - self - end - - # Returns +true+ if contained string contains _other_. Returns +false+ otherwise. - # - # Example: - # 'Café'.mb_chars.include?('é') # => true - def include?(other) - # We have to redefine this method because Enumerable defines it. - @wrapped_string.include?(other) - end - - # Returns the position _needle_ in the string, counting in codepoints. Returns +nil+ if _needle_ isn't found. - # - # Example: - # 'Café périferôl'.mb_chars.index('ô') # => 12 - # 'Café périferôl'.mb_chars.index(/\w/u) # => 0 - def index(needle, offset=0) - wrapped_offset = first(offset).wrapped_string.length - index = @wrapped_string.index(needle, wrapped_offset) - index ? (Unicode.u_unpack(@wrapped_string.slice(0...index)).size) : nil - end - - # Returns the position _needle_ in the string, counting in - # codepoints, searching backward from _offset_ or the end of the - # string. Returns +nil+ if _needle_ isn't found. - # - # Example: - # 'Café périferôl'.mb_chars.rindex('é') # => 6 - # 'Café périferôl'.mb_chars.rindex(/\w/u) # => 13 - def rindex(needle, offset=nil) - offset ||= length - wrapped_offset = first(offset).wrapped_string.length - index = @wrapped_string.rindex(needle, wrapped_offset) - index ? (Unicode.u_unpack(@wrapped_string.slice(0...index)).size) : nil - end - - # Returns the number of codepoints in the string - def size - Unicode.u_unpack(@wrapped_string).size - end - alias_method :length, :size - - # Strips entire range of Unicode whitespace from the right of the string. - def rstrip - chars(@wrapped_string.gsub(Unicode::TRAILERS_PAT, '')) - end - - # Strips entire range of Unicode whitespace from the left of the string. - def lstrip - chars(@wrapped_string.gsub(Unicode::LEADERS_PAT, '')) - end - - # Strips entire range of Unicode whitespace from the right and left of the string. - def strip - rstrip.lstrip - end - - # Returns the codepoint of the first character in the string. - # - # Example: - # 'こんにちは'.mb_chars.ord # => 12371 - def ord - Unicode.u_unpack(@wrapped_string)[0] - end - - # Works just like <tt>String#rjust</tt>, only integer specifies characters instead of bytes. - # - # Example: - # - # "¾ cup".mb_chars.rjust(8).to_s - # # => " ¾ cup" - # - # "¾ cup".mb_chars.rjust(8, " ").to_s # Use non-breaking whitespace - # # => " ¾ cup" - def rjust(integer, padstr=' ') - justify(integer, :right, padstr) - end - - # Works just like <tt>String#ljust</tt>, only integer specifies characters instead of bytes. - # - # Example: - # - # "¾ cup".mb_chars.rjust(8).to_s - # # => "¾ cup " - # - # "¾ cup".mb_chars.rjust(8, " ").to_s # Use non-breaking whitespace - # # => "¾ cup " - def ljust(integer, padstr=' ') - justify(integer, :left, padstr) - end - - # Works just like <tt>String#center</tt>, only integer specifies characters instead of bytes. - # - # Example: - # - # "¾ cup".mb_chars.center(8).to_s - # # => " ¾ cup " - # - # "¾ cup".mb_chars.center(8, " ").to_s # Use non-breaking whitespace - # # => " ¾ cup " - def center(integer, padstr=' ') - justify(integer, :center, padstr) - end - - else - def =~(other) - @wrapped_string =~ other - end + string.encoding == Encoding::UTF_8 end # Works just like <tt>String#split</tt>, with the exception that the items in the resulting list are Chars @@ -250,45 +80,10 @@ module ActiveSupport #:nodoc: @wrapped_string.split(*args).map { |i| i.mb_chars } end - # Like <tt>String#[]=</tt>, except instead of byte offsets you specify character offsets. - # - # Example: - # - # s = "Müller" - # s.mb_chars[2] = "e" # Replace character with offset 2 - # s - # # => "Müeler" - # - # s = "Müller" - # s.mb_chars[1, 2] = "ö" # Replace 2 characters at character offset 1 - # s - # # => "Möler" - def []=(*args) - replace_by = args.pop - # Indexed replace with regular expressions already works - if args.first.is_a?(Regexp) - @wrapped_string[*args] = replace_by - else - result = Unicode.u_unpack(@wrapped_string) - case args.first - when Fixnum - raise IndexError, "index #{args[0]} out of string" if args[0] >= result.length - min = args[0] - max = args[1].nil? ? min : (min + args[1] - 1) - range = Range.new(min, max) - replace_by = [replace_by].pack('U') if replace_by.is_a?(Fixnum) - when Range - raise RangeError, "#{args[0]} out of range" if args[0].min >= result.length - range = args[0] - else - needle = args[0].to_s - min = index(needle) - max = min + Unicode.u_unpack(needle).length - 1 - range = Range.new(min, max) - end - result[range] = Unicode.u_unpack(replace_by) - @wrapped_string.replace(result.pack('U*')) - end + # Works like like <tt>String#slice!</tt>, but returns an instance of Chars, or nil if the string was not + # modified. + def slice!(*args) + chars(@wrapped_string.slice!(*args)) end # Reverses all characters in the string. @@ -296,38 +91,10 @@ module ActiveSupport #:nodoc: # Example: # 'Café'.mb_chars.reverse.to_s # => 'éfaC' def reverse - chars(Unicode.g_unpack(@wrapped_string).reverse.flatten.pack('U*')) + chars(Unicode.unpack_graphemes(@wrapped_string).reverse.flatten.pack('U*')) end - # Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that - # character. - # - # Example: - # 'こんにちは'.mb_chars.slice(2..3).to_s # => "にち" - def slice(*args) - if args.size > 2 - raise ArgumentError, "wrong number of arguments (#{args.size} for 1)" # Do as if we were native - elsif (args.size == 2 && !(args.first.is_a?(Numeric) || args.first.is_a?(Regexp))) - raise TypeError, "cannot convert #{args.first.class} into Integer" # Do as if we were native - elsif (args.size == 2 && !args[1].is_a?(Numeric)) - raise TypeError, "cannot convert #{args[1].class} into Integer" # Do as if we were native - elsif args[0].kind_of? Range - cps = Unicode.u_unpack(@wrapped_string).slice(*args) - result = cps.nil? ? nil : cps.pack('U*') - elsif args[0].kind_of? Regexp - result = @wrapped_string.slice(*args) - elsif args.size == 1 && args[0].kind_of?(Numeric) - character = Unicode.u_unpack(@wrapped_string)[args[0]] - result = character && [character].pack('U') - else - cps = Unicode.u_unpack(@wrapped_string).slice(*args) - result = cps && cps.pack('U*') - end - result && chars(result) - end - alias_method :[], :slice - - # Limit the byte size of the string to a number of bytes without breaking characters. Usable + # Limits the byte size of the string to a number of bytes without breaking characters. Usable # when the storage for a string is limited for some reason. # # Example: @@ -336,20 +103,28 @@ module ActiveSupport #:nodoc: slice(0...translate_offset(limit)) end - # Convert characters in the string to uppercase. + # Converts characters in the string to uppercase. # # Example: # 'Laurent, où sont les tests ?'.mb_chars.upcase.to_s # => "LAURENT, OÙ SONT LES TESTS ?" def upcase - chars(Unicode.apply_mapping @wrapped_string, :uppercase_mapping) + chars Unicode.upcase(@wrapped_string) end - # Convert characters in the string to lowercase. + # Converts characters in the string to lowercase. # # Example: # 'VĚDA A VÝZKUM'.mb_chars.downcase.to_s # => "věda a výzkum" def downcase - chars(Unicode.apply_mapping @wrapped_string, :lowercase_mapping) + chars Unicode.downcase(@wrapped_string) + end + + # Converts characters in the string to the opposite case. + # + # Example: + # 'El Cañón".mb_chars.swapcase.to_s # => "eL cAÑÓN" + def swapcase + chars Unicode.swapcase(@wrapped_string) end # Converts the first character to uppercase and the remainder to lowercase. @@ -366,7 +141,7 @@ module ActiveSupport #:nodoc: # "ÉL QUE SE ENTERÓ".mb_chars.titleize # => "Él Que Se Enteró" # "日本語".mb_chars.titleize # => "日本語" def titleize - chars(downcase.to_s.gsub(/\b('?[\S])/u) { Unicode.apply_mapping $1, :uppercase_mapping }) + chars(downcase.to_s.gsub(/\b('?[\S])/u) { Unicode.upcase($1)}) end alias_method :titlecase, :titleize @@ -386,7 +161,7 @@ module ActiveSupport #:nodoc: # 'é'.length # => 2 # 'é'.mb_chars.decompose.to_s.length # => 3 def decompose - chars(Unicode.decompose_codepoints(:canonical, Unicode.u_unpack(@wrapped_string)).pack('U*')) + chars(Unicode.decompose(:canonical, @wrapped_string.codepoints.to_a).pack('U*')) end # Performs composition on all the characters. @@ -395,16 +170,16 @@ module ActiveSupport #:nodoc: # 'é'.length # => 3 # 'é'.mb_chars.compose.to_s.length # => 2 def compose - chars(Unicode.compose_codepoints(Unicode.u_unpack(@wrapped_string)).pack('U*')) + chars(Unicode.compose(@wrapped_string.codepoints.to_a).pack('U*')) end # Returns the number of grapheme clusters in the string. # # Example: # 'क्षि'.mb_chars.length # => 4 - # 'क्षि'.mb_chars.g_length # => 3 - def g_length - Unicode.g_unpack(@wrapped_string).length + # 'क्षि'.mb_chars.grapheme_length # => 3 + def grapheme_length + Unicode.unpack_graphemes(@wrapped_string).length end # Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string. @@ -414,14 +189,14 @@ module ActiveSupport #:nodoc: chars(Unicode.tidy_bytes(@wrapped_string, force)) end - %w(capitalize downcase lstrip reverse rstrip slice strip tidy_bytes upcase).each do |method| - # Only define a corresponding bang method for methods defined in the proxy; On 1.9 the proxy will - # exclude lstrip!, rstrip! and strip! because they are already work as expected on multibyte strings. - if public_method_defined?(method) - define_method("#{method}!") do |*args| - @wrapped_string = send(args.nil? ? method : method, *args).to_s - self - end + def as_json(options = nil) #:nodoc: + to_s.as_json(options) + end + + %w(capitalize downcase reverse tidy_bytes upcase).each do |method| + define_method("#{method}!") do |*args| + @wrapped_string = send(method, *args).to_s + self end end @@ -431,43 +206,14 @@ module ActiveSupport #:nodoc: return nil if byte_offset.nil? return 0 if @wrapped_string == '' - if @wrapped_string.respond_to?(:force_encoding) - @wrapped_string = @wrapped_string.dup.force_encoding(Encoding::ASCII_8BIT) - end - begin - @wrapped_string[0...byte_offset].unpack('U*').length + @wrapped_string.byteslice(0...byte_offset).unpack('U*').length rescue ArgumentError byte_offset -= 1 retry end end - def justify(integer, way, padstr=' ') #:nodoc: - raise ArgumentError, "zero width padding" if padstr.length == 0 - padsize = integer - size - padsize = padsize > 0 ? padsize : 0 - case way - when :right - result = @wrapped_string.dup.insert(0, padding(padsize, padstr)) - when :left - result = @wrapped_string.dup.insert(-1, padding(padsize, padstr)) - when :center - lpad = padding((padsize / 2.0).floor, padstr) - rpad = padding((padsize / 2.0).ceil, padstr) - result = @wrapped_string.dup.insert(0, lpad).insert(-1, rpad) - end - chars(result) - end - - def padding(padsize, padstr=' ') #:nodoc: - if padsize != 0 - chars(padstr * ((padsize / Unicode.u_unpack(padstr).size) + 1)).slice(0, padsize) - else - '' - end - end - def chars(string) #:nodoc: self.class.new(string) end diff --git a/activesupport/lib/active_support/multibyte/exceptions.rb b/activesupport/lib/active_support/multibyte/exceptions.rb deleted file mode 100644 index 62066e3c71..0000000000 --- a/activesupport/lib/active_support/multibyte/exceptions.rb +++ /dev/null @@ -1,8 +0,0 @@ -# encoding: utf-8 - -module ActiveSupport #:nodoc: - module Multibyte #:nodoc: - # Raised when a problem with the encoding was found. - class EncodingError < StandardError; end - end -end
\ No newline at end of file diff --git a/activesupport/lib/active_support/multibyte/unicode.rb b/activesupport/lib/active_support/multibyte/unicode.rb index 754ca9290b..cb89d45c92 100644 --- a/activesupport/lib/active_support/multibyte/unicode.rb +++ b/activesupport/lib/active_support/multibyte/unicode.rb @@ -10,7 +10,7 @@ module ActiveSupport NORMALIZATION_FORMS = [:c, :kc, :d, :kd] # The Unicode version that is supported by the implementation - UNICODE_VERSION = '5.2.0' + UNICODE_VERSION = '6.1.0' # The default normalization used for operations that require normalization. It can be set to any of the # normalizations in NORMALIZATION_FORMS. @@ -61,19 +61,6 @@ module ActiveSupport TRAILERS_PAT = /(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+\Z/u LEADERS_PAT = /\A(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+/u - # Unpack the string at codepoints boundaries. Raises an EncodingError when the encoding of the string isn't - # valid UTF-8. - # - # Example: - # Unicode.u_unpack('Café') # => [67, 97, 102, 233] - def u_unpack(string) - begin - string.unpack 'U*' - rescue ArgumentError - raise EncodingError, 'malformed UTF-8 character' - end - end - # Detect whether the codepoint is in a certain character class. Returns +true+ when it's in the specified # character class and +false+ otherwise. Valid character classes are: <tt>:cr</tt>, <tt>:lf</tt>, <tt>:l</tt>, # <tt>:v</tt>, <tt>:lv</tt>, <tt>:lvt</tt> and <tt>:t</tt>. @@ -86,10 +73,10 @@ module ActiveSupport # Unpack the string at grapheme boundaries. Returns a list of character lists. # # Example: - # Unicode.g_unpack('क्षि') # => [[2325, 2381], [2359], [2367]] - # Unicode.g_unpack('Café') # => [[67], [97], [102], [233]] - def g_unpack(string) - codepoints = u_unpack(string) + # Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]] + # Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]] + def unpack_graphemes(string) + codepoints = string.codepoints.to_a unpacked = [] pos = 0 marker = 0 @@ -118,12 +105,12 @@ module ActiveSupport unpacked end - # Reverse operation of g_unpack. + # Reverse operation of unpack_graphemes. # # Example: - # Unicode.g_pack(Unicode.g_unpack('क्षि')) # => 'क्षि' - def g_pack(unpacked) - (unpacked.flatten).pack('U*') + # Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि' + def pack_graphemes(unpacked) + unpacked.flatten.pack('U*') end # Re-order codepoints so the string becomes canonical. @@ -143,7 +130,7 @@ module ActiveSupport end # Decompose composed characters to the decomposed form. - def decompose_codepoints(type, codepoints) + def decompose(type, codepoints) codepoints.inject([]) do |decomposed, cp| # if it's a hangul syllable starter character if HANGUL_SBASE <= cp and cp < HANGUL_SLAST @@ -156,7 +143,7 @@ module ActiveSupport decomposed.concat ncp # if the codepoint is decomposable in with the current decomposition type elsif (ncp = database.codepoints[cp].decomp_mapping) and (!database.codepoints[cp].decomp_type || type == :compatability) - decomposed.concat decompose_codepoints(type, ncp.dup) + decomposed.concat decompose(type, ncp.dup) else decomposed << cp end @@ -164,7 +151,7 @@ module ActiveSupport end # Compose decomposed characters to the composed form. - def compose_codepoints(codepoints) + def compose(codepoints) pos = 0 eoa = codepoints.length - 1 starter_pos = 0 @@ -283,35 +270,40 @@ module ActiveSupport def normalize(string, form=nil) form ||= @default_normalization_form # See http://www.unicode.org/reports/tr15, Table 1 - codepoints = u_unpack(string) + codepoints = string.codepoints.to_a case form when :d - reorder_characters(decompose_codepoints(:canonical, codepoints)) + reorder_characters(decompose(:canonical, codepoints)) when :c - compose_codepoints(reorder_characters(decompose_codepoints(:canonical, codepoints))) + compose(reorder_characters(decompose(:canonical, codepoints))) when :kd - reorder_characters(decompose_codepoints(:compatability, codepoints)) + reorder_characters(decompose(:compatability, codepoints)) when :kc - compose_codepoints(reorder_characters(decompose_codepoints(:compatability, codepoints))) + compose(reorder_characters(decompose(:compatability, codepoints))) else raise ArgumentError, "#{form} is not a valid normalization variant", caller end.pack('U*') end - def apply_mapping(string, mapping) #:nodoc: - u_unpack(string).map do |codepoint| - cp = database.codepoints[codepoint] - if cp and (ncp = cp.send(mapping)) and ncp > 0 - ncp - else - codepoint - end - end.pack('U*') + def downcase(string) + apply_mapping string, :lowercase_mapping + end + + def upcase(string) + apply_mapping string, :uppercase_mapping + end + + def swapcase(string) + apply_mapping string, :swapcase_mapping end # Holds data about a codepoint in the Unicode database class Codepoint attr_accessor :code, :combining_class, :decomp_type, :decomp_mapping, :uppercase_mapping, :lowercase_mapping + + def swapcase_mapping + uppercase_mapping > 0 ? uppercase_mapping : lowercase_mapping + end end # Holds static data from the Unicode database @@ -374,6 +366,17 @@ module ActiveSupport private + def apply_mapping(string, mapping) #:nodoc: + string.each_codepoint.map do |codepoint| + cp = database.codepoints[codepoint] + if cp and (ncp = cp.send(mapping)) and ncp > 0 + ncp + else + codepoint + end + end.pack('U*') + end + def tidy_byte(byte) if byte < 160 [database.cp1252[byte] || byte].pack("U").unpack("C*") diff --git a/activesupport/lib/active_support/multibyte/utils.rb b/activesupport/lib/active_support/multibyte/utils.rb deleted file mode 100644 index 94b393cee2..0000000000 --- a/activesupport/lib/active_support/multibyte/utils.rb +++ /dev/null @@ -1,60 +0,0 @@ -# encoding: utf-8 - -module ActiveSupport #:nodoc: - module Multibyte #:nodoc: - if Kernel.const_defined?(:Encoding) - # Returns a regular expression that matches valid characters in the current encoding - def self.valid_character - VALID_CHARACTER[Encoding.default_external.to_s] - end - else - def self.valid_character - case $KCODE - when 'UTF8' - VALID_CHARACTER['UTF-8'] - when 'SJIS' - VALID_CHARACTER['Shift_JIS'] - end - end - end - - if 'string'.respond_to?(:valid_encoding?) - # Verifies the encoding of a string - def self.verify(string) - string.valid_encoding? - end - else - def self.verify(string) - if expression = valid_character - # Splits the string on character boundaries, which are determined based on $KCODE. - string.split(//).all? { |c| expression =~ c } - else - true - end - end - end - - # Verifies the encoding of the string and raises an exception when it's not valid - def self.verify!(string) - raise EncodingError.new("Found characters with invalid encoding") unless verify(string) - end - - if 'string'.respond_to?(:force_encoding) - # Removes all invalid characters from the string. - # - # Note: this method is a no-op in Ruby 1.9 - def self.clean(string) - string - end - else - def self.clean(string) - if expression = valid_character - # Splits the string on character boundaries, which are determined based on $KCODE. - string.split(//).grep(expression).join - else - string - end - end - end - end -end |