aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--actionpack/lib/action_view/helpers/text_helper.rb206
-rw-r--r--activerecord/lib/active_record/validations.rb2
-rwxr-xr-x[-rw-r--r--]activerecord/test/cases/base_test.rb30
-rw-r--r--activerecord/test/cases/finder_test.rb4
-rw-r--r--activerecord/test/cases/sanitize_test.rb8
-rwxr-xr-xactivesupport/bin/generate_tables147
-rw-r--r--activesupport/lib/active_support/core_ext/string.rb6
-rw-r--r--activesupport/lib/active_support/core_ext/string/access.rb10
-rw-r--r--activesupport/lib/active_support/core_ext/string/multibyte.rb81
-rw-r--r--activesupport/lib/active_support/core_ext/string/unicode.rb66
-rw-r--r--activesupport/lib/active_support/inflector.rb2
-rw-r--r--activesupport/lib/active_support/multibyte.rb36
-rw-r--r--activesupport/lib/active_support/multibyte/chars.rb791
-rw-r--r--activesupport/lib/active_support/multibyte/exceptions.rb8
-rwxr-xr-xactivesupport/lib/active_support/multibyte/generators/generate_tables.rb149
-rw-r--r--activesupport/lib/active_support/multibyte/handlers/passthru_handler.rb9
-rw-r--r--activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb564
-rw-r--r--activesupport/lib/active_support/multibyte/handlers/utf8_handler_proc.rb43
-rw-r--r--activesupport/lib/active_support/multibyte/unicode_database.rb71
-rw-r--r--activesupport/lib/active_support/values/unicode_tables.datbin656156 -> 710734 bytes
-rw-r--r--activesupport/test/abstract_unit.rb19
-rw-r--r--activesupport/test/core_ext/string_ext_test.rb59
-rw-r--r--activesupport/test/json/encoding_test.rb12
-rw-r--r--activesupport/test/multibyte_chars_test.rb705
-rw-r--r--activesupport/test/multibyte_conformance.rb105
-rw-r--r--activesupport/test/multibyte_handler_test.rb372
-rw-r--r--activesupport/test/multibyte_test_helpers.rb19
-rw-r--r--activesupport/test/multibyte_unicode_database_test.rb28
28 files changed, 1828 insertions, 1724 deletions
diff --git a/actionpack/lib/action_view/helpers/text_helper.rb b/actionpack/lib/action_view/helpers/text_helper.rb
index ab1fdc80bc..3af7440400 100644
--- a/actionpack/lib/action_view/helpers/text_helper.rb
+++ b/actionpack/lib/action_view/helpers/text_helper.rb
@@ -42,65 +42,46 @@ module ActionView
output_buffer << string
end
- if RUBY_VERSION < '1.9'
- # Truncates a given +text+ after a given <tt>:length</tt> if +text+ is longer than <tt>:length</tt>
- # (defaults to 30). The last characters will be replaced with the <tt>:omission</tt> (defaults to "...").
- #
- # ==== Examples
- #
- # truncate("Once upon a time in a world far far away")
- # # => Once upon a time in a world f...
- #
- # truncate("Once upon a time in a world far far away", :length => 14)
- # # => Once upon a...
- #
- # truncate("And they found that many people were sleeping better.", :length => 25, "(clipped)")
- # # => And they found that many (clipped)
- #
- # truncate("And they found that many people were sleeping better.", :omission => "... (continued)", :length => 15)
- # # => And they found... (continued)
- #
- # You can still use <tt>truncate</tt> with the old API that accepts the
- # +length+ as its optional second and the +ellipsis+ as its
- # optional third parameter:
- # truncate("Once upon a time in a world far far away", 14)
- # # => Once upon a time in a world f...
- #
- # truncate("And they found that many people were sleeping better.", 15, "... (continued)")
- # # => And they found... (continued)
- def truncate(text, *args)
- options = args.extract_options!
- unless args.empty?
- ActiveSupport::Deprecation.warn('truncate takes an option hash instead of separate ' +
- 'length and omission arguments', caller)
-
- options[:length] = args[0] || 30
- options[:omission] = args[1] || "..."
- end
- options.reverse_merge!(:length => 30, :omission => "...")
+ # Truncates a given +text+ after a given <tt>:length</tt> if +text+ is longer than <tt>:length</tt>
+ # (defaults to 30). The last characters will be replaced with the <tt>:omission</tt> (defaults to "...").
+ #
+ # ==== Examples
+ #
+ # truncate("Once upon a time in a world far far away")
+ # # => Once upon a time in a world f...
+ #
+ # truncate("Once upon a time in a world far far away", :length => 14)
+ # # => Once upon a...
+ #
+ # truncate("And they found that many people were sleeping better.", :length => 25, "(clipped)")
+ # # => And they found that many (clipped)
+ #
+ # truncate("And they found that many people were sleeping better.", :omission => "... (continued)", :length => 15)
+ # # => And they found... (continued)
+ #
+ # You can still use <tt>truncate</tt> with the old API that accepts the
+ # +length+ as its optional second and the +ellipsis+ as its
+ # optional third parameter:
+ # truncate("Once upon a time in a world far far away", 14)
+ # # => Once upon a time in a world f...
+ #
+ # truncate("And they found that many people were sleeping better.", 15, "... (continued)")
+ # # => And they found... (continued)
+ def truncate(text, *args)
+ options = args.extract_options!
+ unless args.empty?
+ ActiveSupport::Deprecation.warn('truncate takes an option hash instead of separate ' +
+ 'length and omission arguments', caller)
- if text
- l = options[:length] - options[:omission].chars.length
- chars = text.chars
- (chars.length > options[:length] ? chars[0...l] + options[:omission] : text).to_s
- end
+ options[:length] = args[0] || 30
+ options[:omission] = args[1] || "..."
end
- else
- def truncate(text, *args) #:nodoc:
- options = args.extract_options!
- unless args.empty?
- ActiveSupport::Deprecation.warn('truncate takes an option hash instead of separate ' +
- 'length and omission arguments', caller)
-
- options[:length] = args[0] || 30
- options[:omission] = args[1] || "..."
- end
- options.reverse_merge!(:length => 30, :omission => "...")
+ options.reverse_merge!(:length => 30, :omission => "...")
- if text
- l = options[:length].to_i - options[:omission].length
- (text.length > options[:length].to_i ? text[0...l] + options[:omission] : text).to_s
- end
+ if text
+ l = options[:length] - options[:omission].mb_chars.length
+ chars = text.mb_chars
+ (chars.length > options[:length] ? chars[0...l] + options[:omission] : text).to_s
end
end
@@ -140,81 +121,54 @@ module ActionView
end
end
- if RUBY_VERSION < '1.9'
- # Extracts an excerpt from +text+ that matches the first instance of +phrase+.
- # The <tt>:radius</tt> option expands the excerpt on each side of the first occurrence of +phrase+ by the number of characters
- # defined in <tt>:radius</tt> (which defaults to 100). If the excerpt radius overflows the beginning or end of the +text+,
- # then the <tt>:omission</tt> option (which defaults to "...") will be prepended/appended accordingly. The resulting string
- # will be stripped in any case. If the +phrase+ isn't found, nil is returned.
- #
- # ==== Examples
- # excerpt('This is an example', 'an', :radius => 5)
- # # => ...s is an exam...
- #
- # excerpt('This is an example', 'is', :radius => 5)
- # # => This is a...
- #
- # excerpt('This is an example', 'is')
- # # => This is an example
- #
- # excerpt('This next thing is an example', 'ex', :radius => 2)
- # # => ...next...
- #
- # excerpt('This is also an example', 'an', :radius => 8, :omission => '<chop> ')
- # # => <chop> is also an example
- #
- # You can still use <tt>excerpt</tt> with the old API that accepts the
- # +radius+ as its optional third and the +ellipsis+ as its
- # optional forth parameter:
- # excerpt('This is an example', 'an', 5) # => ...s is an exam...
- # excerpt('This is also an example', 'an', 8, '<chop> ') # => <chop> is also an example
- def excerpt(text, phrase, *args)
- options = args.extract_options!
- unless args.empty?
- options[:radius] = args[0] || 100
- options[:omission] = args[1] || "..."
- end
- options.reverse_merge!(:radius => 100, :omission => "...")
-
- if text && phrase
- phrase = Regexp.escape(phrase)
-
- if found_pos = text.chars =~ /(#{phrase})/i
- start_pos = [ found_pos - options[:radius], 0 ].max
- end_pos = [ [ found_pos + phrase.chars.length + options[:radius] - 1, 0].max, text.chars.length ].min
-
- prefix = start_pos > 0 ? options[:omission] : ""
- postfix = end_pos < text.chars.length - 1 ? options[:omission] : ""
-
- prefix + text.chars[start_pos..end_pos].strip + postfix
- else
- nil
- end
- end
+ # Extracts an excerpt from +text+ that matches the first instance of +phrase+.
+ # The <tt>:radius</tt> option expands the excerpt on each side of the first occurrence of +phrase+ by the number of characters
+ # defined in <tt>:radius</tt> (which defaults to 100). If the excerpt radius overflows the beginning or end of the +text+,
+ # then the <tt>:omission</tt> option (which defaults to "...") will be prepended/appended accordingly. The resulting string
+ # will be stripped in any case. If the +phrase+ isn't found, nil is returned.
+ #
+ # ==== Examples
+ # excerpt('This is an example', 'an', :radius => 5)
+ # # => ...s is an exam...
+ #
+ # excerpt('This is an example', 'is', :radius => 5)
+ # # => This is a...
+ #
+ # excerpt('This is an example', 'is')
+ # # => This is an example
+ #
+ # excerpt('This next thing is an example', 'ex', :radius => 2)
+ # # => ...next...
+ #
+ # excerpt('This is also an example', 'an', :radius => 8, :omission => '<chop> ')
+ # # => <chop> is also an example
+ #
+ # You can still use <tt>excerpt</tt> with the old API that accepts the
+ # +radius+ as its optional third and the +ellipsis+ as its
+ # optional forth parameter:
+ # excerpt('This is an example', 'an', 5) # => ...s is an exam...
+ # excerpt('This is also an example', 'an', 8, '<chop> ') # => <chop> is also an example
+ def excerpt(text, phrase, *args)
+ options = args.extract_options!
+ unless args.empty?
+ options[:radius] = args[0] || 100
+ options[:omission] = args[1] || "..."
end
- else
- def excerpt(text, phrase, *args) #:nodoc:
- options = args.extract_options!
- unless args.empty?
- options[:radius] = args[0] || 100
- options[:omission] = args[1] || "..."
- end
- options.reverse_merge!(:radius => 100, :omission => "...")
+ options.reverse_merge!(:radius => 100, :omission => "...")
- if text && phrase
- phrase = Regexp.escape(phrase)
+ if text && phrase
+ phrase = Regexp.escape(phrase)
- if found_pos = text =~ /(#{phrase})/i
- start_pos = [ found_pos - options[:radius], 0 ].max
- end_pos = [ [ found_pos + phrase.length + options[:radius] - 1, 0].max, text.length ].min
+ if found_pos = text.mb_chars =~ /(#{phrase})/i
+ start_pos = [ found_pos - options[:radius], 0 ].max
+ end_pos = [ [ found_pos + phrase.mb_chars.length + options[:radius] - 1, 0].max, text.mb_chars.length ].min
- prefix = start_pos > 0 ? options[:omission] : ""
- postfix = end_pos < text.length - 1 ? options[:omission] : ""
+ prefix = start_pos > 0 ? options[:omission] : ""
+ postfix = end_pos < text.mb_chars.length - 1 ? options[:omission] : ""
- prefix + text[start_pos..end_pos].strip + postfix
- else
- nil
- end
+ prefix + text.mb_chars[start_pos..end_pos].strip + postfix
+ else
+ nil
end
end
end
diff --git a/activerecord/lib/active_record/validations.rb b/activerecord/lib/active_record/validations.rb
index 518b59e433..684fb1de88 100644
--- a/activerecord/lib/active_record/validations.rb
+++ b/activerecord/lib/active_record/validations.rb
@@ -678,7 +678,7 @@ module ActiveRecord
condition_params = [value]
else
condition_sql = "LOWER(#{sql_attribute}) #{comparison_operator}"
- condition_params = [value.chars.downcase]
+ condition_params = [value.mb_chars.downcase]
end
if scope = configuration[:scope]
diff --git a/activerecord/test/cases/base_test.rb b/activerecord/test/cases/base_test.rb
index d3f00be894..d512834237 100644..100755
--- a/activerecord/test/cases/base_test.rb
+++ b/activerecord/test/cases/base_test.rb
@@ -1464,15 +1464,17 @@ class BasicsTest < ActiveRecord::TestCase
if RUBY_VERSION < '1.9'
def test_quote_chars
- str = 'The Narrator'
- topic = Topic.create(:author_name => str)
- assert_equal str, topic.author_name
+ with_kcode('UTF8') do
+ str = 'The Narrator'
+ topic = Topic.create(:author_name => str)
+ assert_equal str, topic.author_name
- assert_kind_of ActiveSupport::Multibyte::Chars, str.chars
- topic = Topic.find_by_author_name(str.chars)
+ assert_kind_of ActiveSupport::Multibyte.proxy_class, str.mb_chars
+ topic = Topic.find_by_author_name(str.mb_chars)
- assert_kind_of Topic, topic
- assert_equal str, topic.author_name, "The right topic should have been found by name even with name passed as Chars"
+ assert_kind_of Topic, topic
+ assert_equal str, topic.author_name, "The right topic should have been found by name even with name passed as Chars"
+ end
end
end
@@ -2065,4 +2067,18 @@ class BasicsTest < ActiveRecord::TestCase
ensure
ActiveRecord::Base.logger = original_logger
end
+
+ private
+ def with_kcode(kcode)
+ if RUBY_VERSION < '1.9'
+ orig_kcode, $KCODE = $KCODE, kcode
+ begin
+ yield
+ ensure
+ $KCODE = orig_kcode
+ end
+ else
+ yield
+ end
+ end
end
diff --git a/activerecord/test/cases/finder_test.rb b/activerecord/test/cases/finder_test.rb
index cbdff382fe..10f6aa0f2a 100644
--- a/activerecord/test/cases/finder_test.rb
+++ b/activerecord/test/cases/finder_test.rb
@@ -465,8 +465,8 @@ class FinderTest < ActiveRecord::TestCase
quoted_bambi_and_thumper = ActiveRecord::Base.connection.quote("Bambi\nand\nThumper")
assert_equal "name=#{quoted_bambi}", bind('name=?', "Bambi")
assert_equal "name=#{quoted_bambi_and_thumper}", bind('name=?', "Bambi\nand\nThumper")
- assert_equal "name=#{quoted_bambi}", bind('name=?', "Bambi".chars)
- assert_equal "name=#{quoted_bambi_and_thumper}", bind('name=?', "Bambi\nand\nThumper".chars)
+ assert_equal "name=#{quoted_bambi}", bind('name=?', "Bambi".mb_chars)
+ assert_equal "name=#{quoted_bambi_and_thumper}", bind('name=?', "Bambi\nand\nThumper".mb_chars)
end
def test_bind_record
diff --git a/activerecord/test/cases/sanitize_test.rb b/activerecord/test/cases/sanitize_test.rb
index 0106572ced..817897ceac 100644
--- a/activerecord/test/cases/sanitize_test.rb
+++ b/activerecord/test/cases/sanitize_test.rb
@@ -8,18 +8,18 @@ class SanitizeTest < ActiveRecord::TestCase
def test_sanitize_sql_array_handles_string_interpolation
quoted_bambi = ActiveRecord::Base.connection.quote_string("Bambi")
assert_equal "name=#{quoted_bambi}", Binary.send(:sanitize_sql_array, ["name=%s", "Bambi"])
- assert_equal "name=#{quoted_bambi}", Binary.send(:sanitize_sql_array, ["name=%s", "Bambi".chars])
+ assert_equal "name=#{quoted_bambi}", Binary.send(:sanitize_sql_array, ["name=%s", "Bambi".mb_chars])
quoted_bambi_and_thumper = ActiveRecord::Base.connection.quote_string("Bambi\nand\nThumper")
assert_equal "name=#{quoted_bambi_and_thumper}",Binary.send(:sanitize_sql_array, ["name=%s", "Bambi\nand\nThumper"])
- assert_equal "name=#{quoted_bambi_and_thumper}",Binary.send(:sanitize_sql_array, ["name=%s", "Bambi\nand\nThumper".chars])
+ assert_equal "name=#{quoted_bambi_and_thumper}",Binary.send(:sanitize_sql_array, ["name=%s", "Bambi\nand\nThumper".mb_chars])
end
def test_sanitize_sql_array_handles_bind_variables
quoted_bambi = ActiveRecord::Base.connection.quote("Bambi")
assert_equal "name=#{quoted_bambi}", Binary.send(:sanitize_sql_array, ["name=?", "Bambi"])
- assert_equal "name=#{quoted_bambi}", Binary.send(:sanitize_sql_array, ["name=?", "Bambi".chars])
+ assert_equal "name=#{quoted_bambi}", Binary.send(:sanitize_sql_array, ["name=?", "Bambi".mb_chars])
quoted_bambi_and_thumper = ActiveRecord::Base.connection.quote("Bambi\nand\nThumper")
assert_equal "name=#{quoted_bambi_and_thumper}", Binary.send(:sanitize_sql_array, ["name=?", "Bambi\nand\nThumper"])
- assert_equal "name=#{quoted_bambi_and_thumper}", Binary.send(:sanitize_sql_array, ["name=?", "Bambi\nand\nThumper".chars])
+ assert_equal "name=#{quoted_bambi_and_thumper}", Binary.send(:sanitize_sql_array, ["name=?", "Bambi\nand\nThumper".mb_chars])
end
end
diff --git a/activesupport/bin/generate_tables b/activesupport/bin/generate_tables
new file mode 100755
index 0000000000..f8e032139f
--- /dev/null
+++ b/activesupport/bin/generate_tables
@@ -0,0 +1,147 @@
+#!/usr/bin/env ruby
+
+begin
+ $:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+ require 'active_support'
+rescue IOError
+end
+
+require 'open-uri'
+require 'tmpdir'
+
+module ActiveSupport
+ module Multibyte
+ class UnicodeDatabase
+ def load; end
+ end
+
+ class UnicodeDatabaseGenerator
+ BASE_URI = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::UNICODE_VERSION}/ucd/"
+ SOURCES = {
+ :codepoints => BASE_URI + 'UnicodeData.txt',
+ :composition_exclusion => BASE_URI + 'CompositionExclusions.txt',
+ :grapheme_break_property => BASE_URI + 'auxiliary/GraphemeBreakProperty.txt',
+ :cp1252 => 'http://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT'
+ }
+
+ def initialize
+ @ucd = UnicodeDatabase.new
+
+ default = Codepoint.new
+ default.combining_class = 0
+ default.uppercase_mapping = 0
+ default.lowercase_mapping = 0
+ @ucd.codepoints = Hash.new(default)
+ end
+
+ def parse_codepoints(line)
+ codepoint = Codepoint.new
+ raise "Could not parse input." unless line =~ /^
+ ([0-9A-F]+); # code
+ ([^;]+); # name
+ ([A-Z]+); # general category
+ ([0-9]+); # canonical combining class
+ ([A-Z]+); # bidi class
+ (<([A-Z]*)>)? # decomposition type
+ ((\ ?[0-9A-F]+)*); # decompomposition mapping
+ ([0-9]*); # decimal digit
+ ([0-9]*); # digit
+ ([^;]*); # numeric
+ ([YN]*); # bidi mirrored
+ ([^;]*); # unicode 1.0 name
+ ([^;]*); # iso comment
+ ([0-9A-F]*); # simple uppercase mapping
+ ([0-9A-F]*); # simple lowercase mapping
+ ([0-9A-F]*)$/ix # simple titlecase mapping
+ codepoint.code = $1.hex
+ #codepoint.name = $2
+ #codepoint.category = $3
+ codepoint.combining_class = Integer($4)
+ #codepoint.bidi_class = $5
+ codepoint.decomp_type = $7
+ codepoint.decomp_mapping = ($8=='') ? nil : $8.split.collect { |element| element.hex }
+ #codepoint.bidi_mirrored = ($13=='Y') ? true : false
+ codepoint.uppercase_mapping = ($16=='') ? 0 : $16.hex
+ codepoint.lowercase_mapping = ($17=='') ? 0 : $17.hex
+ #codepoint.titlecase_mapping = ($18=='') ? nil : $18.hex
+ @ucd.codepoints[codepoint.code] = codepoint
+ end
+
+ def parse_grapheme_break_property(line)
+ if line =~ /^([0-9A-F\.]+)\s*;\s*([\w]+)\s*#/
+ type = $2.downcase.intern
+ @ucd.boundary[type] ||= []
+ if $1.include? '..'
+ parts = $1.split '..'
+ @ucd.boundary[type] << (parts[0].hex..parts[1].hex)
+ else
+ @ucd.boundary[type] << $1.hex
+ end
+ end
+ end
+
+ def parse_composition_exclusion(line)
+ if line =~ /^([0-9A-F]+)/i
+ @ucd.composition_exclusion << $1.hex
+ end
+ end
+
+ def parse_cp1252(line)
+ if line =~ /^([0-9A-Fx]+)\s([0-9A-Fx]+)/i
+ @ucd.cp1252[$1.hex] = $2.hex
+ end
+ end
+
+ def create_composition_map
+ @ucd.codepoints.each do |_, cp|
+ if !cp.nil? and cp.combining_class == 0 and cp.decomp_type.nil? and !cp.decomp_mapping.nil? and cp.decomp_mapping.length == 2 and @ucd.codepoints[cp.decomp_mapping[0]].combining_class == 0 and !@ucd.composition_exclusion.include?(cp.code)
+ @ucd.composition_map[cp.decomp_mapping[0]] ||= {}
+ @ucd.composition_map[cp.decomp_mapping[0]][cp.decomp_mapping[1]] = cp.code
+ end
+ end
+ end
+
+ def normalize_boundary_map
+ @ucd.boundary.each do |k,v|
+ if [:lf, :cr].include? k
+ @ucd.boundary[k] = v[0]
+ end
+ end
+ end
+
+ def parse
+ SOURCES.each do |type, url|
+ filename = File.join(Dir.tmpdir, "#{url.split('/').last}")
+ unless File.exist?(filename)
+ $stderr.puts "Downloading #{url.split('/').last}"
+ File.open(filename, 'wb') do |target|
+ open(url) do |source|
+ source.each_line { |line| target.write line }
+ end
+ end
+ end
+ File.open(filename) do |file|
+ file.each_line { |line| send "parse_#{type}".intern, line }
+ end
+ end
+ create_composition_map
+ normalize_boundary_map
+ end
+
+ def dump_to(filename)
+ File.open(filename, 'wb') do |f|
+ f.write Marshal.dump([@ucd.codepoints, @ucd.composition_exclusion, @ucd.composition_map, @ucd.boundary, @ucd.cp1252])
+ end
+ end
+ end
+ end
+end
+
+if __FILE__ == $0
+ filename = ActiveSupport::Multibyte::UnicodeDatabase.filename
+ generator = ActiveSupport::Multibyte::UnicodeDatabaseGenerator.new
+ generator.parse
+ print "Writing to: #{filename}"
+ generator.dump_to filename
+ puts " (#{File.size(filename)} bytes)"
+end
diff --git a/activesupport/lib/active_support/core_ext/string.rb b/activesupport/lib/active_support/core_ext/string.rb
index 7ff2f11eff..16c544a577 100644
--- a/activesupport/lib/active_support/core_ext/string.rb
+++ b/activesupport/lib/active_support/core_ext/string.rb
@@ -1,9 +1,11 @@
+# encoding: utf-8
+
require 'active_support/core_ext/string/inflections'
require 'active_support/core_ext/string/conversions'
require 'active_support/core_ext/string/access'
require 'active_support/core_ext/string/starts_ends_with'
require 'active_support/core_ext/string/iterators'
-require 'active_support/core_ext/string/unicode'
+require 'active_support/core_ext/string/multibyte'
require 'active_support/core_ext/string/xchar'
require 'active_support/core_ext/string/filters'
require 'active_support/core_ext/string/behavior'
@@ -15,6 +17,6 @@ class String #:nodoc:
include ActiveSupport::CoreExtensions::String::Inflections
include ActiveSupport::CoreExtensions::String::StartsEndsWith
include ActiveSupport::CoreExtensions::String::Iterators
- include ActiveSupport::CoreExtensions::String::Unicode
include ActiveSupport::CoreExtensions::String::Behavior
+ include ActiveSupport::CoreExtensions::String::Multibyte
end
diff --git a/activesupport/lib/active_support/core_ext/string/access.rb b/activesupport/lib/active_support/core_ext/string/access.rb
index 1a949c0b77..7fb21fa4dd 100644
--- a/activesupport/lib/active_support/core_ext/string/access.rb
+++ b/activesupport/lib/active_support/core_ext/string/access.rb
@@ -11,7 +11,7 @@ module ActiveSupport #:nodoc:
# "hello".at(4) # => "o"
# "hello".at(10) # => nil
def at(position)
- chars[position, 1].to_s
+ mb_chars[position, 1].to_s
end
# Returns the remaining of the string from the +position+ treating the string as an array (where 0 is the first character).
@@ -21,7 +21,7 @@ module ActiveSupport #:nodoc:
# "hello".from(2) # => "llo"
# "hello".from(10) # => nil
def from(position)
- chars[position..-1].to_s
+ mb_chars[position..-1].to_s
end
# Returns the beginning of the string up to the +position+ treating the string as an array (where 0 is the first character).
@@ -31,7 +31,7 @@ module ActiveSupport #:nodoc:
# "hello".to(2) # => "hel"
# "hello".to(10) # => "hello"
def to(position)
- chars[0..position].to_s
+ mb_chars[0..position].to_s
end
# Returns the first character of the string or the first +limit+ characters.
@@ -41,7 +41,7 @@ module ActiveSupport #:nodoc:
# "hello".first(2) # => "he"
# "hello".first(10) # => "hello"
def first(limit = 1)
- chars[0..(limit - 1)].to_s
+ mb_chars[0..(limit - 1)].to_s
end
# Returns the last character of the string or the last +limit+ characters.
@@ -51,7 +51,7 @@ module ActiveSupport #:nodoc:
# "hello".last(2) # => "lo"
# "hello".last(10) # => "hello"
def last(limit = 1)
- (chars[(-limit)..-1] || self).to_s
+ (mb_chars[(-limit)..-1] || self).to_s
end
end
else
diff --git a/activesupport/lib/active_support/core_ext/string/multibyte.rb b/activesupport/lib/active_support/core_ext/string/multibyte.rb
new file mode 100644
index 0000000000..a4caa83b74
--- /dev/null
+++ b/activesupport/lib/active_support/core_ext/string/multibyte.rb
@@ -0,0 +1,81 @@
+# encoding: utf-8
+
+module ActiveSupport #:nodoc:
+ module CoreExtensions #:nodoc:
+ module String #:nodoc:
+ # Implements multibyte methods for easier access to multibyte characters in a String instance.
+ module Multibyte
+ unless '1.9'.respond_to?(:force_encoding)
+ # == Multibyte proxy
+ #
+ # +mb_chars+ is a multibyte safe proxy for string methods.
+ #
+ # In Ruby 1.8 and older it creates and returns an instance of the ActiveSupport::Multibyte::Chars class which
+ # encapsulates the original string. A Unicode safe version of all the String methods are defined on this proxy
+ # class. If the proxy class doesn't respond to a certain method, it's forwarded to the encapsuled string.
+ #
+ # name = 'Claus Müller'
+ # name.reverse #=> "rell??M sualC"
+ # name.length #=> 13
+ #
+ # name.mb_chars.reverse.to_s #=> "rellüM sualC"
+ # name.mb_chars.length #=> 12
+ #
+ # In Ruby 1.9 and newer +mb_chars+ returns +self+ because String is (mostly) encoding aware. This means that
+ # it becomes easy to run one version of your code on multiple Ruby versions.
+ #
+ # == Method chaining
+ #
+ # All the methods on the Chars proxy which normally return a string will return a Chars object. This allows
+ # method chaining on the result of any of these methods.
+ #
+ # name.mb_chars.reverse.length #=> 12
+ #
+ # == Interoperability and configuration
+ #
+ # The Chars object tries to be as interchangeable with String objects as possible: sorting and comparing between
+ # String and Char work like expected. The bang! methods change the internal string representation in the Chars
+ # object. Interoperability problems can be resolved easily with a +to_s+ call.
+ #
+ # For more information about the methods defined on the Chars proxy see ActiveSupport::Multibyte::Chars. For
+ # information about how to change the default Multibyte behaviour see ActiveSupport::Multibyte.
+ def mb_chars
+ if ActiveSupport::Multibyte.proxy_class.wants?(self)
+ ActiveSupport::Multibyte.proxy_class.new(self)
+ else
+ self
+ end
+ end
+
+ # Returns true if the string has UTF-8 semantics (a String used for purely byte resources is unlikely to have
+ # them), returns false otherwise.
+ def is_utf8?
+ ActiveSupport::Multibyte::Chars.consumes?(self)
+ end
+
+ unless '1.8.7 and later'.respond_to?(:chars)
+ def chars
+ ActiveSupport::Deprecation.warn('String#chars has been deprecated in favor of String#mb_chars.', caller)
+ mb_chars
+ end
+ end
+ else
+ def mb_chars #:nodoc
+ self
+ end
+
+ def is_utf8? #:nodoc
+ case encoding
+ when Encoding::UTF_8
+ valid_encoding?
+ when Encoding::ASCII_8BIT, Encoding::US_ASCII
+ dup.force_encoding(Encoding::UTF_8).valid_encoding?
+ else
+ false
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/activesupport/lib/active_support/core_ext/string/unicode.rb b/activesupport/lib/active_support/core_ext/string/unicode.rb
deleted file mode 100644
index 666f7bcb65..0000000000
--- a/activesupport/lib/active_support/core_ext/string/unicode.rb
+++ /dev/null
@@ -1,66 +0,0 @@
-module ActiveSupport #:nodoc:
- module CoreExtensions #:nodoc:
- module String #:nodoc:
- # Define methods for handling unicode data.
- module Unicode
- def self.append_features(base)
- if '1.8.7 and later'.respond_to?(:chars)
- base.class_eval { remove_method :chars }
- end
- super
- end
-
- unless '1.9'.respond_to?(:force_encoding)
- # +chars+ is a Unicode safe proxy for string methods. It creates and returns an instance of the
- # ActiveSupport::Multibyte::Chars class which encapsulates the original string. A Unicode safe version of all
- # the String methods are defined on this proxy class. Undefined methods are forwarded to String, so all of the
- # string overrides can also be called through the +chars+ proxy.
- #
- # name = 'Claus Müller'
- # name.reverse # => "rell??M sualC"
- # name.length # => 13
- #
- # name.chars.reverse.to_s # => "rellüM sualC"
- # name.chars.length # => 12
- #
- #
- # All the methods on the chars proxy which normally return a string will return a Chars object. This allows
- # method chaining on the result of any of these methods.
- #
- # name.chars.reverse.length # => 12
- #
- # The Char object tries to be as interchangeable with String objects as possible: sorting and comparing between
- # String and Char work like expected. The bang! methods change the internal string representation in the Chars
- # object. Interoperability problems can be resolved easily with a +to_s+ call.
- #
- # For more information about the methods defined on the Chars proxy see ActiveSupport::Multibyte::Chars and
- # ActiveSupport::Multibyte::Handlers::UTF8Handler.
- def chars
- ActiveSupport::Multibyte::Chars.new(self)
- end
-
- # Returns true if the string has UTF-8 semantics (a String used for purely byte resources is unlikely to have
- # them), returns false otherwise.
- def is_utf8?
- ActiveSupport::Multibyte::Handlers::UTF8Handler.consumes?(self)
- end
- else
- def chars #:nodoc:
- self
- end
-
- def is_utf8? #:nodoc:
- case encoding
- when Encoding::UTF_8
- valid_encoding?
- when Encoding::ASCII_8BIT
- dup.force_encoding('UTF-8').valid_encoding?
- else
- false
- end
- end
- end
- end
- end
- end
-end
diff --git a/activesupport/lib/active_support/inflector.rb b/activesupport/lib/active_support/inflector.rb
index 8a917a9eb2..b2046f26de 100644
--- a/activesupport/lib/active_support/inflector.rb
+++ b/activesupport/lib/active_support/inflector.rb
@@ -257,7 +257,7 @@ module ActiveSupport
# <%= link_to(@person.name, person_path %>
# # => <a href="/person/1-donald-e-knuth">Donald E. Knuth</a>
def parameterize(string, sep = '-')
- string.chars.normalize(:kd).to_s.gsub(/[^\x00-\x7F]+/, '').gsub(/[^a-z0-9_\-]+/i, sep).downcase
+ string.mb_chars.normalize(:kd).to_s.gsub(/[^\x00-\x7F]+/, '').gsub(/[^a-z0-9_\-]+/i, sep).downcase
end
# Create the name of a table like Rails does for models to table names. This method
diff --git a/activesupport/lib/active_support/multibyte.rb b/activesupport/lib/active_support/multibyte.rb
index 27c0d180a5..018aafe607 100644
--- a/activesupport/lib/active_support/multibyte.rb
+++ b/activesupport/lib/active_support/multibyte.rb
@@ -1,9 +1,33 @@
-module ActiveSupport
- module Multibyte #:nodoc:
- DEFAULT_NORMALIZATION_FORM = :kc
+# encoding: utf-8
+
+require 'active_support/multibyte/chars'
+require 'active_support/multibyte/exceptions'
+require 'active_support/multibyte/unicode_database'
+
+module ActiveSupport #:nodoc:
+ module Multibyte
+ # A list of all available normalization forms. See http://www.unicode.org/reports/tr15/tr15-29.html for more
+ # information about normalization.
NORMALIZATIONS_FORMS = [:c, :kc, :d, :kd]
- UNICODE_VERSION = '5.0.0'
+
+ # The Unicode version that is supported by the implementation
+ UNICODE_VERSION = '5.1.0'
+
+ # The default normalization used for operations that require normalization. It can be set to any of the
+ # normalizations in NORMALIZATIONS_FORMS.
+ #
+ # Example:
+ # ActiveSupport::Multibyte.default_normalization_form = :c
+ mattr_accessor :default_normalization_form
+ self.default_normalization_form = :kc
+
+ # The proxy class returned when calling mb_chars. You can use this accessor to configure your own proxy
+ # class so you can support other encodings. See the ActiveSupport::Multibyte::Chars implementation for
+ # an example how to do this.
+ #
+ # Example:
+ # ActiveSupport::Multibyte.proxy_class = CharsForUTF32
+ mattr_accessor :proxy_class
+ self.proxy_class = ActiveSupport::Multibyte::Chars
end
end
-
-require 'active_support/multibyte/chars'
diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb
index de2c83f8d1..5184026c63 100644
--- a/activesupport/lib/active_support/multibyte/chars.rb
+++ b/activesupport/lib/active_support/multibyte/chars.rb
@@ -1,142 +1,679 @@
-require 'active_support/multibyte/handlers/utf8_handler'
-require 'active_support/multibyte/handlers/passthru_handler'
-
-# Encapsulates all the functionality related to the Chars proxy.
-module ActiveSupport::Multibyte #:nodoc:
- # Chars enables you to work transparently with multibyte encodings in the Ruby String class without having extensive
- # knowledge about the encoding. A Chars object accepts a string upon initialization and proxies String methods in an
- # encoding safe manner. All the normal String methods are also implemented on the proxy.
- #
- # String methods are proxied through the Chars object, and can be accessed through the +chars+ method. Methods
- # which would normally return a String object now return a Chars object so methods can be chained.
- #
- # "The Perfect String ".chars.downcase.strip.normalize # => "the perfect string"
- #
- # Chars objects are perfectly interchangeable with String objects as long as no explicit class checks are made.
- # If certain methods do explicitly check the class, call +to_s+ before you pass chars objects to them.
- #
- # bad.explicit_checking_method "T".chars.downcase.to_s
- #
- # The actual operations on the string are delegated to handlers. Theoretically handlers can be implemented for
- # any encoding, but the default handler handles UTF-8. This handler is set during initialization, if you want to
- # use you own handler, you can set it on the Chars class. Look at the UTF8Handler source for an example how to
- # implement your own handler. If you your own handler to work on anything but UTF-8 you probably also
- # want to override Chars#handler.
- #
- # ActiveSupport::Multibyte::Chars.handler = MyHandler
- #
- # Note that a few methods are defined on Chars instead of the handler because they are defined on Object or Kernel
- # and method_missing can't catch them.
- class Chars
-
- attr_reader :string # The contained string
- alias_method :to_s, :string
-
- include Comparable
-
- # The magic method to make String and Chars comparable
- def to_str
- # Using any other ways of overriding the String itself will lead you all the way from infinite loops to
- # core dumps. Don't go there.
- @string
- end
+# encoding: utf-8
+
+module ActiveSupport #:nodoc:
+ module Multibyte #:nodoc:
+ # Chars enables you to work transparently with UTF-8 encoding in the Ruby String class without having extensive
+ # knowledge about the encoding. A Chars object accepts a string upon initialization and proxies String methods in an
+ # encoding safe manner. All the normal String methods are also implemented on the proxy.
+ #
+ # String methods are proxied through the Chars object, and can be accessed through the +mb_chars+ method. Methods
+ # which would normally return a String object now return a Chars object so methods can be chained.
+ #
+ # "The Perfect String ".mb_chars.downcase.strip.normalize #=> "the perfect string"
+ #
+ # Chars objects are perfectly interchangeable with String objects as long as no explicit class checks are made.
+ # If certain methods do explicitly check the class, call +to_s+ before you pass chars objects to them.
+ #
+ # bad.explicit_checking_method "T".mb_chars.downcase.to_s
+ #
+ # The default Chars implementation assumes that the encoding of the string is UTF-8, if you want to handle different
+ # encodings you can write your own multibyte string handler and configure it through
+ # ActiveSupport::Multibyte.proxy_class.
+ #
+ # class CharsForUTF32
+ # def size
+ # @wrapped_string.size / 4
+ # end
+ #
+ # def self.accepts?(string)
+ # string.length % 4 == 0
+ # end
+ # end
+ #
+ # ActiveSupport::Multibyte.proxy_class = CharsForUTF32
+ class Chars
+ # Hangul character boundaries and properties
+ HANGUL_SBASE = 0xAC00
+ HANGUL_LBASE = 0x1100
+ HANGUL_VBASE = 0x1161
+ HANGUL_TBASE = 0x11A7
+ HANGUL_LCOUNT = 19
+ HANGUL_VCOUNT = 21
+ HANGUL_TCOUNT = 28
+ HANGUL_NCOUNT = HANGUL_VCOUNT * HANGUL_TCOUNT
+ HANGUL_SCOUNT = 11172
+ HANGUL_SLAST = HANGUL_SBASE + HANGUL_SCOUNT
+ HANGUL_JAMO_FIRST = 0x1100
+ HANGUL_JAMO_LAST = 0x11FF
+
+ # All the unicode whitespace
+ UNICODE_WHITESPACE = [
+ (0x0009..0x000D).to_a, # White_Space # Cc [5] <control-0009>..<control-000D>
+ 0x0020, # White_Space # Zs SPACE
+ 0x0085, # White_Space # Cc <control-0085>
+ 0x00A0, # White_Space # Zs NO-BREAK SPACE
+ 0x1680, # White_Space # Zs OGHAM SPACE MARK
+ 0x180E, # White_Space # Zs MONGOLIAN VOWEL SEPARATOR
+ (0x2000..0x200A).to_a, # White_Space # Zs [11] EN QUAD..HAIR SPACE
+ 0x2028, # White_Space # Zl LINE SEPARATOR
+ 0x2029, # White_Space # Zp PARAGRAPH SEPARATOR
+ 0x202F, # White_Space # Zs NARROW NO-BREAK SPACE
+ 0x205F, # White_Space # Zs MEDIUM MATHEMATICAL SPACE
+ 0x3000, # White_Space # Zs IDEOGRAPHIC SPACE
+ ].flatten.freeze
+
+ # BOM (byte order mark) can also be seen as whitespace, it's a non-rendering character used to distinguish
+ # between little and big endian. This is not an issue in utf-8, so it must be ignored.
+ UNICODE_LEADERS_AND_TRAILERS = UNICODE_WHITESPACE + [65279] # ZERO-WIDTH NO-BREAK SPACE aka BOM
+
+ # Returns a regular expression pattern that matches the passed Unicode codepoints
+ def self.codepoints_to_pattern(array_of_codepoints) #:nodoc:
+ array_of_codepoints.collect{ |e| [e].pack 'U*' }.join('|')
+ end
+ UNICODE_TRAILERS_PAT = /(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+\Z/
+ UNICODE_LEADERS_PAT = /\A(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+/
+
+ # Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site)
+ UTF8_PAT = /\A(?:
+ [\x00-\x7f] |
+ [\xc2-\xdf] [\x80-\xbf] |
+ \xe0 [\xa0-\xbf] [\x80-\xbf] |
+ [\xe1-\xef] [\x80-\xbf] [\x80-\xbf] |
+ \xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] |
+ [\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] |
+ \xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf]
+ )*\z/xn
+
+ attr_reader :wrapped_string
+ alias to_s wrapped_string
+ alias to_str wrapped_string
+
+ if '1.9'.respond_to?(:force_encoding)
+ # Creates a new Chars instance by wrapping _string_.
+ def initialize(string)
+ @wrapped_string = string
+ @wrapped_string.force_encoding(Encoding::UTF_8) unless @wrapped_string.frozen?
+ end
+ else
+ def initialize(string) #:nodoc:
+ @wrapped_string = string
+ end
+ end
+
+ # Forward all undefined methods to the wrapped string.
+ def method_missing(method, *args, &block)
+ if method.to_s =~ /!$/
+ @wrapped_string.__send__(method, *args, &block)
+ self
+ else
+ result = @wrapped_string.__send__(method, *args, &block)
+ result.kind_of?(String) ? chars(result) : result
+ end
+ end
+
+ # Returns +true+ if _obj_ responds to the given method. Private methods are included in the search
+ # only if the optional second parameter evaluates to +true+.
+ def respond_to?(method, include_private=false)
+ super || @wrapped_string.respond_to?(method, include_private) || false
+ end
+
+ # Enable more predictable duck-typing on String-like classes. See Object#acts_like?.
+ def acts_like_string?
+ true
+ end
+
+ # Returns +true+ if the Chars class can and should act as a proxy for the string _string_. Returns
+ # +false+ otherwise.
+ def self.wants?(string)
+ $KCODE == 'UTF8' && consumes?(string)
+ end
- # Make duck-typing with String possible
- def respond_to?(method, include_priv = false)
- super || @string.respond_to?(method, include_priv) ||
- handler.respond_to?(method, include_priv) ||
- (method.to_s =~ /(.*)!/ && handler.respond_to?($1, include_priv)) ||
+ # Returns +true+ when the proxy class can handle the string. Returns +false+ otherwise.
+ def self.consumes?(string)
+ # Unpack is a little bit faster than regular expressions.
+ string.unpack('U*')
+ true
+ rescue ArgumentError
false
- end
+ end
- # Enable more predictable duck-typing on String-like classes. See Object#acts_like?.
- def acts_like_string?
- true
- end
+ include Comparable
- # Create a new Chars instance.
- def initialize(str)
- @string = str.respond_to?(:string) ? str.string : str
- end
-
- # Returns -1, 0 or +1 depending on whether the Chars object is to be sorted before, equal or after the
- # object on the right side of the operation. It accepts any object that implements +to_s+. See String.<=>
- # for more details.
- def <=>(other); @string <=> other.to_s; end
-
- # Works just like String#split, with the exception that the items in the resulting list are Chars
- # instances instead of String. This makes chaining methods easier.
- def split(*args)
- @string.split(*args).map { |i| i.chars }
- end
-
- # Gsub works exactly the same as gsub on a normal string.
- def gsub(*a, &b); @string.gsub(*a, &b).chars; end
-
- # Like String.=~ only it returns the character offset (in codepoints) instead of the byte offset.
- def =~(other)
- handler.translate_offset(@string, @string =~ other)
- end
-
- # Try to forward all undefined methods to the handler, when a method is not defined on the handler, send it to
- # the contained string. Method_missing is also responsible for making the bang! methods destructive.
- def method_missing(m, *a, &b)
- begin
- # Simulate methods with a ! at the end because we can't touch the enclosed string from the handlers.
- if m.to_s =~ /^(.*)\!$/ && handler.respond_to?($1)
- result = handler.send($1, @string, *a, &b)
- if result == @string
- result = nil
+ # Returns <tt>-1</tt>, <tt>0</tt> or <tt>+1</tt> depending on whether the Chars object is to be sorted before,
+ # equal or after the object on the right side of the operation. It accepts any object that implements +to_s+.
+ # See <tt>String#<=></tt> for more details.
+ #
+ # Example:
+ # 'é'.mb_chars <=> 'ü'.mb_chars #=> -1
+ def <=>(other)
+ @wrapped_string <=> other.to_s
+ end
+
+ # Returns a new Chars object containing the _other_ object concatenated to the string.
+ #
+ # Example:
+ # ('Café'.mb_chars + ' périferôl').to_s #=> "Café périferôl"
+ def +(other)
+ self << other
+ end
+
+ # Like <tt>String#=~</tt> only it returns the character offset (in codepoints) instead of the byte offset.
+ #
+ # Example:
+ # 'Café périferôl'.mb_chars =~ /ô/ #=> 12
+ def =~(other)
+ translate_offset(@wrapped_string =~ other)
+ end
+
+ # Works just like <tt>String#split</tt>, with the exception that the items in the resulting list are Chars
+ # instances instead of String. This makes chaining methods easier.
+ #
+ # Example:
+ # 'Café périferôl'.mb_chars.split(/é/).map { |part| part.upcase.to_s } #=> ["CAF", " P", "RIFERÔL"]
+ def split(*args)
+ @wrapped_string.split(*args).map { |i| i.mb_chars }
+ end
+
+ # Inserts the passed string at specified codepoint offsets.
+ #
+ # Example:
+ # 'Café'.mb_chars.insert(4, ' périferôl').to_s #=> "Café périferôl"
+ def insert(offset, fragment)
+ unpacked = self.class.u_unpack(@wrapped_string)
+ unless offset > unpacked.length
+ @wrapped_string.replace(
+ self.class.u_unpack(@wrapped_string).insert(offset, *self.class.u_unpack(fragment)).pack('U*')
+ )
+ else
+ raise IndexError, "index #{offset} out of string"
+ end
+ self
+ end
+
+ # Returns +true+ if contained string contains _other_. Returns +false+ otherwise.
+ #
+ # Example:
+ # 'Café'.mb_chars.include?('é') #=> true
+ def include?(other)
+ # We have to redefine this method because Enumerable defines it.
+ @wrapped_string.include?(other)
+ end
+
+ # Returns the position _needle_ in the string, counting in codepoints. Returns +nil+ if _needle_ isn't found.
+ #
+ # Example:
+ # 'Café périferôl'.mb_chars.index('ô') #=> 12
+ # 'Café périferôl'.mb_chars.index(/\w/u) #=> 0
+ def index(needle, offset=0)
+ index = @wrapped_string.index(needle, offset)
+ index ? (self.class.u_unpack(@wrapped_string.slice(0...index)).size) : nil
+ end
+
+ # Like <tt>String#[]=</tt>, except instead of byte offsets you specify character offsets.
+ #
+ # Example:
+ #
+ # s = "Müller"
+ # s.mb_chars[2] = "e" # Replace character with offset 2
+ # s
+ # #=> "Müeler"
+ #
+ # s = "Müller"
+ # s.mb_chars[1, 2] = "ö" # Replace 2 characters at character offset 1
+ # s
+ # #=> "Möler"
+ def []=(*args)
+ replace_by = args.pop
+ # Indexed replace with regular expressions already works
+ if args.first.is_a?(Regexp)
+ @wrapped_string[*args] = replace_by
+ else
+ result = self.class.u_unpack(@wrapped_string)
+ if args[0].is_a?(Fixnum)
+ raise IndexError, "index #{args[0]} out of string" if args[0] >= result.length
+ min = args[0]
+ max = args[1].nil? ? min : (min + args[1] - 1)
+ range = Range.new(min, max)
+ replace_by = [replace_by].pack('U') if replace_by.is_a?(Fixnum)
+ elsif args.first.is_a?(Range)
+ raise RangeError, "#{args[0]} out of range" if args[0].min >= result.length
+ range = args[0]
else
- @string.replace result
+ needle = args[0].to_s
+ min = index(needle)
+ max = min + self.class.u_unpack(needle).length - 1
+ range = Range.new(min, max)
end
- elsif handler.respond_to?(m)
- result = handler.send(m, @string, *a, &b)
- else
- result = @string.send(m, *a, &b)
+ result[range] = self.class.u_unpack(replace_by)
+ @wrapped_string.replace(result.pack('U*'))
end
- rescue Handlers::EncodingError
- @string.replace handler.tidy_bytes(@string)
- retry
end
-
- if result.kind_of?(String)
- result.chars
- else
- result
+
+ # Works just like <tt>String#rjust</tt>, only integer specifies characters instead of bytes.
+ #
+ # Example:
+ #
+ # "¾ cup".mb_chars.rjust(8).to_s
+ # #=> " ¾ cup"
+ #
+ # "¾ cup".mb_chars.rjust(8, " ").to_s # Use non-breaking whitespace
+ # #=> "   ¾ cup"
+ def rjust(integer, padstr=' ')
+ justify(integer, :right, padstr)
end
- end
-
- # Set the handler class for the Char objects.
- def self.handler=(klass)
- @@handler = klass
- end
- # Returns the proper handler for the contained string depending on $KCODE and the encoding of the string. This
- # method is used internally to always redirect messages to the proper classes depending on the context.
- def handler
- if utf8_pragma?
- @@handler
- else
- ActiveSupport::Multibyte::Handlers::PassthruHandler
+ # Works just like <tt>String#ljust</tt>, only integer specifies characters instead of bytes.
+ #
+ # Example:
+ #
+ # "¾ cup".mb_chars.rjust(8).to_s
+ # #=> "¾ cup "
+ #
+ # "¾ cup".mb_chars.rjust(8, " ").to_s # Use non-breaking whitespace
+ # #=> "¾ cup   "
+ def ljust(integer, padstr=' ')
+ justify(integer, :left, padstr)
+ end
+
+ # Works just like <tt>String#center</tt>, only integer specifies characters instead of bytes.
+ #
+ # Example:
+ #
+ # "¾ cup".mb_chars.center(8).to_s
+ # #=> " ¾ cup "
+ #
+ # "¾ cup".mb_chars.center(8, " ").to_s # Use non-breaking whitespace
+ # #=> " ¾ cup  "
+ def center(integer, padstr=' ')
+ justify(integer, :center, padstr)
end
- end
- private
+ # Strips entire range of Unicode whitespace from the right of the string.
+ def rstrip
+ chars(@wrapped_string.gsub(UNICODE_TRAILERS_PAT, ''))
+ end
+
+ # Strips entire range of Unicode whitespace from the left of the string.
+ def lstrip
+ chars(@wrapped_string.gsub(UNICODE_LEADERS_PAT, ''))
+ end
+
+ # Strips entire range of Unicode whitespace from the right and left of the string.
+ def strip
+ rstrip.lstrip
+ end
+
+ # Returns the number of codepoints in the string
+ def size
+ self.class.u_unpack(@wrapped_string).size
+ end
+ alias_method :length, :size
- # +utf8_pragma+ checks if it can send this string to the handlers. It makes sure @string isn't nil and $KCODE is
- # set to 'UTF8'.
- def utf8_pragma?
- !@string.nil? && ($KCODE == 'UTF8')
+ # Reverses all characters in the string.
+ #
+ # Example:
+ # 'Café'.mb_chars.reverse.to_s #=> 'éfaC'
+ def reverse
+ chars(self.class.u_unpack(@wrapped_string).reverse.pack('U*'))
end
+
+ # Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that
+ # character.
+ #
+ # Example:
+ # 'こにちわ'.mb_chars.slice(2..3).to_s #=> "ちわ"
+ def slice(*args)
+ if args.size > 2
+ raise ArgumentError, "wrong number of arguments (#{args.size} for 1)" # Do as if we were native
+ elsif (args.size == 2 && !(args.first.is_a?(Numeric) || args.first.is_a?(Regexp)))
+ raise TypeError, "cannot convert #{args.first.class} into Integer" # Do as if we were native
+ elsif (args.size == 2 && !args[1].is_a?(Numeric))
+ raise TypeError, "cannot convert #{args[1].class} into Integer" # Do as if we were native
+ elsif args[0].kind_of? Range
+ cps = self.class.u_unpack(@wrapped_string).slice(*args)
+ result = cps.nil? ? nil : cps.pack('U*')
+ elsif args[0].kind_of? Regexp
+ result = @wrapped_string.slice(*args)
+ elsif args.size == 1 && args[0].kind_of?(Numeric)
+ character = self.class.u_unpack(@wrapped_string)[args[0]]
+ result = character.nil? ? nil : [character].pack('U')
+ else
+ result = self.class.u_unpack(@wrapped_string).slice(*args).pack('U*')
+ end
+ result.nil? ? nil : chars(result)
+ end
+ alias_method :[], :slice
+
+ # Convert characters in the string to uppercase.
+ #
+ # Example:
+ # 'Laurent, òu sont les tests?'.mb_chars.upcase.to_s #=> "LAURENT, ÒU SONT LES TESTS?"
+ def upcase
+ apply_mapping :uppercase_mapping
+ end
+
+ # Convert characters in the string to lowercase.
+ #
+ # Example:
+ # 'VĚDA A VÝZKUM'.mb_chars.downcase.to_s #=> "věda a výzkum"
+ def downcase
+ apply_mapping :lowercase_mapping
+ end
+
+ # Converts the first character to uppercase and the remainder to lowercase.
+ #
+ # Example:
+ # 'über'.mb_chars.capitalize.to_s #=> "Über"
+ def capitalize
+ (slice(0) || chars('')).upcase + (slice(1..-1) || chars('')).downcase
+ end
+
+ # Returns the KC normalization of the string by default. NFKC is considered the best normalization form for
+ # passing strings to databases and validations.
+ #
+ # * <tt>str</tt> - The string to perform normalization on.
+ # * <tt>form</tt> - The form you want to normalize in. Should be one of the following:
+ # <tt>:c</tt>, <tt>:kc</tt>, <tt>:d</tt>, or <tt>:kd</tt>. Default is
+ # ActiveSupport::Multibyte.default_normalization_form
+ def normalize(form=ActiveSupport::Multibyte.default_normalization_form)
+ # See http://www.unicode.org/reports/tr15, Table 1
+ codepoints = self.class.u_unpack(@wrapped_string)
+ chars(case form
+ when :d
+ self.class.reorder_characters(self.class.decompose_codepoints(:canonical, codepoints))
+ when :c
+ self.class.compose_codepoints(self.class.reorder_characters(self.class.decompose_codepoints(:canonical, codepoints)))
+ when :kd
+ self.class.reorder_characters(self.class.decompose_codepoints(:compatability, codepoints))
+ when :kc
+ self.class.compose_codepoints(self.class.reorder_characters(self.class.decompose_codepoints(:compatability, codepoints)))
+ else
+ raise ArgumentError, "#{form} is not a valid normalization variant", caller
+ end.pack('U*'))
+ end
+
+ # Performs canonical decomposition on all the characters.
+ #
+ # Example:
+ # 'é'.length #=> 2
+ # 'é'.mb_chars.decompose.to_s.length #=> 3
+ def decompose
+ chars(self.class.decompose_codepoints(:canonical, self.class.u_unpack(@wrapped_string)).pack('U*'))
+ end
+
+ # Performs composition on all the characters.
+ #
+ # Example:
+ # 'é'.length #=> 3
+ # 'é'.mb_chars.compose.to_s.length #=> 2
+ def compose
+ chars(self.class.compose_codepoints(self.class.u_unpack(@wrapped_string)).pack('U*'))
+ end
+
+ # Returns the number of grapheme clusters in the string.
+ #
+ # Example:
+ # 'क्षि'.mb_chars.length #=> 4
+ # 'क्षि'.mb_chars.g_length #=> 3
+ def g_length
+ self.class.g_unpack(@wrapped_string).length
+ end
+
+ # Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string.
+ def tidy_bytes
+ chars(self.class.tidy_bytes(@wrapped_string))
+ end
+
+ %w(lstrip rstrip strip reverse upcase downcase slice tidy_bytes capitalize).each do |method|
+ define_method("#{method}!") do |*args|
+ unless args.nil?
+ @wrapped_string = send(method, *args).to_s
+ else
+ @wrapped_string = send(method).to_s
+ end
+ self
+ end
+ end
+
+ class << self
+
+ # Unpack the string at codepoints boundaries. Raises an EncodingError when the encoding of the string isn't
+ # valid UTF-8.
+ #
+ # Example:
+ # Chars.u_unpack('Café') #=> [67, 97, 102, 233]
+ def u_unpack(string)
+ begin
+ string.unpack 'U*'
+ rescue ArgumentError
+ raise EncodingError, 'malformed UTF-8 character'
+ end
+ end
+
+ # Detect whether the codepoint is in a certain character class. Returns +true+ when it's in the specified
+ # character class and +false+ otherwise. Valid character classes are: <tt>:cr</tt>, <tt>:lf</tt>, <tt>:l</tt>,
+ # <tt>:v</tt>, <tt>:lv</tt>, <tt>:lvt</tt> and <tt>:t</tt>.
+ #
+ # Primarily used by the grapheme cluster support.
+ def in_char_class?(codepoint, classes)
+ classes.detect { |c| UCD.boundary[c] === codepoint } ? true : false
+ end
+
+ # Unpack the string at grapheme boundaries. Returns a list of character lists.
+ #
+ # Example:
+ # Chars.g_unpack('क्षि') #=> [[2325, 2381], [2359], [2367]]
+ # Chars.g_unpack('Café') #=> [[67], [97], [102], [233]]
+ def g_unpack(string)
+ codepoints = u_unpack(string)
+ unpacked = []
+ pos = 0
+ marker = 0
+ eoc = codepoints.length
+ while(pos < eoc)
+ pos += 1
+ previous = codepoints[pos-1]
+ current = codepoints[pos]
+ if (
+ # CR X LF
+ one = ( previous == UCD.boundary[:cr] and current == UCD.boundary[:lf] ) or
+ # L X (L|V|LV|LVT)
+ two = ( UCD.boundary[:l] === previous and in_char_class?(current, [:l,:v,:lv,:lvt]) ) or
+ # (LV|V) X (V|T)
+ three = ( in_char_class?(previous, [:lv,:v]) and in_char_class?(current, [:v,:t]) ) or
+ # (LVT|T) X (T)
+ four = ( in_char_class?(previous, [:lvt,:t]) and UCD.boundary[:t] === current ) or
+ # X Extend
+ five = (UCD.boundary[:extend] === current)
+ )
+ else
+ unpacked << codepoints[marker..pos-1]
+ marker = pos
+ end
+ end
+ unpacked
+ end
+
+ # Reverse operation of g_unpack.
+ #
+ # Example:
+ # Chars.g_pack(Chars.g_unpack('क्षि')) #=> 'क्षि'
+ def g_pack(unpacked)
+ (unpacked.flatten).pack('U*')
+ end
+
+ def padding(padsize, padstr=' ') #:nodoc:
+ if padsize != 0
+ new(padstr * ((padsize / u_unpack(padstr).size) + 1)).slice(0, padsize)
+ else
+ ''
+ end
+ end
+
+ # Re-order codepoints so the string becomes canonical.
+ def reorder_characters(codepoints)
+ length = codepoints.length- 1
+ pos = 0
+ while pos < length do
+ cp1, cp2 = UCD.codepoints[codepoints[pos]], UCD.codepoints[codepoints[pos+1]]
+ if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
+ codepoints[pos..pos+1] = cp2.code, cp1.code
+ pos += (pos > 0 ? -1 : 1)
+ else
+ pos += 1
+ end
+ end
+ codepoints
+ end
+
+ # Decompose composed characters to the decomposed form.
+ def decompose_codepoints(type, codepoints)
+ codepoints.inject([]) do |decomposed, cp|
+ # if it's a hangul syllable starter character
+ if HANGUL_SBASE <= cp and cp < HANGUL_SLAST
+ sindex = cp - HANGUL_SBASE
+ ncp = [] # new codepoints
+ ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
+ ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
+ tindex = sindex % HANGUL_TCOUNT
+ ncp << (HANGUL_TBASE + tindex) unless tindex == 0
+ decomposed.concat ncp
+ # if the codepoint is decomposable in with the current decomposition type
+ elsif (ncp = UCD.codepoints[cp].decomp_mapping) and (!UCD.codepoints[cp].decomp_type || type == :compatability)
+ decomposed.concat decompose_codepoints(type, ncp.dup)
+ else
+ decomposed << cp
+ end
+ end
+ end
+
+ # Compose decomposed characters to the composed form.
+ def compose_codepoints(codepoints)
+ pos = 0
+ eoa = codepoints.length - 1
+ starter_pos = 0
+ starter_char = codepoints[0]
+ previous_combining_class = -1
+ while pos < eoa
+ pos += 1
+ lindex = starter_char - HANGUL_LBASE
+ # -- Hangul
+ if 0 <= lindex and lindex < HANGUL_LCOUNT
+ vindex = codepoints[starter_pos+1] - HANGUL_VBASE rescue vindex = -1
+ if 0 <= vindex and vindex < HANGUL_VCOUNT
+ tindex = codepoints[starter_pos+2] - HANGUL_TBASE rescue tindex = -1
+ if 0 <= tindex and tindex < HANGUL_TCOUNT
+ j = starter_pos + 2
+ eoa -= 2
+ else
+ tindex = 0
+ j = starter_pos + 1
+ eoa -= 1
+ end
+ codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
+ end
+ starter_pos += 1
+ starter_char = codepoints[starter_pos]
+ # -- Other characters
+ else
+ current_char = codepoints[pos]
+ current = UCD.codepoints[current_char]
+ if current.combining_class > previous_combining_class
+ if ref = UCD.composition_map[starter_char]
+ composition = ref[current_char]
+ else
+ composition = nil
+ end
+ unless composition.nil?
+ codepoints[starter_pos] = composition
+ starter_char = composition
+ codepoints.delete_at pos
+ eoa -= 1
+ pos -= 1
+ previous_combining_class = -1
+ else
+ previous_combining_class = current.combining_class
+ end
+ else
+ previous_combining_class = current.combining_class
+ end
+ if current.combining_class == 0
+ starter_pos = pos
+ starter_char = codepoints[pos]
+ end
+ end
+ end
+ codepoints
+ end
+
+ # Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string.
+ def tidy_bytes(string)
+ string.split(//u).map do |c|
+ if !UTF8_PAT.match(c)
+ n = c.unpack('C')[0]
+ n < 128 ? n.chr :
+ n < 160 ? [UCD.cp1252[n] || n].pack('U') :
+ n < 192 ? "\xC2" + n.chr : "\xC3" + (n-64).chr
+ else
+ c
+ end
+ end.join
+ end
+ end
+
+ protected
+
+ def translate_offset(byte_offset) #:nodoc:
+ return nil if byte_offset.nil?
+ return 0 if @wrapped_string == ''
+ chunk = @wrapped_string[0..byte_offset]
+ begin
+ begin
+ chunk.unpack('U*').length - 1
+ rescue ArgumentError => e
+ chunk = @wrapped_string[0..(byte_offset+=1)]
+ # Stop retrying at the end of the string
+ raise e unless byte_offset < chunk.length
+ # We damaged a character, retry
+ retry
+ end
+ # Catch the ArgumentError so we can throw our own
+ rescue ArgumentError
+ raise EncodingError, 'malformed UTF-8 character'
+ end
+ end
+
+ def justify(integer, way, padstr=' ') #:nodoc:
+ raise ArgumentError, "zero width padding" if padstr.length == 0
+ padsize = integer - size
+ padsize = padsize > 0 ? padsize : 0
+ case way
+ when :right
+ result = @wrapped_string.dup.insert(0, self.class.padding(padsize, padstr))
+ when :left
+ result = @wrapped_string.dup.insert(-1, self.class.padding(padsize, padstr))
+ when :center
+ lpad = self.class.padding((padsize / 2.0).floor, padstr)
+ rpad = self.class.padding((padsize / 2.0).ceil, padstr)
+ result = @wrapped_string.dup.insert(0, lpad).insert(-1, rpad)
+ end
+ chars(result)
+ end
+
+ def apply_mapping(mapping) #:nodoc:
+ chars(self.class.u_unpack(@wrapped_string).map do |codepoint|
+ cp = UCD.codepoints[codepoint]
+ if cp and (ncp = cp.send(mapping)) and ncp > 0
+ ncp
+ else
+ codepoint
+ end
+ end.pack('U*'))
+ end
+
+ def chars(string) #:nodoc:
+ self.class.new(string)
+ end
+ end
end
-end
-
-# When we can load the utf8proc library, override normalization with the faster methods
-begin
- require 'utf8proc_native'
- require 'active_support/multibyte/handlers/utf8_handler_proc'
- ActiveSupport::Multibyte::Chars.handler = ActiveSupport::Multibyte::Handlers::UTF8HandlerProc
-rescue LoadError
- ActiveSupport::Multibyte::Chars.handler = ActiveSupport::Multibyte::Handlers::UTF8Handler
-end
+end \ No newline at end of file
diff --git a/activesupport/lib/active_support/multibyte/exceptions.rb b/activesupport/lib/active_support/multibyte/exceptions.rb
new file mode 100644
index 0000000000..62066e3c71
--- /dev/null
+++ b/activesupport/lib/active_support/multibyte/exceptions.rb
@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+module ActiveSupport #:nodoc:
+ module Multibyte #:nodoc:
+ # Raised when a problem with the encoding was found.
+ class EncodingError < StandardError; end
+ end
+end \ No newline at end of file
diff --git a/activesupport/lib/active_support/multibyte/generators/generate_tables.rb b/activesupport/lib/active_support/multibyte/generators/generate_tables.rb
deleted file mode 100755
index 7f807585c5..0000000000
--- a/activesupport/lib/active_support/multibyte/generators/generate_tables.rb
+++ /dev/null
@@ -1,149 +0,0 @@
-#!/usr/bin/env ruby
-begin
- require File.dirname(__FILE__) + '/../../../active_support'
-rescue IOError
-end
-require 'open-uri'
-require 'tmpdir'
-
-module ActiveSupport::Multibyte::Handlers #:nodoc:
- class UnicodeDatabase #:nodoc:
- def self.load
- [Hash.new(Codepoint.new),[],{},{}]
- end
- end
-
- class UnicodeTableGenerator #:nodoc:
- BASE_URI = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::UNICODE_VERSION}/ucd/"
- SOURCES = {
- :codepoints => BASE_URI + 'UnicodeData.txt',
- :composition_exclusion => BASE_URI + 'CompositionExclusions.txt',
- :grapheme_break_property => BASE_URI + 'auxiliary/GraphemeBreakProperty.txt',
- :cp1252 => 'http://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT'
- }
-
- def initialize
- @ucd = UnicodeDatabase.new
-
- default = Codepoint.new
- default.combining_class = 0
- default.uppercase_mapping = 0
- default.lowercase_mapping = 0
- @ucd.codepoints = Hash.new(default)
-
- @ucd.composition_exclusion = []
- @ucd.composition_map = {}
- @ucd.boundary = {}
- @ucd.cp1252 = {}
- end
-
- def parse_codepoints(line)
- codepoint = Codepoint.new
- raise "Could not parse input." unless line =~ /^
- ([0-9A-F]+); # code
- ([^;]+); # name
- ([A-Z]+); # general category
- ([0-9]+); # canonical combining class
- ([A-Z]+); # bidi class
- (<([A-Z]*)>)? # decomposition type
- ((\ ?[0-9A-F]+)*); # decompomposition mapping
- ([0-9]*); # decimal digit
- ([0-9]*); # digit
- ([^;]*); # numeric
- ([YN]*); # bidi mirrored
- ([^;]*); # unicode 1.0 name
- ([^;]*); # iso comment
- ([0-9A-F]*); # simple uppercase mapping
- ([0-9A-F]*); # simple lowercase mapping
- ([0-9A-F]*)$/ix # simple titlecase mapping
- codepoint.code = $1.hex
- #codepoint.name = $2
- #codepoint.category = $3
- codepoint.combining_class = Integer($4)
- #codepoint.bidi_class = $5
- codepoint.decomp_type = $7
- codepoint.decomp_mapping = ($8=='') ? nil : $8.split.collect { |element| element.hex }
- #codepoint.bidi_mirrored = ($13=='Y') ? true : false
- codepoint.uppercase_mapping = ($16=='') ? 0 : $16.hex
- codepoint.lowercase_mapping = ($17=='') ? 0 : $17.hex
- #codepoint.titlecase_mapping = ($18=='') ? nil : $18.hex
- @ucd.codepoints[codepoint.code] = codepoint
- end
-
- def parse_grapheme_break_property(line)
- if line =~ /^([0-9A-F\.]+)\s*;\s*([\w]+)\s*#/
- type = $2.downcase.intern
- @ucd.boundary[type] ||= []
- if $1.include? '..'
- parts = $1.split '..'
- @ucd.boundary[type] << (parts[0].hex..parts[1].hex)
- else
- @ucd.boundary[type] << $1.hex
- end
- end
- end
-
- def parse_composition_exclusion(line)
- if line =~ /^([0-9A-F]+)/i
- @ucd.composition_exclusion << $1.hex
- end
- end
-
- def parse_cp1252(line)
- if line =~ /^([0-9A-Fx]+)\s([0-9A-Fx]+)/i
- @ucd.cp1252[$1.hex] = $2.hex
- end
- end
-
- def create_composition_map
- @ucd.codepoints.each do |_, cp|
- if !cp.nil? and cp.combining_class == 0 and cp.decomp_type.nil? and !cp.decomp_mapping.nil? and cp.decomp_mapping.length == 2 and @ucd[cp.decomp_mapping[0]].combining_class == 0 and !@ucd.composition_exclusion.include?(cp.code)
- @ucd.composition_map[cp.decomp_mapping[0]] ||= {}
- @ucd.composition_map[cp.decomp_mapping[0]][cp.decomp_mapping[1]] = cp.code
- end
- end
- end
-
- def normalize_boundary_map
- @ucd.boundary.each do |k,v|
- if [:lf, :cr].include? k
- @ucd.boundary[k] = v[0]
- end
- end
- end
-
- def parse
- SOURCES.each do |type, url|
- filename = File.join(Dir.tmpdir, "#{url.split('/').last}")
- unless File.exist?(filename)
- $stderr.puts "Downloading #{url.split('/').last}"
- File.open(filename, 'wb') do |target|
- open(url) do |source|
- source.each_line { |line| target.write line }
- end
- end
- end
- File.open(filename) do |file|
- file.each_line { |line| send "parse_#{type}".intern, line }
- end
- end
- create_composition_map
- normalize_boundary_map
- end
-
- def dump_to(filename)
- File.open(filename, 'wb') do |f|
- f.write Marshal.dump([@ucd.codepoints, @ucd.composition_exclusion, @ucd.composition_map, @ucd.boundary, @ucd.cp1252])
- end
- end
- end
-end
-
-if __FILE__ == $0
- filename = ActiveSupport::Multibyte::Handlers::UnicodeDatabase.filename
- generator = ActiveSupport::Multibyte::Handlers::UnicodeTableGenerator.new
- generator.parse
- print "Writing to: #{filename}"
- generator.dump_to filename
- puts " (#{File.size(filename)} bytes)"
-end
diff --git a/activesupport/lib/active_support/multibyte/handlers/passthru_handler.rb b/activesupport/lib/active_support/multibyte/handlers/passthru_handler.rb
deleted file mode 100644
index 916215c2ce..0000000000
--- a/activesupport/lib/active_support/multibyte/handlers/passthru_handler.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-# Chars uses this handler when $KCODE is not set to 'UTF8'. Because this handler doesn't define any methods all call
-# will be forwarded to String.
-class ActiveSupport::Multibyte::Handlers::PassthruHandler #:nodoc:
-
- # Return the original byteoffset
- def self.translate_offset(string, byte_offset) #:nodoc:
- byte_offset
- end
-end \ No newline at end of file
diff --git a/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb b/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb
deleted file mode 100644
index aa9c16f575..0000000000
--- a/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb
+++ /dev/null
@@ -1,564 +0,0 @@
-# Contains all the handlers and helper classes
-module ActiveSupport::Multibyte::Handlers #:nodoc:
- class EncodingError < ArgumentError #:nodoc:
- end
-
- class Codepoint #:nodoc:
- attr_accessor :code, :combining_class, :decomp_type, :decomp_mapping, :uppercase_mapping, :lowercase_mapping
- end
-
- class UnicodeDatabase #:nodoc:
- attr_writer :codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252
-
- # self-expiring methods that lazily load the Unicode database and then return the value.
- [:codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252].each do |attr_name|
- class_eval(<<-EOS, __FILE__, __LINE__)
- def #{attr_name}
- load
- @#{attr_name}
- end
- EOS
- end
-
- # Shortcut to ucd.codepoints[]
- def [](index); codepoints[index]; end
-
- # Returns the directory in which the data files are stored
- def self.dirname
- File.dirname(__FILE__) + '/../../values/'
- end
-
- # Returns the filename for the data file for this version
- def self.filename
- File.expand_path File.join(dirname, "unicode_tables.dat")
- end
-
- # Loads the unicode database and returns all the internal objects of UnicodeDatabase
- # Once the values have been loaded, define attr_reader methods for the instance variables.
- def load
- begin
- @codepoints, @composition_exclusion, @composition_map, @boundary, @cp1252 = File.open(self.class.filename, 'rb') { |f| Marshal.load f.read }
- rescue Exception => e
- raise IOError.new("Couldn't load the unicode tables for UTF8Handler (#{e.message}), handler is unusable")
- end
- @codepoints ||= Hash.new(Codepoint.new)
- @composition_exclusion ||= []
- @composition_map ||= {}
- @boundary ||= {}
- @cp1252 ||= {}
-
- # Redefine the === method so we can write shorter rules for grapheme cluster breaks
- @boundary.each do |k,_|
- @boundary[k].instance_eval do
- def ===(other)
- detect { |i| i === other } ? true : false
- end
- end if @boundary[k].kind_of?(Array)
- end
-
- # define attr_reader methods for the instance variables
- class << self
- attr_reader :codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252
- end
- end
- end
-
- # UTF8Handler implements Unicode aware operations for strings, these operations will be used by the Chars
- # proxy when $KCODE is set to 'UTF8'.
- class UTF8Handler
- # Hangul character boundaries and properties
- HANGUL_SBASE = 0xAC00
- HANGUL_LBASE = 0x1100
- HANGUL_VBASE = 0x1161
- HANGUL_TBASE = 0x11A7
- HANGUL_LCOUNT = 19
- HANGUL_VCOUNT = 21
- HANGUL_TCOUNT = 28
- HANGUL_NCOUNT = HANGUL_VCOUNT * HANGUL_TCOUNT
- HANGUL_SCOUNT = 11172
- HANGUL_SLAST = HANGUL_SBASE + HANGUL_SCOUNT
- HANGUL_JAMO_FIRST = 0x1100
- HANGUL_JAMO_LAST = 0x11FF
-
- # All the unicode whitespace
- UNICODE_WHITESPACE = [
- (0x0009..0x000D).to_a, # White_Space # Cc [5] <control-0009>..<control-000D>
- 0x0020, # White_Space # Zs SPACE
- 0x0085, # White_Space # Cc <control-0085>
- 0x00A0, # White_Space # Zs NO-BREAK SPACE
- 0x1680, # White_Space # Zs OGHAM SPACE MARK
- 0x180E, # White_Space # Zs MONGOLIAN VOWEL SEPARATOR
- (0x2000..0x200A).to_a, # White_Space # Zs [11] EN QUAD..HAIR SPACE
- 0x2028, # White_Space # Zl LINE SEPARATOR
- 0x2029, # White_Space # Zp PARAGRAPH SEPARATOR
- 0x202F, # White_Space # Zs NARROW NO-BREAK SPACE
- 0x205F, # White_Space # Zs MEDIUM MATHEMATICAL SPACE
- 0x3000, # White_Space # Zs IDEOGRAPHIC SPACE
- ].flatten.freeze
-
- # BOM (byte order mark) can also be seen as whitespace, it's a non-rendering character used to distinguish
- # between little and big endian. This is not an issue in utf-8, so it must be ignored.
- UNICODE_LEADERS_AND_TRAILERS = UNICODE_WHITESPACE + [65279] # ZERO-WIDTH NO-BREAK SPACE aka BOM
-
- # Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site)
- UTF8_PAT = /\A(?:
- [\x00-\x7f] |
- [\xc2-\xdf] [\x80-\xbf] |
- \xe0 [\xa0-\xbf] [\x80-\xbf] |
- [\xe1-\xef] [\x80-\xbf] [\x80-\xbf] |
- \xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] |
- [\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] |
- \xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf]
- )*\z/xn
-
- # Returns a regular expression pattern that matches the passed Unicode codepoints
- def self.codepoints_to_pattern(array_of_codepoints) #:nodoc:
- array_of_codepoints.collect{ |e| [e].pack 'U*' }.join('|')
- end
- UNICODE_TRAILERS_PAT = /(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+\Z/
- UNICODE_LEADERS_PAT = /\A(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+/
-
- class << self
-
- # ///
- # /// BEGIN String method overrides
- # ///
-
- # Inserts the passed string at specified codepoint offsets
- def insert(str, offset, fragment)
- str.replace(
- u_unpack(str).insert(
- offset,
- u_unpack(fragment)
- ).flatten.pack('U*')
- )
- end
-
- # Returns the position of the passed argument in the string, counting in codepoints
- def index(str, *args)
- bidx = str.index(*args)
- bidx ? (u_unpack(str.slice(0...bidx)).size) : nil
- end
-
- # Works just like the indexed replace method on string, except instead of byte offsets you specify
- # character offsets.
- #
- # Example:
- #
- # s = "Müller"
- # s.chars[2] = "e" # Replace character with offset 2
- # s # => "Müeler"
- #
- # s = "Müller"
- # s.chars[1, 2] = "ö" # Replace 2 characters at character offset 1
- # s # => "Möler"
- def []=(str, *args)
- replace_by = args.pop
- # Indexed replace with regular expressions already works
- return str[*args] = replace_by if args.first.is_a?(Regexp)
- result = u_unpack(str)
- if args[0].is_a?(Fixnum)
- raise IndexError, "index #{args[0]} out of string" if args[0] >= result.length
- min = args[0]
- max = args[1].nil? ? min : (min + args[1] - 1)
- range = Range.new(min, max)
- replace_by = [replace_by].pack('U') if replace_by.is_a?(Fixnum)
- elsif args.first.is_a?(Range)
- raise RangeError, "#{args[0]} out of range" if args[0].min >= result.length
- range = args[0]
- else
- needle = args[0].to_s
- min = index(str, needle)
- max = min + length(needle) - 1
- range = Range.new(min, max)
- end
- result[range] = u_unpack(replace_by)
- str.replace(result.pack('U*'))
- end
-
- # Works just like String#rjust, only integer specifies characters instead of bytes.
- #
- # Example:
- #
- # "¾ cup".chars.rjust(8).to_s
- # # => " ¾ cup"
- #
- # "¾ cup".chars.rjust(8, " ").to_s # Use non-breaking whitespace
- # # => "   ¾ cup"
- def rjust(str, integer, padstr=' ')
- justify(str, integer, :right, padstr)
- end
-
- # Works just like String#ljust, only integer specifies characters instead of bytes.
- #
- # Example:
- #
- # "¾ cup".chars.rjust(8).to_s
- # # => "¾ cup "
- #
- # "¾ cup".chars.rjust(8, " ").to_s # Use non-breaking whitespace
- # # => "¾ cup   "
- def ljust(str, integer, padstr=' ')
- justify(str, integer, :left, padstr)
- end
-
- # Works just like String#center, only integer specifies characters instead of bytes.
- #
- # Example:
- #
- # "¾ cup".chars.center(8).to_s
- # # => " ¾ cup "
- #
- # "¾ cup".chars.center(8, " ").to_s # Use non-breaking whitespace
- # # => " ¾ cup  "
- def center(str, integer, padstr=' ')
- justify(str, integer, :center, padstr)
- end
-
- # Does Unicode-aware rstrip
- def rstrip(str)
- str.gsub(UNICODE_TRAILERS_PAT, '')
- end
-
- # Does Unicode-aware lstrip
- def lstrip(str)
- str.gsub(UNICODE_LEADERS_PAT, '')
- end
-
- # Removed leading and trailing whitespace
- def strip(str)
- str.gsub(UNICODE_LEADERS_PAT, '').gsub(UNICODE_TRAILERS_PAT, '')
- end
-
- # Returns the number of codepoints in the string
- def size(str)
- u_unpack(str).size
- end
- alias_method :length, :size
-
- # Reverses codepoints in the string.
- def reverse(str)
- u_unpack(str).reverse.pack('U*')
- end
-
- # Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that
- # character.
- def slice(str, *args)
- if args.size > 2
- raise ArgumentError, "wrong number of arguments (#{args.size} for 1)" # Do as if we were native
- elsif (args.size == 2 && !(args.first.is_a?(Numeric) || args.first.is_a?(Regexp)))
- raise TypeError, "cannot convert #{args.first.class} into Integer" # Do as if we were native
- elsif (args.size == 2 && !args[1].is_a?(Numeric))
- raise TypeError, "cannot convert #{args[1].class} into Integer" # Do as if we were native
- elsif args[0].kind_of? Range
- cps = u_unpack(str).slice(*args)
- cps.nil? ? nil : cps.pack('U*')
- elsif args[0].kind_of? Regexp
- str.slice(*args)
- elsif args.size == 1 && args[0].kind_of?(Numeric)
- u_unpack(str)[args[0]]
- else
- u_unpack(str).slice(*args).pack('U*')
- end
- end
- alias_method :[], :slice
-
- # Convert characters in the string to uppercase
- def upcase(str); to_case :uppercase_mapping, str; end
-
- # Convert characters in the string to lowercase
- def downcase(str); to_case :lowercase_mapping, str; end
-
- # Returns a copy of +str+ with the first character converted to uppercase and the remainder to lowercase
- def capitalize(str)
- upcase(slice(str, 0..0)) + downcase(slice(str, 1..-1) || '')
- end
-
- # ///
- # /// Extra String methods for unicode operations
- # ///
-
- # Returns the KC normalization of the string by default. NFKC is considered the best normalization form for
- # passing strings to databases and validations.
- #
- # * <tt>str</tt> - The string to perform normalization on.
- # * <tt>form</tt> - The form you want to normalize in. Should be one of the following:
- # <tt>:c</tt>, <tt>:kc</tt>, <tt>:d</tt>, or <tt>:kd</tt>. Default is
- # ActiveSupport::Multibyte::DEFAULT_NORMALIZATION_FORM.
- def normalize(str, form=ActiveSupport::Multibyte::DEFAULT_NORMALIZATION_FORM)
- # See http://www.unicode.org/reports/tr15, Table 1
- codepoints = u_unpack(str)
- case form
- when :d
- reorder_characters(decompose_codepoints(:canonical, codepoints))
- when :c
- compose_codepoints reorder_characters(decompose_codepoints(:canonical, codepoints))
- when :kd
- reorder_characters(decompose_codepoints(:compatability, codepoints))
- when :kc
- compose_codepoints reorder_characters(decompose_codepoints(:compatability, codepoints))
- else
- raise ArgumentError, "#{form} is not a valid normalization variant", caller
- end.pack('U*')
- end
-
- # Perform decomposition on the characters in the string
- def decompose(str)
- decompose_codepoints(:canonical, u_unpack(str)).pack('U*')
- end
-
- # Perform composition on the characters in the string
- def compose(str)
- compose_codepoints u_unpack(str).pack('U*')
- end
-
- # ///
- # /// BEGIN Helper methods for unicode operation
- # ///
-
- # Used to translate an offset from bytes to characters, for instance one received from a regular expression match
- def translate_offset(str, byte_offset)
- return nil if byte_offset.nil?
- return 0 if str == ''
- chunk = str[0..byte_offset]
- begin
- begin
- chunk.unpack('U*').length - 1
- rescue ArgumentError => e
- chunk = str[0..(byte_offset+=1)]
- # Stop retrying at the end of the string
- raise e unless byte_offset < chunk.length
- # We damaged a character, retry
- retry
- end
- # Catch the ArgumentError so we can throw our own
- rescue ArgumentError
- raise EncodingError.new('malformed UTF-8 character')
- end
- end
-
- # Checks if the string is valid UTF8.
- def consumes?(str)
- # Unpack is a little bit faster than regular expressions
- begin
- str.unpack('U*')
- true
- rescue ArgumentError
- false
- end
- end
-
- # Returns the number of grapheme clusters in the string. This method is very likely to be moved or renamed
- # in future versions.
- def g_length(str)
- g_unpack(str).length
- end
-
- # Replaces all the non-utf-8 bytes by their iso-8859-1 or cp1252 equivalent resulting in a valid utf-8 string
- def tidy_bytes(str)
- str.split(//u).map do |c|
- if !UTF8_PAT.match(c)
- n = c.unpack('C')[0]
- n < 128 ? n.chr :
- n < 160 ? [UCD.cp1252[n] || n].pack('U') :
- n < 192 ? "\xC2" + n.chr : "\xC3" + (n-64).chr
- else
- c
- end
- end.join
- end
-
- protected
-
- # Detect whether the codepoint is in a certain character class. Primarily used by the
- # grapheme cluster support.
- def in_char_class?(codepoint, classes)
- classes.detect { |c| UCD.boundary[c] === codepoint } ? true : false
- end
-
- # Unpack the string at codepoints boundaries
- def u_unpack(str)
- begin
- str.unpack 'U*'
- rescue ArgumentError
- raise EncodingError.new('malformed UTF-8 character')
- end
- end
-
- # Unpack the string at grapheme boundaries instead of codepoint boundaries
- def g_unpack(str)
- codepoints = u_unpack(str)
- unpacked = []
- pos = 0
- marker = 0
- eoc = codepoints.length
- while(pos < eoc)
- pos += 1
- previous = codepoints[pos-1]
- current = codepoints[pos]
- if (
- # CR X LF
- one = ( previous == UCD.boundary[:cr] and current == UCD.boundary[:lf] ) or
- # L X (L|V|LV|LVT)
- two = ( UCD.boundary[:l] === previous and in_char_class?(current, [:l,:v,:lv,:lvt]) ) or
- # (LV|V) X (V|T)
- three = ( in_char_class?(previous, [:lv,:v]) and in_char_class?(current, [:v,:t]) ) or
- # (LVT|T) X (T)
- four = ( in_char_class?(previous, [:lvt,:t]) and UCD.boundary[:t] === current ) or
- # X Extend
- five = (UCD.boundary[:extend] === current)
- )
- else
- unpacked << codepoints[marker..pos-1]
- marker = pos
- end
- end
- unpacked
- end
-
- # Reverse operation of g_unpack
- def g_pack(unpacked)
- unpacked.flatten
- end
-
- # Justifies a string in a certain way. Valid values for <tt>way</tt> are <tt>:right</tt>, <tt>:left</tt> and
- # <tt>:center</tt>. Is primarily used as a helper method by <tt>rjust</tt>, <tt>ljust</tt> and <tt>center</tt>.
- def justify(str, integer, way, padstr=' ')
- raise ArgumentError, "zero width padding" if padstr.length == 0
- padsize = integer - size(str)
- padsize = padsize > 0 ? padsize : 0
- case way
- when :right
- str.dup.insert(0, padding(padsize, padstr))
- when :left
- str.dup.insert(-1, padding(padsize, padstr))
- when :center
- lpad = padding((padsize / 2.0).floor, padstr)
- rpad = padding((padsize / 2.0).ceil, padstr)
- str.dup.insert(0, lpad).insert(-1, rpad)
- end
- end
-
- # Generates a padding string of a certain size.
- def padding(padsize, padstr=' ')
- if padsize != 0
- slice(padstr * ((padsize / size(padstr)) + 1), 0, padsize)
- else
- ''
- end
- end
-
- # Convert characters to a different case
- def to_case(way, str)
- u_unpack(str).map do |codepoint|
- cp = UCD[codepoint]
- unless cp.nil?
- ncp = cp.send(way)
- ncp > 0 ? ncp : codepoint
- else
- codepoint
- end
- end.pack('U*')
- end
-
- # Re-order codepoints so the string becomes canonical
- def reorder_characters(codepoints)
- length = codepoints.length- 1
- pos = 0
- while pos < length do
- cp1, cp2 = UCD[codepoints[pos]], UCD[codepoints[pos+1]]
- if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
- codepoints[pos..pos+1] = cp2.code, cp1.code
- pos += (pos > 0 ? -1 : 1)
- else
- pos += 1
- end
- end
- codepoints
- end
-
- # Decompose composed characters to the decomposed form
- def decompose_codepoints(type, codepoints)
- codepoints.inject([]) do |decomposed, cp|
- # if it's a hangul syllable starter character
- if HANGUL_SBASE <= cp and cp < HANGUL_SLAST
- sindex = cp - HANGUL_SBASE
- ncp = [] # new codepoints
- ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
- ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
- tindex = sindex % HANGUL_TCOUNT
- ncp << (HANGUL_TBASE + tindex) unless tindex == 0
- decomposed.concat ncp
- # if the codepoint is decomposable in with the current decomposition type
- elsif (ncp = UCD[cp].decomp_mapping) and (!UCD[cp].decomp_type || type == :compatability)
- decomposed.concat decompose_codepoints(type, ncp.dup)
- else
- decomposed << cp
- end
- end
- end
-
- # Compose decomposed characters to the composed form
- def compose_codepoints(codepoints)
- pos = 0
- eoa = codepoints.length - 1
- starter_pos = 0
- starter_char = codepoints[0]
- previous_combining_class = -1
- while pos < eoa
- pos += 1
- lindex = starter_char - HANGUL_LBASE
- # -- Hangul
- if 0 <= lindex and lindex < HANGUL_LCOUNT
- vindex = codepoints[starter_pos+1] - HANGUL_VBASE rescue vindex = -1
- if 0 <= vindex and vindex < HANGUL_VCOUNT
- tindex = codepoints[starter_pos+2] - HANGUL_TBASE rescue tindex = -1
- if 0 <= tindex and tindex < HANGUL_TCOUNT
- j = starter_pos + 2
- eoa -= 2
- else
- tindex = 0
- j = starter_pos + 1
- eoa -= 1
- end
- codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
- end
- starter_pos += 1
- starter_char = codepoints[starter_pos]
- # -- Other characters
- else
- current_char = codepoints[pos]
- current = UCD[current_char]
- if current.combining_class > previous_combining_class
- if ref = UCD.composition_map[starter_char]
- composition = ref[current_char]
- else
- composition = nil
- end
- unless composition.nil?
- codepoints[starter_pos] = composition
- starter_char = composition
- codepoints.delete_at pos
- eoa -= 1
- pos -= 1
- previous_combining_class = -1
- else
- previous_combining_class = current.combining_class
- end
- else
- previous_combining_class = current.combining_class
- end
- if current.combining_class == 0
- starter_pos = pos
- starter_char = codepoints[pos]
- end
- end
- end
- codepoints
- end
-
- # UniCode Database
- UCD = UnicodeDatabase.new
- end
- end
-end
diff --git a/activesupport/lib/active_support/multibyte/handlers/utf8_handler_proc.rb b/activesupport/lib/active_support/multibyte/handlers/utf8_handler_proc.rb
deleted file mode 100644
index f10eecc622..0000000000
--- a/activesupport/lib/active_support/multibyte/handlers/utf8_handler_proc.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-# Methods in this handler call functions in the utf8proc ruby extension. These are significantly faster than the
-# pure ruby versions. Chars automatically uses this handler when it can load the utf8proc extension. For
-# documentation on handler methods see UTF8Handler.
-class ActiveSupport::Multibyte::Handlers::UTF8HandlerProc < ActiveSupport::Multibyte::Handlers::UTF8Handler #:nodoc:
- class << self
- def normalize(str, form=ActiveSupport::Multibyte::DEFAULT_NORMALIZATION_FORM) #:nodoc:
- codepoints = str.unpack('U*')
- case form
- when :d
- utf8map(str, :stable)
- when :c
- utf8map(str, :stable, :compose)
- when :kd
- utf8map(str, :stable, :compat)
- when :kc
- utf8map(str, :stable, :compose, :compat)
- else
- raise ArgumentError, "#{form} is not a valid normalization variant", caller
- end
- end
-
- def decompose(str) #:nodoc:
- utf8map(str, :stable)
- end
-
- def downcase(str) #:nodoc:c
- utf8map(str, :casefold)
- end
-
- protected
-
- def utf8map(str, *option_array) #:nodoc:
- options = 0
- option_array.each do |option|
- flag = Utf8Proc::Options[option]
- raise ArgumentError, "Unknown argument given to utf8map." unless
- flag
- options |= flag
- end
- return Utf8Proc::utf8map(str, options)
- end
- end
-end
diff --git a/activesupport/lib/active_support/multibyte/unicode_database.rb b/activesupport/lib/active_support/multibyte/unicode_database.rb
new file mode 100644
index 0000000000..3b8cf8f9eb
--- /dev/null
+++ b/activesupport/lib/active_support/multibyte/unicode_database.rb
@@ -0,0 +1,71 @@
+# encoding: utf-8
+
+module ActiveSupport #:nodoc:
+ module Multibyte #:nodoc:
+ # Holds data about a codepoint in the Unicode database
+ class Codepoint
+ attr_accessor :code, :combining_class, :decomp_type, :decomp_mapping, :uppercase_mapping, :lowercase_mapping
+ end
+
+ # Holds static data from the Unicode database
+ class UnicodeDatabase
+ ATTRIBUTES = :codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252
+
+ attr_writer(*ATTRIBUTES)
+
+ def initialize
+ @codepoints = Hash.new(Codepoint.new)
+ @composition_exclusion = []
+ @composition_map = {}
+ @boundary = {}
+ @cp1252 = {}
+ end
+
+ # Lazy load the Unicode database so it's only loaded when it's actually used
+ ATTRIBUTES.each do |attr_name|
+ class_eval(<<-EOS, __FILE__, __LINE__)
+ def #{attr_name}
+ load
+ @#{attr_name}
+ end
+ EOS
+ end
+
+ # Loads the Unicode database and returns all the internal objects of UnicodeDatabase.
+ def load
+ begin
+ @codepoints, @composition_exclusion, @composition_map, @boundary, @cp1252 = File.open(self.class.filename, 'rb') { |f| Marshal.load f.read }
+ rescue Exception => e
+ raise IOError.new("Couldn't load the Unicode tables for UTF8Handler (#{e.message}), ActiveSupport::Multibyte is unusable")
+ end
+
+ # Redefine the === method so we can write shorter rules for grapheme cluster breaks
+ @boundary.each do |k,_|
+ @boundary[k].instance_eval do
+ def ===(other)
+ detect { |i| i === other } ? true : false
+ end
+ end if @boundary[k].kind_of?(Array)
+ end
+
+ # define attr_reader methods for the instance variables
+ class << self
+ attr_reader(*ATTRIBUTES)
+ end
+ end
+
+ # Returns the directory in which the data files are stored
+ def self.dirname
+ File.dirname(__FILE__) + '/../values/'
+ end
+
+ # Returns the filename for the data file for this version
+ def self.filename
+ File.expand_path File.join(dirname, "unicode_tables.dat")
+ end
+ end
+
+ # UniCode Database
+ UCD = UnicodeDatabase.new
+ end
+end \ No newline at end of file
diff --git a/activesupport/lib/active_support/values/unicode_tables.dat b/activesupport/lib/active_support/values/unicode_tables.dat
index 35edb148c3..74b333d416 100644
--- a/activesupport/lib/active_support/values/unicode_tables.dat
+++ b/activesupport/lib/active_support/values/unicode_tables.dat
Binary files differ
diff --git a/activesupport/test/abstract_unit.rb b/activesupport/test/abstract_unit.rb
index cce8d5d220..a698beaa0e 100644
--- a/activesupport/test/abstract_unit.rb
+++ b/activesupport/test/abstract_unit.rb
@@ -1,9 +1,15 @@
+# encoding: utf-8
+
require 'test/unit'
$:.unshift "#{File.dirname(__FILE__)}/../lib"
$:.unshift File.dirname(__FILE__)
require 'active_support'
+if RUBY_VERSION < '1.9'
+ $KCODE = 'UTF8'
+end
+
def uses_gem(gem_name, test_name, version = '> 0')
require 'rubygems'
gem gem_name.to_s, version
@@ -22,3 +28,16 @@ end
# Show backtraces for deprecated behavior for quicker cleanup.
ActiveSupport::Deprecation.debug = true
+
+def with_kcode(code)
+ if RUBY_VERSION < '1.9'
+ begin
+ old_kcode, $KCODE = $KCODE, code
+ yield
+ ensure
+ $KCODE = old_kcode
+ end
+ else
+ yield
+ end
+end \ No newline at end of file
diff --git a/activesupport/test/core_ext/string_ext_test.rb b/activesupport/test/core_ext/string_ext_test.rb
index c0decf2c3f..b086c957fe 100644
--- a/activesupport/test/core_ext/string_ext_test.rb
+++ b/activesupport/test/core_ext/string_ext_test.rb
@@ -191,13 +191,12 @@ class StringInflectionsTest < Test::Unit::TestCase
if RUBY_VERSION < '1.9'
def test_each_char_with_utf8_string_when_kcode_is_utf8
- old_kcode, $KCODE = $KCODE, 'UTF8'
- '€2.99'.each_char do |char|
- assert_not_equal 1, char.length
- break
+ with_kcode('UTF8') do
+ '€2.99'.each_char do |char|
+ assert_not_equal 1, char.length
+ break
+ end
end
- ensure
- $KCODE = old_kcode
end
end
end
@@ -206,4 +205,52 @@ class StringBehaviourTest < Test::Unit::TestCase
def test_acts_like_string
assert 'Bambi'.acts_like_string?
end
+end
+
+class CoreExtStringMultibyteTest < Test::Unit::TestCase
+ UNICODE_STRING = 'こにちわ'
+ ASCII_STRING = 'ohayo'
+ BYTE_STRING = "\270\236\010\210\245"
+
+ def test_core_ext_adds_mb_chars
+ assert UNICODE_STRING.respond_to?(:mb_chars)
+ end
+
+ def test_string_should_recognize_utf8_strings
+ assert UNICODE_STRING.is_utf8?
+ assert ASCII_STRING.is_utf8?
+ assert !BYTE_STRING.is_utf8?
+ end
+
+ if RUBY_VERSION < '1.8.7'
+ def test_core_ext_adds_chars
+ assert UNICODE_STRING.respond_to?(:chars)
+ end
+
+ def test_chars_warns_about_deprecation
+ assert_deprecated("String#chars") do
+ ''.chars
+ end
+ end
+ end
+
+ if RUBY_VERSION < '1.9'
+ def test_mb_chars_returns_self_when_kcode_not_set
+ with_kcode('none') do
+ assert UNICODE_STRING.mb_chars.kind_of?(String)
+ end
+ end
+
+ def test_mb_chars_returns_an_instance_of_the_chars_proxy_when_kcode_utf8
+ with_kcode('UTF8') do
+ assert UNICODE_STRING.mb_chars.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ end
+ end
+ end
+
+ if RUBY_VERSION >= '1.9'
+ def test_mb_chars_returns_string
+ assert UNICODE_STRING.mb_chars.kind_of?(String)
+ end
+ end
end \ No newline at end of file
diff --git a/activesupport/test/json/encoding_test.rb b/activesupport/test/json/encoding_test.rb
index 38bb8f3e79..497f028369 100644
--- a/activesupport/test/json/encoding_test.rb
+++ b/activesupport/test/json/encoding_test.rb
@@ -101,18 +101,6 @@ class TestJSONEncoding < Test::Unit::TestCase
end
protected
- def with_kcode(code)
- if RUBY_VERSION < '1.9'
- begin
- old_kcode, $KCODE = $KCODE, 'UTF8'
- yield
- ensure
- $KCODE = old_kcode
- end
- else
- yield
- end
- end
def object_keys(json_object)
json_object[1..-2].scan(/([^{}:,\s]+):/).flatten.sort
diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb
index a87309b038..ca2af9b986 100644
--- a/activesupport/test/multibyte_chars_test.rb
+++ b/activesupport/test/multibyte_chars_test.rb
@@ -1,186 +1,569 @@
# encoding: utf-8
+
require 'abstract_unit'
+require 'multibyte_test_helpers'
-if RUBY_VERSION < '1.9'
+class String
+ def __method_for_multibyte_testing_with_integer_result; 1; end
+ def __method_for_multibyte_testing; 'result'; end
+ def __method_for_multibyte_testing!; 'result'; end
+end
-$KCODE = 'UTF8'
+class MultibyteCharsTest < Test::Unit::TestCase
+ include MultibyteTestHelpers
-class CharsTest < Test::Unit::TestCase
-
def setup
- @s = {
- :utf8 => "Abcd Блå ffi блa 埋",
- :ascii => "asci ias c iia s",
- :bytes => "\270\236\010\210\245"
- }
+ @proxy_class = ActiveSupport::Multibyte::Chars
+ @chars = @proxy_class.new UNICODE_STRING
+ end
+
+ def test_wraps_the_original_string
+ assert_equal UNICODE_STRING, @chars.to_s
+ assert_equal UNICODE_STRING, @chars.wrapped_string
end
-
- def test_sanity
- @s.each do |t, s|
- assert s.respond_to?(:chars), "All string should have the chars method (#{t})"
- assert s.respond_to?(:to_s), "All string should have the to_s method (#{t})"
- assert_kind_of ActiveSupport::Multibyte::Chars, s.chars, "#chars should return an instance of Chars (#{t})"
+
+ def test_should_allow_method_calls_to_string
+ assert_nothing_raised do
+ @chars.__method_for_multibyte_testing
+ end
+ assert_raises NoMethodError do
+ @chars.__unknown_method
end
end
-
- def test_comparability
- @s.each do |t, s|
- assert_equal s, s.chars.to_s, "Chars#to_s should return enclosed string unchanged"
+
+ def test_forwarded_method_calls_should_return_new_chars_instance
+ assert @chars.__method_for_multibyte_testing.kind_of?(@proxy_class)
+ assert_not_equal @chars.object_id, @chars.__method_for_multibyte_testing.object_id
+ end
+
+ def test_forwarded_bang_method_calls_should_return_the_original_chars_instance
+ assert @chars.__method_for_multibyte_testing!.kind_of?(@proxy_class)
+ assert_equal @chars.object_id, @chars.__method_for_multibyte_testing!.object_id
+ end
+
+ def test_methods_are_forwarded_to_wrapped_string_for_byte_strings
+ assert_equal BYTE_STRING.class, BYTE_STRING.mb_chars.class
+ end
+
+ def test_forwarded_method_with_non_string_result_should_be_returned_vertabim
+ assert_equal ''.__method_for_multibyte_testing_with_integer_result, @chars.__method_for_multibyte_testing_with_integer_result
+ end
+
+ def test_should_concatenate
+ assert_equal 'ab', 'a'.mb_chars + 'b'
+ assert_equal 'ab', 'a' + 'b'.mb_chars
+ assert_equal 'ab', 'a'.mb_chars + 'b'.mb_chars
+
+ assert_equal 'ab', 'a'.mb_chars << 'b'
+ assert_equal 'ab', 'a' << 'b'.mb_chars
+ assert_equal 'ab', 'a'.mb_chars << 'b'.mb_chars
+ end
+
+ def test_consumes_utf8_strings
+ assert @proxy_class.consumes?(UNICODE_STRING)
+ assert @proxy_class.consumes?(ASCII_STRING)
+ assert !@proxy_class.consumes?(BYTE_STRING)
+ end
+
+ def test_unpack_utf8_strings
+ assert_equal 4, @proxy_class.u_unpack(UNICODE_STRING).length
+ assert_equal 5, @proxy_class.u_unpack(ASCII_STRING).length
+ end
+
+ def test_unpack_raises_encoding_error_on_broken_strings
+ assert_raises(ActiveSupport::Multibyte::EncodingError) do
+ @proxy_class.u_unpack(BYTE_STRING)
end
- assert_nothing_raised do
- assert_equal "a", "a", "Normal string comparisons should be unaffected"
- assert_not_equal "a", "b", "Normal string comparisons should be unaffected"
- assert_not_equal "a".chars, "b".chars, "Chars objects should be comparable"
- assert_equal "a".chars, "A".downcase.chars, "Chars objects should be comparable to each other"
- assert_equal "a".chars, "A".downcase, "Chars objects should be comparable to strings coming from elsewhere"
- end
-
- assert !@s[:utf8].eql?(@s[:utf8].chars), "Strict comparison is not supported"
- assert_equal @s[:utf8], @s[:utf8].chars, "Chars should be compared by their enclosed string"
-
- other_string = @s[:utf8].dup
- assert_equal other_string, @s[:utf8].chars, "Chars should be compared by their enclosed string"
- assert_equal other_string.chars, @s[:utf8].chars, "Chars should be compared by their enclosed string"
-
- strings = ['builder'.chars, 'armor'.chars, 'zebra'.chars]
- strings.sort!
- assert_equal ['armor', 'builder', 'zebra'], strings, "Chars should be sortable based on their enclosed string"
-
- # This leads to a StackLevelTooDeep exception if the comparison is not wired properly
- assert_raise(NameError) do
- Chars
- end
- end
-
- def test_utf8?
- assert @s[:utf8].is_utf8?, "UTF-8 strings are UTF-8"
- assert @s[:ascii].is_utf8?, "All ASCII strings are also valid UTF-8"
- assert !@s[:bytes].is_utf8?, "This bytestring isn't UTF-8"
- end
-
- # The test for the following methods are defined here because they can only be defined on the Chars class for
- # various reasons
-
- def test_gsub
- assert_equal 'éxa', 'éda'.chars.gsub(/d/, 'x')
- with_kcode('none') do
- assert_equal 'éxa', 'éda'.chars.gsub(/d/, 'x')
- end
- end
-
- def test_split
- word = "efficient"
- chars = ["e", "ffi", "c", "i", "e", "n", "t"]
- assert_equal chars, word.split(//)
- assert_equal chars, word.chars.split(//)
- assert_kind_of ActiveSupport::Multibyte::Chars, word.chars.split(//).first, "Split should return Chars instances"
- end
-
- def test_regexp
- with_kcode('none') do
- assert_equal 12, (@s[:utf8].chars =~ /ffi/),
- "Regex matching should be bypassed to String"
- end
- with_kcode('UTF8') do
- assert_equal 9, (@s[:utf8].chars =~ /ffi/),
- "Regex matching should be unicode aware"
- assert_nil((''.chars =~ /\d+/),
- "Non-matching regular expressions should return nil")
- end
- end
-
- def test_pragma
- if RUBY_VERSION < '1.9'
- with_kcode('UTF8') do
- assert " ".chars.send(:utf8_pragma?), "UTF8 pragma should be on because KCODE is UTF8"
+ end
+
+ if RUBY_VERSION < '1.9'
+ def test_concatenation_should_return_a_proxy_class_instance
+ assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars + 'b').class
+ assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars << 'b').class
+ end
+
+ def test_ascii_strings_are_treated_at_utf8_strings
+ assert_equal ActiveSupport::Multibyte.proxy_class, ASCII_STRING.mb_chars.class
+ end
+
+ def test_concatenate_should_return_proxy_instance
+ assert(('a'.mb_chars + 'b').kind_of?(@proxy_class))
+ assert(('a'.mb_chars + 'b'.mb_chars).kind_of?(@proxy_class))
+ assert(('a'.mb_chars << 'b').kind_of?(@proxy_class))
+ assert(('a'.mb_chars << 'b'.mb_chars).kind_of?(@proxy_class))
+ end
+ end
+end
+
+class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase
+ include MultibyteTestHelpers
+
+ def setup
+ @chars = UNICODE_STRING.dup.mb_chars
+
+ # NEWLINE, SPACE, EM SPACE
+ @whitespace = "\n#{[32, 8195].pack('U*')}"
+ @whitespace.force_encoding(Encoding::UTF_8) if @whitespace.respond_to?(:force_encoding)
+ @byte_order_mark = [65279].pack('U')
+ end
+
+ if RUBY_VERSION < '1.9'
+ def test_split_should_return_an_array_of_chars_instances
+ @chars.split(//).each do |character|
+ assert character.kind_of?(ActiveSupport::Multibyte.proxy_class)
end
- with_kcode('none') do
- assert !" ".chars.send(:utf8_pragma?), "UTF8 pragma should be off because KCODE is not UTF8"
+ end
+
+ def test_indexed_insert_accepts_fixnums
+ @chars[2] = 32
+ assert_equal 'こに わ', @chars
+ end
+
+ def test_overridden_bang_methods_return_self
+ [:rstrip!, :lstrip!, :strip!, :reverse!, :upcase!, :downcase!, :capitalize!].each do |method|
+ assert_equal @chars.object_id, @chars.send(method).object_id
end
- else
- assert !" ".chars.send(:utf8_pragma?), "UTF8 pragma should be off in Ruby 1.9"
- end
- end
-
- def test_handler_setting
- handler = ''.chars.handler
-
- ActiveSupport::Multibyte::Chars.handler = :first
- assert_equal :first, ''.chars.handler
- ActiveSupport::Multibyte::Chars.handler = :second
- assert_equal :second, ''.chars.handler
- assert_raise(NoMethodError) do
- ''.chars.handler.split
- end
-
- ActiveSupport::Multibyte::Chars.handler = handler
- end
-
- def test_method_chaining
- assert_kind_of ActiveSupport::Multibyte::Chars, ''.chars.downcase
- assert_kind_of ActiveSupport::Multibyte::Chars, ''.chars.strip, "Strip should return a Chars object"
- assert_kind_of ActiveSupport::Multibyte::Chars, ''.chars.downcase.strip, "The Chars object should be " +
- "forwarded down the call path for chaining"
- assert_equal 'foo', " FOO ".chars.normalize.downcase.strip, "The Chars that results from the " +
- " operations should be comparable to the string value of the result"
- end
-
- def test_passthrough_on_kcode
- # The easiest way to check if the passthrough is in place is through #size
- with_kcode('none') do
- assert_equal 26, @s[:utf8].chars.size
- end
- with_kcode('UTF8') do
- assert_equal 17, @s[:utf8].chars.size
- end
- end
-
- def test_destructiveness
- # Note that we're testing the destructiveness here and not the correct behaviour of the methods
- str = 'ac'
- str.chars.insert(1, 'b')
- assert_equal 'abc', str, 'Insert should be destructive for a string'
-
- str = 'ac'
- str.chars.reverse!
- assert_equal 'ca', str, 'reverse! should be destructive for a string'
- end
-
- def test_resilience
- assert_nothing_raised do
- assert_equal 5, @s[:bytes].chars.size, "The sequence contains five interpretable bytes"
+ assert_equal @chars.object_id, @chars.slice!(1).object_id
end
- reversed = [0xb8, 0x17e, 0x8, 0x2c6, 0xa5].reverse.pack('U*')
- assert_nothing_raised do
- assert_equal reversed, @s[:bytes].chars.reverse.to_s, "Reversing the string should only yield interpretable bytes"
+
+ def test_overridden_bang_methods_change_wrapped_string
+ [:rstrip!, :lstrip!, :strip!, :reverse!, :upcase!, :downcase!].each do |method|
+ original = ' Café '
+ proxy = chars(original.dup)
+ proxy.send(method)
+ assert_not_equal original, proxy.to_s
+ end
+ proxy = chars('Café')
+ proxy.slice!(3)
+ assert_equal 'é', proxy.to_s
+
+ proxy = chars('òu')
+ proxy.capitalize!
+ assert_equal 'Òu', proxy.to_s
end
- assert_nothing_raised do
- @s[:bytes].chars.reverse!
- assert_equal reversed, @s[:bytes].to_s, "Reversing the string should only yield interpretable bytes"
+ end
+
+ if RUBY_VERSION >= '1.9'
+ def test_unicode_string_should_have_utf8_encoding
+ assert_equal Encoding::UTF_8, UNICODE_STRING.encoding
end
end
-
- def test_duck_typing
- assert_equal true, 'test'.chars.respond_to?(:strip)
- assert_equal true, 'test'.chars.respond_to?(:normalize)
- assert_equal true, 'test'.chars.respond_to?(:normalize!)
- assert_equal false, 'test'.chars.respond_to?(:a_method_that_doesnt_exist)
+
+ def test_identity
+ assert_equal @chars, @chars
+ assert @chars.eql?(@chars)
+ if RUBY_VERSION <= '1.9'
+ assert !@chars.eql?(UNICODE_STRING)
+ else
+ assert @chars.eql?(UNICODE_STRING)
+ end
end
-
- def test_acts_like_string
- assert 'Bambi'.chars.acts_like_string?
+
+ def test_string_methods_are_chainable
+ assert chars('').insert(0, '').kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').rjust(1).kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').ljust(1).kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').center(1).kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').rstrip.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').lstrip.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').strip.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').reverse.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars(' ').slice(0).kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').upcase.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').downcase.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').capitalize.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').normalize.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').decompose.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').compose.kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').tidy_bytes.kind_of?(ActiveSupport::Multibyte.proxy_class)
end
- protected
+ def test_should_be_equal_to_the_wrapped_string
+ assert_equal UNICODE_STRING, @chars
+ assert_equal @chars, UNICODE_STRING
+ end
- def with_kcode(kcode)
- old_kcode, $KCODE = $KCODE, kcode
- begin
- yield
- ensure
- $KCODE = old_kcode
+ def test_should_not_be_equal_to_an_other_string
+ assert_not_equal @chars, 'other'
+ assert_not_equal 'other', @chars
+ end
+
+ def test_sortability
+ words = %w(builder armor zebra).map(&:mb_chars).sort
+ assert_equal %w(armor builder zebra), words
+ end
+
+ def test_should_return_character_offset_for_regexp_matches
+ assert_nil(@chars =~ /wrong/u)
+ assert_equal 0, (@chars =~ /こ/u)
+ assert_equal 1, (@chars =~ /に/u)
+ assert_equal 3, (@chars =~ /わ/u)
+ end
+
+ def test_should_use_character_offsets_for_insert_offsets
+ assert_equal '', ''.mb_chars.insert(0, '')
+ assert_equal 'こわにちわ', @chars.insert(1, 'わ')
+ assert_equal 'こわわわにちわ', @chars.insert(2, 'わわ')
+ assert_equal 'わこわわわにちわ', @chars.insert(0, 'わ')
+ assert_equal 'わこわわわにちわ', @chars.wrapped_string if RUBY_VERSION < '1.9'
+ end
+
+ def test_insert_should_be_destructive
+ @chars.insert(1, 'わ')
+ assert_equal 'こわにちわ', @chars
+ end
+
+ def test_insert_throws_index_error
+ assert_raises(IndexError) { @chars.insert(-12, 'わ')}
+ assert_raises(IndexError) { @chars.insert(12, 'わ') }
+ end
+
+ def test_should_know_if_one_includes_the_other
+ assert @chars.include?('')
+ assert @chars.include?('ち')
+ assert @chars.include?('わ')
+ assert !@chars.include?('こちわ')
+ assert !@chars.include?('a')
+ end
+
+ def test_include_raises_type_error_when_nil_is_passed
+ assert_raises(TypeError) do
+ @chars.include?(nil)
end
end
+
+ def test_index_should_return_character_offset
+ assert_nil @chars.index('u')
+ assert_equal 0, @chars.index('こに')
+ assert_equal 2, @chars.index('ち')
+ assert_equal 3, @chars.index('わ')
+ end
+
+ def test_indexed_insert_should_take_character_offsets
+ @chars[2] = 'a'
+ assert_equal 'こにaわ', @chars
+ @chars[2] = 'ηη'
+ assert_equal 'こにηηわ', @chars
+ @chars[3, 2] = 'λλλ'
+ assert_equal 'こにηλλλ', @chars
+ @chars[1, 0] = "λ"
+ assert_equal 'こλにηλλλ', @chars
+ @chars[4..6] = "ηη"
+ assert_equal 'こλにηηη', @chars
+ @chars[/ηη/] = "λλλ"
+ assert_equal 'こλにλλλη', @chars
+ @chars[/(λλ)(.)/, 2] = "α"
+ assert_equal 'こλにλλαη', @chars
+ @chars["α"] = "¢"
+ assert_equal 'こλにλλ¢η', @chars
+ @chars["λλ"] = "ααα"
+ assert_equal 'こλにααα¢η', @chars
+ end
+
+ def test_indexed_insert_should_raise_on_index_overflow
+ before = @chars.to_s
+ assert_raises(IndexError) { @chars[10] = 'a' }
+ assert_raises(IndexError) { @chars[10, 4] = 'a' }
+ assert_raises(IndexError) { @chars[/ii/] = 'a' }
+ assert_raises(IndexError) { @chars[/()/, 10] = 'a' }
+ assert_equal before, @chars
+ end
+
+ def test_indexed_insert_should_raise_on_range_overflow
+ before = @chars.to_s
+ assert_raises(RangeError) { @chars[10..12] = 'a' }
+ assert_equal before, @chars
+ end
+
+ def test_rjust_should_raise_argument_errors_on_bad_arguments
+ assert_raises(ArgumentError) { @chars.rjust(10, '') }
+ assert_raises(ArgumentError) { @chars.rjust }
+ end
+
+ def test_rjust_should_count_characters_instead_of_bytes
+ assert_equal UNICODE_STRING, @chars.rjust(-3)
+ assert_equal UNICODE_STRING, @chars.rjust(0)
+ assert_equal UNICODE_STRING, @chars.rjust(4)
+ assert_equal " #{UNICODE_STRING}", @chars.rjust(5)
+ assert_equal " #{UNICODE_STRING}", @chars.rjust(7)
+ assert_equal "---#{UNICODE_STRING}", @chars.rjust(7, '-')
+ assert_equal "ααα#{UNICODE_STRING}", @chars.rjust(7, 'α')
+ assert_equal "aba#{UNICODE_STRING}", @chars.rjust(7, 'ab')
+ assert_equal "αηα#{UNICODE_STRING}", @chars.rjust(7, 'αη')
+ assert_equal "αηαη#{UNICODE_STRING}", @chars.rjust(8, 'αη')
+ end
+
+ def test_ljust_should_raise_argument_errors_on_bad_arguments
+ assert_raises(ArgumentError) { @chars.ljust(10, '') }
+ assert_raises(ArgumentError) { @chars.ljust }
+ end
+
+ def test_ljust_should_count_characters_instead_of_bytes
+ assert_equal UNICODE_STRING, @chars.ljust(-3)
+ assert_equal UNICODE_STRING, @chars.ljust(0)
+ assert_equal UNICODE_STRING, @chars.ljust(4)
+ assert_equal "#{UNICODE_STRING} ", @chars.ljust(5)
+ assert_equal "#{UNICODE_STRING} ", @chars.ljust(7)
+ assert_equal "#{UNICODE_STRING}---", @chars.ljust(7, '-')
+ assert_equal "#{UNICODE_STRING}ααα", @chars.ljust(7, 'α')
+ assert_equal "#{UNICODE_STRING}aba", @chars.ljust(7, 'ab')
+ assert_equal "#{UNICODE_STRING}αηα", @chars.ljust(7, 'αη')
+ assert_equal "#{UNICODE_STRING}αηαη", @chars.ljust(8, 'αη')
+ end
+
+ def test_center_should_raise_argument_errors_on_bad_arguments
+ assert_raises(ArgumentError) { @chars.center(10, '') }
+ assert_raises(ArgumentError) { @chars.center }
+ end
+
+ def test_center_should_count_charactes_instead_of_bytes
+ assert_equal UNICODE_STRING, @chars.center(-3)
+ assert_equal UNICODE_STRING, @chars.center(0)
+ assert_equal UNICODE_STRING, @chars.center(4)
+ assert_equal "#{UNICODE_STRING} ", @chars.center(5)
+ assert_equal " #{UNICODE_STRING} ", @chars.center(6)
+ assert_equal " #{UNICODE_STRING} ", @chars.center(7)
+ assert_equal "--#{UNICODE_STRING}--", @chars.center(8, '-')
+ assert_equal "--#{UNICODE_STRING}---", @chars.center(9, '-')
+ assert_equal "αα#{UNICODE_STRING}αα", @chars.center(8, 'α')
+ assert_equal "αα#{UNICODE_STRING}ααα", @chars.center(9, 'α')
+ assert_equal "a#{UNICODE_STRING}ab", @chars.center(7, 'ab')
+ assert_equal "ab#{UNICODE_STRING}ab", @chars.center(8, 'ab')
+ assert_equal "abab#{UNICODE_STRING}abab", @chars.center(12, 'ab')
+ assert_equal "α#{UNICODE_STRING}αη", @chars.center(7, 'αη')
+ assert_equal "αη#{UNICODE_STRING}αη", @chars.center(8, 'αη')
+ end
+
+ def test_lstrip_strips_whitespace_from_the_left_of_the_string
+ assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.lstrip
+ assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING).mb_chars.lstrip
+ assert_equal UNICODE_STRING + @whitespace, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.lstrip
+ end
+
+ def test_rstrip_strips_whitespace_from_the_right_of_the_string
+ assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.rstrip
+ assert_equal UNICODE_STRING, (UNICODE_STRING + @whitespace).mb_chars.rstrip
+ assert_equal @whitespace + UNICODE_STRING, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.rstrip
+ end
+
+ def test_strip_strips_whitespace
+ assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.strip
+ assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING).mb_chars.strip
+ assert_equal UNICODE_STRING, (UNICODE_STRING + @whitespace).mb_chars.strip
+ assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.strip
+ end
+
+ def test_stripping_whitespace_leaves_whitespace_within_the_string_intact
+ string_with_whitespace = UNICODE_STRING + @whitespace + UNICODE_STRING
+ assert_equal string_with_whitespace, string_with_whitespace.mb_chars.strip
+ assert_equal string_with_whitespace, string_with_whitespace.mb_chars.lstrip
+ assert_equal string_with_whitespace, string_with_whitespace.mb_chars.rstrip
+ end
+
+ def test_size_returns_characters_instead_of_bytes
+ assert_equal 0, ''.mb_chars.size
+ assert_equal 4, @chars.size
+ assert_equal 4, @chars.length
+ assert_equal 5, ASCII_STRING.mb_chars.size
+ end
+
+ def test_reverse_reverses_characters
+ assert_equal '', ''.mb_chars.reverse
+ assert_equal 'わちにこ', @chars.reverse
+ end
+
+ def test_slice_should_take_character_offsets
+ assert_equal nil, ''.mb_chars.slice(0)
+ assert_equal 'こ', @chars.slice(0)
+ assert_equal 'わ', @chars.slice(3)
+ assert_equal nil, ''.mb_chars.slice(-1..1)
+ assert_equal '', ''.mb_chars.slice(0..10)
+ assert_equal 'にちわ', @chars.slice(1..3)
+ assert_equal 'にちわ', @chars.slice(1, 3)
+ assert_equal 'こ', @chars.slice(0, 1)
+ assert_equal 'ちわ', @chars.slice(2..10)
+ assert_equal '', @chars.slice(4..10)
+ assert_equal 'に', @chars.slice(/に/u)
+ assert_equal 'にち', @chars.slice(/に\w/u)
+ assert_equal nil, @chars.slice(/unknown/u)
+ assert_equal 'にち', @chars.slice(/(にち)/u, 1)
+ assert_equal nil, @chars.slice(/(にち)/u, 2)
+ assert_equal nil, @chars.slice(7..6)
+ end
+
+ def test_slice_should_throw_exceptions_on_invalid_arguments
+ assert_raise(TypeError) { @chars.slice(2..3, 1) }
+ assert_raise(TypeError) { @chars.slice(1, 2..3) }
+ assert_raise(ArgumentError) { @chars.slice(1, 1, 1) }
+ end
+
+ def test_upcase_should_upcase_ascii_characters
+ assert_equal '', ''.mb_chars.upcase
+ assert_equal 'ABC', 'aBc'.mb_chars.upcase
+ end
+
+ def test_downcase_should_downcase_ascii_characters
+ assert_equal '', ''.mb_chars.downcase
+ assert_equal 'abc', 'aBc'.mb_chars.downcase
+ end
+
+ def test_capitalize_should_work_on_ascii_characters
+ assert_equal '', ''.mb_chars.capitalize
+ assert_equal 'Abc', 'abc'.mb_chars.capitalize
+ end
+
+ def test_respond_to_knows_which_methods_the_proxy_responds_to
+ assert ''.mb_chars.respond_to?(:slice) # Defined on Chars
+ assert ''.mb_chars.respond_to?(:capitalize!) # Defined on Chars
+ assert ''.mb_chars.respond_to?(:gsub) # Defined on String
+ assert !''.mb_chars.respond_to?(:undefined_method) # Not defined
+ end
+
+ def test_acts_like_string
+ assert 'Bambi'.mb_chars.acts_like_string?
+ end
end
+# The default Multibyte Chars proxy has more features than the normal string implementation. Tests
+# for the implementation of these features should run on all Ruby versions and shouldn't be tested
+# through the proxy methods.
+class MultibyteCharsExtrasTest < Test::Unit::TestCase
+ include MultibyteTestHelpers
+
+ if RUBY_VERSION >= '1.9'
+ def test_tidy_bytes_is_broken_on_1_9_0
+ assert_raises(ArgumentError) do
+ assert_equal_codepoints [0xfffd].pack('U'), chars("\xef\xbf\xbd").tidy_bytes
+ end
+ end
+ end
+
+ def test_upcase_should_be_unicode_aware
+ assert_equal "АБВГД\0F", chars("аБвгд\0f").upcase
+ assert_equal 'こにちわ', chars('こにちわ').upcase
+ end
+
+ def test_downcase_should_be_unicode_aware
+ assert_equal "абвгд\0f", chars("аБвгд\0f").downcase
+ assert_equal 'こにちわ', chars('こにちわ').downcase
+ end
+
+ def test_capitalize_should_be_unicode_aware
+ { 'аБвг аБвг' => 'Абвг абвг',
+ 'аБвг АБВГ' => 'Абвг абвг',
+ 'АБВГ АБВГ' => 'Абвг абвг',
+ '' => '' }.each do |f,t|
+ assert_equal t, chars(f).capitalize
+ end
+ end
+
+ def test_composition_exclusion_is_set_up_properly
+ # Normalization of DEVANAGARI LETTER QA breaks when composition exclusion isn't used correctly
+ qa = [0x915, 0x93c].pack('U*')
+ assert_equal qa, chars(qa).normalize(:c)
+ end
+
+ # Test for the Public Review Issue #29, bad explanation of composition might lead to a
+ # bad implementation: http://www.unicode.org/review/pr-29.html
+ def test_normalization_C_pri_29
+ [
+ [0x0B47, 0x0300, 0x0B3E],
+ [0x1100, 0x0300, 0x1161]
+ ].map { |c| c.pack('U*') }.each do |c|
+ assert_equal_codepoints c, chars(c).normalize(:c)
+ end
+ end
+
+ def test_normalization_shouldnt_strip_null_bytes
+ null_byte_str = "Test\0test"
+
+ assert_equal null_byte_str, chars(null_byte_str).normalize(:kc)
+ assert_equal null_byte_str, chars(null_byte_str).normalize(:c)
+ assert_equal null_byte_str, chars(null_byte_str).normalize(:d)
+ assert_equal null_byte_str, chars(null_byte_str).normalize(:kd)
+ assert_equal null_byte_str, chars(null_byte_str).decompose
+ assert_equal null_byte_str, chars(null_byte_str).compose
+ end
+
+ def test_simple_normalization
+ comp_str = [
+ 44, # LATIN CAPITAL LETTER D
+ 307, # COMBINING DOT ABOVE
+ 328, # COMBINING OGONEK
+ 323 # COMBINING DOT BELOW
+ ].pack("U*")
+
+ assert_equal_codepoints '', chars('').normalize
+ assert_equal_codepoints [44,105,106,328,323].pack("U*"), chars(comp_str).normalize(:kc).to_s
+ assert_equal_codepoints [44,307,328,323].pack("U*"), chars(comp_str).normalize(:c).to_s
+ assert_equal_codepoints [44,307,110,780,78,769].pack("U*"), chars(comp_str).normalize(:d).to_s
+ assert_equal_codepoints [44,105,106,110,780,78,769].pack("U*"), chars(comp_str).normalize(:kd).to_s
+ end
+
+ def test_should_compute_grapheme_length
+ [
+ ['', 0],
+ ['abc', 3],
+ ['こにちわ', 4],
+ [[0x0924, 0x094D, 0x0930].pack('U*'), 2],
+ [%w(cr lf), 1],
+ [%w(l l), 1],
+ [%w(l v), 1],
+ [%w(l lv), 1],
+ [%w(l lvt), 1],
+ [%w(lv v), 1],
+ [%w(lv t), 1],
+ [%w(v v), 1],
+ [%w(v t), 1],
+ [%w(lvt t), 1],
+ [%w(t t), 1],
+ [%w(n extend), 1],
+ [%w(n n), 2],
+ [%w(n cr lf n), 3],
+ [%w(n l v t), 2]
+ ].each do |input, expected_length|
+ if input.kind_of?(Array)
+ str = string_from_classes(input)
+ else
+ str = input
+ end
+ assert_equal expected_length, chars(str).g_length
+ end
+ end
+
+ def test_tidy_bytes_should_tidy_bytes
+ byte_string = "\270\236\010\210\245"
+ tidy_string = [0xb8, 0x17e, 0x8, 0x2c6, 0xa5].pack('U*')
+ ascii_padding = 'aa'
+ utf8_padding = 'éé'
+
+ assert_equal_codepoints tidy_string, chars(byte_string).tidy_bytes
+
+ assert_equal_codepoints ascii_padding.dup.insert(1, tidy_string),
+ chars(ascii_padding.dup.insert(1, byte_string)).tidy_bytes
+ assert_equal_codepoints utf8_padding.dup.insert(2, tidy_string),
+ chars(utf8_padding.dup.insert(2, byte_string)).tidy_bytes
+ assert_nothing_raised { chars(byte_string).tidy_bytes.to_s.unpack('U*') }
+
+ assert_equal_codepoints "\xC3\xA7", chars("\xE7").tidy_bytes # iso_8859_1: small c cedilla
+ assert_equal_codepoints "\xE2\x80\x9C", chars("\x93").tidy_bytes # win_1252: left smart quote
+ assert_equal_codepoints "\xE2\x82\xAC", chars("\x80").tidy_bytes # win_1252: euro
+ assert_equal_codepoints "\x00", chars("\x00").tidy_bytes # null char
+ assert_equal_codepoints [0xfffd].pack('U'), chars("\xef\xbf\xbd").tidy_bytes # invalid char
+ rescue ArgumentError => e
+ raise e if RUBY_VERSION < '1.9'
+ end
+
+ private
+
+ def string_from_classes(classes)
+ # Characters from the character classes as described in UAX #29
+ character_from_class = {
+ :l => 0x1100, :v => 0x1160, :t => 0x11A8, :lv => 0xAC00, :lvt => 0xAC01, :cr => 0x000D, :lf => 0x000A,
+ :extend => 0x094D, :n => 0x64
+ }
+ classes.collect do |k|
+ character_from_class[k.intern]
+ end.pack('U*')
+ end
end
diff --git a/activesupport/test/multibyte_conformance.rb b/activesupport/test/multibyte_conformance.rb
index 05fb9ef7a7..caae4791b4 100644
--- a/activesupport/test/multibyte_conformance.rb
+++ b/activesupport/test/multibyte_conformance.rb
@@ -1,13 +1,11 @@
-require 'abstract_unit'
-require 'open-uri'
-
-if RUBY_VERSION < '1.9'
+# encoding: utf-8
-$KCODE = 'UTF8'
+require 'abstract_unit'
+require 'multibyte_test_helpers'
-UNIDATA_URL = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::UNICODE_VERSION}/ucd"
-UNIDATA_FILE = '/NormalizationTest.txt'
-CACHE_DIR = File.dirname(__FILE__) + '/cache'
+require 'fileutils'
+require 'open-uri'
+require 'tmpdir'
class Downloader
def self.download(from, to)
@@ -27,17 +25,19 @@ class Downloader
end
end
-class String
- # Unicode Inspect returns the codepoints of the string in hex
- def ui
- "#{self} " + ("[%s]" % unpack("U*").map{|cp| cp.to_s(16) }.join(' '))
- end unless ''.respond_to?(:ui)
-end
-
-Dir.mkdir(CACHE_DIR) unless File.exist?(CACHE_DIR)
-Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE)
-
-module ConformanceTest
+class MultibyteConformanceTest < Test::Unit::TestCase
+ include MultibyteTestHelpers
+
+ UNIDATA_URL = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::UNICODE_VERSION}/ucd"
+ UNIDATA_FILE = '/NormalizationTest.txt'
+ CACHE_DIR = File.join(Dir.tmpdir, 'cache')
+
+ def setup
+ FileUtils.mkdir_p(CACHE_DIR)
+ Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE)
+ @proxy = ActiveSupport::Multibyte::Chars
+ end
+
def test_normalizations_C
each_line_of_norm_tests do |*cols|
col1, col2, col3, col4, col5, comment = *cols
@@ -47,13 +47,13 @@ module ConformanceTest
#
# NFC
# c2 == NFC(c1) == NFC(c2) == NFC(c3)
- assert_equal col2.ui, @handler.normalize(col1, :c).ui, "Form C - Col 2 has to be NFC(1) - #{comment}"
- assert_equal col2.ui, @handler.normalize(col2, :c).ui, "Form C - Col 2 has to be NFC(2) - #{comment}"
- assert_equal col2.ui, @handler.normalize(col3, :c).ui, "Form C - Col 2 has to be NFC(3) - #{comment}"
+ assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}"
+ assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}"
+ assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}"
#
# c4 == NFC(c4) == NFC(c5)
- assert_equal col4.ui, @handler.normalize(col4, :c).ui, "Form C - Col 4 has to be C(4) - #{comment}"
- assert_equal col4.ui, @handler.normalize(col5, :c).ui, "Form C - Col 4 has to be C(5) - #{comment}"
+ assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}"
+ assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}"
end
end
@@ -63,12 +63,12 @@ module ConformanceTest
#
# NFD
# c3 == NFD(c1) == NFD(c2) == NFD(c3)
- assert_equal col3.ui, @handler.normalize(col1, :d).ui, "Form D - Col 3 has to be NFD(1) - #{comment}"
- assert_equal col3.ui, @handler.normalize(col2, :d).ui, "Form D - Col 3 has to be NFD(2) - #{comment}"
- assert_equal col3.ui, @handler.normalize(col3, :d).ui, "Form D - Col 3 has to be NFD(3) - #{comment}"
+ assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}"
+ assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}"
+ assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}"
# c5 == NFD(c4) == NFD(c5)
- assert_equal col5.ui, @handler.normalize(col4, :d).ui, "Form D - Col 5 has to be NFD(4) - #{comment}"
- assert_equal col5.ui, @handler.normalize(col5, :d).ui, "Form D - Col 5 has to be NFD(5) - #{comment}"
+ assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}"
+ assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}"
end
end
@@ -78,11 +78,11 @@ module ConformanceTest
#
# NFKC
# c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
- assert_equal col4.ui, @handler.normalize(col1, :kc).ui, "Form D - Col 4 has to be NFKC(1) - #{comment}"
- assert_equal col4.ui, @handler.normalize(col2, :kc).ui, "Form D - Col 4 has to be NFKC(2) - #{comment}"
- assert_equal col4.ui, @handler.normalize(col3, :kc).ui, "Form D - Col 4 has to be NFKC(3) - #{comment}"
- assert_equal col4.ui, @handler.normalize(col4, :kc).ui, "Form D - Col 4 has to be NFKC(4) - #{comment}"
- assert_equal col4.ui, @handler.normalize(col5, :kc).ui, "Form D - Col 4 has to be NFKC(5) - #{comment}"
+ assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}"
+ assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}"
+ assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}"
+ assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}"
+ assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}"
end
end
@@ -92,11 +92,11 @@ module ConformanceTest
#
# NFKD
# c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
- assert_equal col5.ui, @handler.normalize(col1, :kd).ui, "Form KD - Col 5 has to be NFKD(1) - #{comment}"
- assert_equal col5.ui, @handler.normalize(col2, :kd).ui, "Form KD - Col 5 has to be NFKD(2) - #{comment}"
- assert_equal col5.ui, @handler.normalize(col3, :kd).ui, "Form KD - Col 5 has to be NFKD(3) - #{comment}"
- assert_equal col5.ui, @handler.normalize(col4, :kd).ui, "Form KD - Col 5 has to be NFKD(4) - #{comment}"
- assert_equal col5.ui, @handler.normalize(col5, :kd).ui, "Form KD - Col 5 has to be NFKD(5) - #{comment}"
+ assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}"
+ assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}"
+ assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}"
+ assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}"
+ assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}"
end
end
@@ -104,7 +104,7 @@ module ConformanceTest
def each_line_of_norm_tests(&block)
lines = 0
max_test_lines = 0 # Don't limit below 38, because that's the header of the testfile
- File.open(File.dirname(__FILE__) + '/cache' + UNIDATA_FILE, 'r') do | f |
+ File.open(File.join(CACHE_DIR, UNIDATA_FILE), 'r') do | f |
until f.eof? || (max_test_lines > 38 and lines > max_test_lines)
lines += 1
line = f.gets.chomp!
@@ -122,25 +122,8 @@ module ConformanceTest
end
end
end
-end
-
-begin
- require_library_or_gem('utf8proc_native')
- require 'active_record/multibyte/handlers/utf8_handler_proc'
- class ConformanceTestProc < Test::Unit::TestCase
- include ConformanceTest
- def setup
- @handler = ::ActiveSupport::Multibyte::Handlers::UTF8HandlerProc
+
+ def inspect_codepoints(str)
+ str.to_s.unpack("U*").map{|cp| cp.to_s(16) }.join(' ')
end
- end
-rescue LoadError
-end
-
-class ConformanceTestPure < Test::Unit::TestCase
- include ConformanceTest
- def setup
- @handler = ::ActiveSupport::Multibyte::Handlers::UTF8Handler
- end
-end
-
-end
+end \ No newline at end of file
diff --git a/activesupport/test/multibyte_handler_test.rb b/activesupport/test/multibyte_handler_test.rb
deleted file mode 100644
index 5575ecc32d..0000000000
--- a/activesupport/test/multibyte_handler_test.rb
+++ /dev/null
@@ -1,372 +0,0 @@
-# encoding: utf-8
-require 'abstract_unit'
-
-if RUBY_VERSION < '1.9'
-
-$KCODE = 'UTF8'
-
-class String
- # Unicode Inspect returns the codepoints of the string in hex
- def ui
- "#{self} " + ("[%s]" % unpack("U*").map{|cp| cp.to_s(16) }.join(' '))
- end unless ''.respond_to?(:ui)
-end
-
-module UTF8HandlingTest
-
- def common_setup
- # This is an ASCII string with some russian strings and a ligature. It's nicely calibrated, because
- # slicing it at some specific bytes will kill your characters if you use standard Ruby routines.
- # It has both capital and standard letters, so that we can test case conversions easily.
- # It has 26 characters and 28 when the ligature gets split during normalization.
- @string = "Abcd Блå ffi бла бла бла бла"
- @string_kd = "Abcd Блå ffi бла бла бла бла"
- @string_kc = "Abcd Блå ffi бла бла бла бла"
- @string_c = "Abcd Блå ffi бла бла бла бла"
- @string_d = "Abcd Блå ffi бла бла бла бла"
- @bytestring = "\270\236\010\210\245" # Not UTF-8
-
- # Characters from the character classes as described in UAX #29
- @character_from_class = {
- :l => 0x1100, :v => 0x1160, :t => 0x11A8, :lv => 0xAC00, :lvt => 0xAC01, :cr => 0x000D, :lf => 0x000A,
- :extend => 0x094D, :n => 0x64
- }
- end
-
- def test_utf8_recognition
- assert ActiveSupport::Multibyte::Handlers::UTF8Handler.consumes?(@string),
- "Should recognize as a valid UTF-8 string"
- assert !ActiveSupport::Multibyte::Handlers::UTF8Handler.consumes?(@bytestring), "This is bytestring, not UTF-8"
- end
-
- def test_simple_normalization
- # Normalization of DEVANAGARI LETTER QA breaks when composition exclusion isn't used correctly
- assert_equal [0x915, 0x93c].pack('U*').ui, [0x915, 0x93c].pack('U*').chars.normalize(:c).to_s.ui
-
- null_byte_str = "Test\0test"
-
- assert_equal '', @handler.normalize(''), "Empty string should not break things"
- assert_equal null_byte_str.ui, @handler.normalize(null_byte_str, :kc).ui, "Null byte should remain"
- assert_equal null_byte_str.ui, @handler.normalize(null_byte_str, :c).ui, "Null byte should remain"
- assert_equal null_byte_str.ui, @handler.normalize(null_byte_str, :d).ui, "Null byte should remain"
- assert_equal null_byte_str.ui, @handler.normalize(null_byte_str, :kd).ui, "Null byte should remain"
- assert_equal null_byte_str.ui, @handler.decompose(null_byte_str).ui, "Null byte should remain"
- assert_equal null_byte_str.ui, @handler.compose(null_byte_str).ui, "Null byte should remain"
-
- comp_str = [
- 44, # LATIN CAPITAL LETTER D
- 307, # COMBINING DOT ABOVE
- 328, # COMBINING OGONEK
- 323 # COMBINING DOT BELOW
- ].pack("U*")
- norm_str_KC = [44,105,106,328,323].pack("U*")
- norm_str_C = [44,307,328,323].pack("U*")
- norm_str_D = [44,307,110,780,78,769].pack("U*")
- norm_str_KD = [44,105,106,110,780,78,769].pack("U*")
-
- assert_equal norm_str_KC.ui, @handler.normalize(comp_str, :kc).ui, "Should normalize KC"
- assert_equal norm_str_C.ui, @handler.normalize(comp_str, :c).ui, "Should normalize C"
- assert_equal norm_str_D.ui, @handler.normalize(comp_str, :d).ui, "Should normalize D"
- assert_equal norm_str_KD.ui, @handler.normalize(comp_str, :kd).ui, "Should normalize KD"
-
- assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.normalize(@bytestring) }
- end
-
- # Test for the Public Review Issue #29, bad explanation of composition might lead to a
- # bad implementation: http://www.unicode.org/review/pr-29.html
- def test_normalization_C_pri_29
- [
- [0x0B47, 0x0300, 0x0B3E],
- [0x1100, 0x0300, 0x1161]
- ].map { |c| c.pack('U*') }.each do |c|
- assert_equal c.ui, @handler.normalize(c, :c).ui, "Composition is implemented incorrectly"
- end
- end
-
- def test_casefolding
- simple_str = "abCdef"
- simple_str_upcase = "ABCDEF"
- simple_str_downcase = "abcdef"
-
- assert_equal '', @handler.downcase(@handler.upcase('')), "Empty string should not break things"
- assert_equal simple_str_upcase, @handler.upcase(simple_str), "should upcase properly"
- assert_equal simple_str_downcase, @handler.downcase(simple_str), "should downcase properly"
- assert_equal simple_str_downcase, @handler.downcase(@handler.upcase(simple_str_downcase)), "upcase and downcase should be mirrors"
-
- rus_str = "аБвгд\0f"
- rus_str_upcase = "АБВГД\0F"
- rus_str_downcase = "абвгд\0f"
- assert_equal rus_str_upcase, @handler.upcase(rus_str), "should upcase properly honoring null-byte"
- assert_equal rus_str_downcase, @handler.downcase(rus_str), "should downcase properly honoring null-byte"
-
- jap_str = "の埋め込み化対応はほぼ完成"
- assert_equal jap_str, @handler.upcase(jap_str), "Japanse has no upcase, should remain unchanged"
- assert_equal jap_str, @handler.downcase(jap_str), "Japanse has no downcase, should remain unchanged"
-
- assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.upcase(@bytestring) }
- end
-
- def test_capitalize
- { 'аБвг аБвг' => 'Абвг абвг',
- 'аБвг АБВГ' => 'Абвг абвг',
- 'АБВГ АБВГ' => 'Абвг абвг',
- '' => '' }.each do |f,t|
- assert_equal t, @handler.capitalize(f), "Capitalize should work as expected"
- end
- assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.capitalize(@bytestring) }
- end
-
- def test_translate_offset
- str = "Блaå" # [2, 2, 1, 2] bytes
- assert_equal 0, @handler.translate_offset('', 0), "Offset for an empty string makes no sense, return 0"
- assert_equal 0, @handler.translate_offset(str, 0), "First character, first byte"
- assert_equal 0, @handler.translate_offset(str, 1), "First character, second byte"
- assert_equal 1, @handler.translate_offset(str, 2), "Second character, third byte"
- assert_equal 1, @handler.translate_offset(str, 3), "Second character, fourth byte"
- assert_equal 2, @handler.translate_offset(str, 4), "Third character, fifth byte"
- assert_equal 3, @handler.translate_offset(str, 5), "Fourth character, sixth byte"
- assert_equal 3, @handler.translate_offset(str, 6), "Fourth character, seventh byte"
- assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.translate_offset(@bytestring, 3) }
- end
-
- def test_insert
- assert_equal '', @handler.insert('', 0, ''), "Empty string should not break things"
- assert_equal "Abcd Блå ffiБУМ бла бла бла бла", @handler.insert(@string, 10, "БУМ"),
- "Text should be inserted at right codepoints"
- assert_equal "Abcd Блå ffiБУМ бла бла бла бла", @string, "Insert should be destructive"
- assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) do
- @handler.insert(@bytestring, 2, "\210")
- end
- end
-
- def test_reverse
- str = "wБлåa \n"
- rev = "\n aåлБw"
- assert_equal '', @handler.reverse(''), "Empty string shouldn't change"
- assert_equal rev.ui, @handler.reverse(str).ui, "Should reverse properly"
- assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.reverse(@bytestring) }
- end
-
- def test_size
- assert_equal 0, @handler.size(''), "Empty string has size 0"
- assert_equal 26, @handler.size(@string), "String length should be 26"
- assert_equal 26, @handler.length(@string), "String length method should be properly aliased"
- assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.size(@bytestring) }
- end
-
- def test_slice
- assert_equal 0x41, @handler.slice(@string, 0), "Singular characters should return codepoints"
- assert_equal 0xE5, @handler.slice(@string, 7), "Singular characters should return codepoints"
- assert_equal nil, @handler.slice('', -1..1), "Broken range should return nil"
- assert_equal '', @handler.slice('', 0..10), "Empty string should not break things"
- assert_equal "d Блå ffi", @handler.slice(@string, 3..9), "Unicode characters have to be returned"
- assert_equal "d Блå ffi", @handler.slice(@string, 3, 7), "Unicode characters have to be returned"
- assert_equal "A", @handler.slice(@string, 0, 1), "Slicing from an offset should return characters"
- assert_equal " Блå ffi ", @handler.slice(@string, 4..10), "Unicode characters have to be returned"
- assert_equal "ffi бла", @handler.slice(@string, /ffi бла/u), "Slicing on Regexps should be supported"
- assert_equal "ffi бла", @handler.slice(@string, /ffi \w\wа/u), "Slicing on Regexps should be supported"
- assert_equal nil, @handler.slice(@string, /unknown/u), "Slicing on Regexps with no match should return nil"
- assert_equal "ffi бла", @handler.slice(@string, /(ffi бла)/u,1), "Slicing on Regexps with a match group should be supported"
- assert_equal nil, @handler.slice(@string, /(ffi)/u,2), "Slicing with a Regexp and asking for an invalid match group should return nil"
- assert_equal "", @handler.slice(@string, 7..6), "Range is empty, should return an empty string"
- assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.slice(@bytestring, 2..3) }
- assert_raise(TypeError, "With 2 args, should raise TypeError for non-Numeric or Regexp first argument") { @handler.slice(@string, 2..3, 1) }
- assert_raise(TypeError, "With 2 args, should raise TypeError for non-Numeric or Regexp second argument") { @handler.slice(@string, 1, 2..3) }
- assert_raise(ArgumentError, "Should raise ArgumentError when there are more than 2 args") { @handler.slice(@string, 1, 1, 1) }
- end
-
- def test_grapheme_cluster_length
- assert_equal 0, @handler.g_length(''), "String should count 0 grapheme clusters"
- assert_equal 2, @handler.g_length([0x0924, 0x094D, 0x0930].pack('U*')), "String should count 2 grapheme clusters"
- assert_equal 1, @handler.g_length(string_from_classes(%w(cr lf))), "Don't cut between CR and LF"
- assert_equal 1, @handler.g_length(string_from_classes(%w(l l))), "Don't cut between L"
- assert_equal 1, @handler.g_length(string_from_classes(%w(l v))), "Don't cut between L and V"
- assert_equal 1, @handler.g_length(string_from_classes(%w(l lv))), "Don't cut between L and LV"
- assert_equal 1, @handler.g_length(string_from_classes(%w(l lvt))), "Don't cut between L and LVT"
- assert_equal 1, @handler.g_length(string_from_classes(%w(lv v))), "Don't cut between LV and V"
- assert_equal 1, @handler.g_length(string_from_classes(%w(lv t))), "Don't cut between LV and T"
- assert_equal 1, @handler.g_length(string_from_classes(%w(v v))), "Don't cut between V and V"
- assert_equal 1, @handler.g_length(string_from_classes(%w(v t))), "Don't cut between V and T"
- assert_equal 1, @handler.g_length(string_from_classes(%w(lvt t))), "Don't cut between LVT and T"
- assert_equal 1, @handler.g_length(string_from_classes(%w(t t))), "Don't cut between T and T"
- assert_equal 1, @handler.g_length(string_from_classes(%w(n extend))), "Don't cut before Extend"
- assert_equal 2, @handler.g_length(string_from_classes(%w(n n))), "Cut between normal characters"
- assert_equal 3, @handler.g_length(string_from_classes(%w(n cr lf n))), "Don't cut between CR and LF"
- assert_equal 2, @handler.g_length(string_from_classes(%w(n l v t))), "Don't cut between L, V and T"
- assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.g_length(@bytestring) }
- end
-
- def test_index
- s = "Καλημέρα κόσμε!"
- assert_equal 0, @handler.index('', ''), "The empty string is always found at the beginning of the string"
- assert_equal 0, @handler.index('haystack', ''), "The empty string is always found at the beginning of the string"
- assert_equal 0, @handler.index(s, 'Κ'), "Greek K is at 0"
- assert_equal 1, @handler.index(s, 'α'), "Greek Alpha is at 1"
-
- assert_equal nil, @handler.index(@bytestring, 'a')
- assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.index(@bytestring, "\010") }
- end
-
- def test_indexed_insert
- s = "Καλη!"
- @handler[s, 2] = "a"
- assert_equal "Καaη!", s
- @handler[s, 2] = "ηη"
- assert_equal "Καηηη!", s
- assert_raises(IndexError) { @handler[s, 10] = 'a' }
- assert_equal "Καηηη!", s
- @handler[s, 2] = 32
- assert_equal "Κα ηη!", s
- @handler[s, 3, 2] = "λλλ"
- assert_equal "Κα λλλ!", s
- @handler[s, 1, 0] = "λ"
- assert_equal "Κλα λλλ!", s
- assert_raises(IndexError) { @handler[s, 10, 4] = 'a' }
- assert_equal "Κλα λλλ!", s
- @handler[s, 4..6] = "ηη"
- assert_equal "Κλα ηη!", s
- assert_raises(RangeError) { @handler[s, 10..12] = 'a' }
- assert_equal "Κλα ηη!", s
- @handler[s, /ηη/] = "λλλ"
- assert_equal "Κλα λλλ!", s
- assert_raises(IndexError) { @handler[s, /ii/] = 'a' }
- assert_equal "Κλα λλλ!", s
- @handler[s, /(λλ)(.)/, 2] = "α"
- assert_equal "Κλα λλα!", s
- assert_raises(IndexError) { @handler[s, /()/, 10] = 'a' }
- assert_equal "Κλα λλα!", s
- @handler[s, "α"] = "η"
- assert_equal "Κλη λλα!", s
- @handler[s, "λλ"] = "ααα"
- assert_equal "Κλη αααα!", s
- end
-
- def test_rjust
- s = "Καη"
- assert_raises(ArgumentError) { @handler.rjust(s, 10, '') }
- assert_raises(ArgumentError) { @handler.rjust(s) }
- assert_equal "Καη", @handler.rjust(s, -3)
- assert_equal "Καη", @handler.rjust(s, 0)
- assert_equal "Καη", @handler.rjust(s, 3)
- assert_equal " Καη", @handler.rjust(s, 5)
- assert_equal " Καη", @handler.rjust(s, 7)
- assert_equal "----Καη", @handler.rjust(s, 7, '-')
- assert_equal "ααααΚαη", @handler.rjust(s, 7, 'α')
- assert_equal "abaΚαη", @handler.rjust(s, 6, 'ab')
- assert_equal "αηαΚαη", @handler.rjust(s, 6, 'αη')
- end
-
- def test_ljust
- s = "Καη"
- assert_raises(ArgumentError) { @handler.ljust(s, 10, '') }
- assert_raises(ArgumentError) { @handler.ljust(s) }
- assert_equal "Καη", @handler.ljust(s, -3)
- assert_equal "Καη", @handler.ljust(s, 0)
- assert_equal "Καη", @handler.ljust(s, 3)
- assert_equal "Καη ", @handler.ljust(s, 5)
- assert_equal "Καη ", @handler.ljust(s, 7)
- assert_equal "Καη----", @handler.ljust(s, 7, '-')
- assert_equal "Καηαααα", @handler.ljust(s, 7, 'α')
- assert_equal "Καηaba", @handler.ljust(s, 6, 'ab')
- assert_equal "Καηαηα", @handler.ljust(s, 6, 'αη')
- end
-
- def test_center
- s = "Καη"
- assert_raises(ArgumentError) { @handler.center(s, 10, '') }
- assert_raises(ArgumentError) { @handler.center(s) }
- assert_equal "Καη", @handler.center(s, -3)
- assert_equal "Καη", @handler.center(s, 0)
- assert_equal "Καη", @handler.center(s, 3)
- assert_equal "Καη ", @handler.center(s, 4)
- assert_equal " Καη ", @handler.center(s, 5)
- assert_equal " Καη ", @handler.center(s, 6)
- assert_equal "--Καη--", @handler.center(s, 7, '-')
- assert_equal "--Καη---", @handler.center(s, 8, '-')
- assert_equal "ααΚαηαα", @handler.center(s, 7, 'α')
- assert_equal "ααΚαηααα", @handler.center(s, 8, 'α')
- assert_equal "aΚαηab", @handler.center(s, 6, 'ab')
- assert_equal "abΚαηab", @handler.center(s, 7, 'ab')
- assert_equal "ababΚαηabab", @handler.center(s, 11, 'ab')
- assert_equal "αΚαηαη", @handler.center(s, 6, 'αη')
- assert_equal "αηΚαηαη", @handler.center(s, 7, 'αη')
- end
-
- def test_strip
- # A unicode aware version of strip should strip all 26 types of whitespace. This includes the NO BREAK SPACE
- # aka BOM (byte order mark). The byte order mark has no place in UTF-8 because it's used to detect LE and BE.
- b = "\n" + [
- 32, # SPACE
- 8195, # EM SPACE
- 8199, # FIGURE SPACE,
- 8201, # THIN SPACE
- 8202, # HAIR SPACE
- 65279, # NO BREAK SPACE (ZW)
- ].pack('U*')
- m = "word блин\n\n\n word"
- e = [
- 65279, # NO BREAK SPACE (ZW)
- 8201, # THIN SPACE
- 8199, # FIGURE SPACE,
- 32, # SPACE
- ].pack('U*')
- string = b+m+e
-
- assert_equal '', @handler.strip(''), "Empty string should stay empty"
- assert_equal m+e, @handler.lstrip(string), "Whitespace should be gone on the left"
- assert_equal b+m, @handler.rstrip(string), "Whitespace should be gone on the right"
- assert_equal m, @handler.strip(string), "Whitespace should be stripped on both sides"
-
- bs = "\n #{@bytestring} \n\n"
- assert_equal @bytestring, @handler.strip(bs), "Invalid unicode strings should still strip"
- end
-
- def test_tidy_bytes
- result = [0xb8, 0x17e, 0x8, 0x2c6, 0xa5].pack('U*')
- assert_equal result, @handler.tidy_bytes(@bytestring)
- assert_equal "a#{result}a", @handler.tidy_bytes('a' + @bytestring + 'a'),
- 'tidy_bytes should leave surrounding characters intact'
- assert_equal "é#{result}é", @handler.tidy_bytes('é' + @bytestring + 'é'),
- 'tidy_bytes should leave surrounding characters intact'
- assert_nothing_raised { @handler.tidy_bytes(@bytestring).unpack('U*') }
-
- assert_equal "\xC3\xA7", @handler.tidy_bytes("\xE7") # iso_8859_1: small c cedilla
- assert_equal "\xC2\xA9", @handler.tidy_bytes("\xA9") # iso_8859_1: copyright symbol
- assert_equal "\xE2\x80\x9C", @handler.tidy_bytes("\x93") # win_1252: left smart quote
- assert_equal "\xE2\x82\xAC", @handler.tidy_bytes("\x80") # win_1252: euro
- assert_equal "\x00", @handler.tidy_bytes("\x00") # null char
- assert_equal [0xfffd].pack('U'), @handler.tidy_bytes("\xef\xbf\xbd") # invalid char
- end
-
- protected
-
- def string_from_classes(classes)
- classes.collect do |k|
- @character_from_class[k.intern]
- end.pack('U*')
- end
-end
-
-
-begin
- require_library_or_gem('utf8proc_native')
- require 'active_record/multibyte/handlers/utf8_handler_proc'
- class UTF8HandlingTestProc < Test::Unit::TestCase
- include UTF8HandlingTest
- def setup
- common_setup
- @handler = ::ActiveSupport::Multibyte::Handlers::UTF8HandlerProc
- end
- end
-rescue LoadError
-end
-
-class UTF8HandlingTestPure < Test::Unit::TestCase
- include UTF8HandlingTest
- def setup
- common_setup
- @handler = ::ActiveSupport::Multibyte::Handlers::UTF8Handler
- end
-end
-
-end
diff --git a/activesupport/test/multibyte_test_helpers.rb b/activesupport/test/multibyte_test_helpers.rb
new file mode 100644
index 0000000000..597f949059
--- /dev/null
+++ b/activesupport/test/multibyte_test_helpers.rb
@@ -0,0 +1,19 @@
+# encoding: utf-8
+
+module MultibyteTestHelpers
+ UNICODE_STRING = 'こにちわ'
+ ASCII_STRING = 'ohayo'
+ BYTE_STRING = "\270\236\010\210\245"
+
+ def chars(str)
+ ActiveSupport::Multibyte::Chars.new(str)
+ end
+
+ def inspect_codepoints(str)
+ str.to_s.unpack("U*").map{|cp| cp.to_s(16) }.join(' ')
+ end
+
+ def assert_equal_codepoints(expected, actual, message=nil)
+ assert_equal(inspect_codepoints(expected), inspect_codepoints(actual), message)
+ end
+end \ No newline at end of file
diff --git a/activesupport/test/multibyte_unicode_database_test.rb b/activesupport/test/multibyte_unicode_database_test.rb
new file mode 100644
index 0000000000..fb415e08d3
--- /dev/null
+++ b/activesupport/test/multibyte_unicode_database_test.rb
@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+require 'abstract_unit'
+
+uses_mocha "MultibyteUnicodeDatabaseTest" do
+
+class MultibyteUnicodeDatabaseTest < Test::Unit::TestCase
+
+ def setup
+ @ucd = ActiveSupport::Multibyte::UnicodeDatabase.new
+ end
+
+ ActiveSupport::Multibyte::UnicodeDatabase::ATTRIBUTES.each do |attribute|
+ define_method "test_lazy_loading_on_attribute_access_of_#{attribute}" do
+ @ucd.expects(:load)
+ @ucd.send(attribute)
+ end
+ end
+
+ def test_load
+ @ucd.load
+ ActiveSupport::Multibyte::UnicodeDatabase::ATTRIBUTES.each do |attribute|
+ assert @ucd.send(attribute).length > 1
+ end
+ end
+end
+
+end \ No newline at end of file