Simplify ActiveSupport::Multibyte and make it run on Ruby 1.9.

* Unicode methods are now defined directly on Chars instead of a handler
* Updated Unicode database to Unicode 5.1.0
* Improved documentation

1 files changed, 492 insertions, 150 deletions

diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb
index a87309b038..31b8f1b760 100644
--- a/activesupport/test/multibyte_chars_test.rb
+++ b/activesupport/test/multibyte_chars_test.rb
@@ -1,186 +1,528 @@
 # encoding: utf-8
+
 require 'abstract_unit'
 
-if RUBY_VERSION < '1.9'
+module MultibyteTest
+  UNICODE_STRING = 'こにちわ'
+  ASCII_STRING = 'ohayo'
+  BYTE_STRING = "\270\236\010\210\245"
+
+  def chars(str)
+    ActiveSupport::Multibyte::Chars.new(str)
+  end
+
+  def inspect_codepoints(str)
+    str.to_s.unpack("U*").map{|cp| cp.to_s(16) }.join(' ')
+  end
+
+  def assert_equal_codepoints(expected, actual, message=nil)
+    assert_equal(inspect_codepoints(expected), inspect_codepoints(actual), message)
+  end
+end
+
+class String
+  def __string_for_multibyte_testing; self; end
+  def __string_for_multibyte_testing!; self; end
+end
 
-$KCODE = 'UTF8'
+class MultibyteCharsTest < Test::Unit::TestCase
+  include MultibyteTest
 
-class CharsTest < Test::Unit::TestCase
-  
   def setup
-    @s = {
-      :utf8 => "Abcd Блå ffi блa  埋",
-      :ascii => "asci ias c iia s",
-      :bytes => "\270\236\010\210\245"
-    }
+    @chars = ActiveSupport::Multibyte::Chars.new UNICODE_STRING
   end
-  
-  def test_sanity
-    @s.each do |t, s|
-      assert s.respond_to?(:chars), "All string should have the chars method (#{t})"
-      assert s.respond_to?(:to_s), "All string should have the to_s method (#{t})"
-      assert_kind_of ActiveSupport::Multibyte::Chars, s.chars, "#chars should return an instance of Chars (#{t})"
-    end
+
+  def test_wraps_the_original_string
+    assert_equal UNICODE_STRING, @chars.to_s
+    assert_equal UNICODE_STRING, @chars.wrapped_string
   end
-  
-  def test_comparability
-    @s.each do |t, s|
-      assert_equal s, s.chars.to_s, "Chars#to_s should return enclosed string unchanged"
-    end
+
+  def test_should_allow_method_calls_to_string
     assert_nothing_raised do
-      assert_equal "a", "a", "Normal string comparisons should be unaffected"
-      assert_not_equal "a", "b", "Normal string comparisons should be unaffected"
-      assert_not_equal "a".chars, "b".chars, "Chars objects should be comparable"
-      assert_equal "a".chars, "A".downcase.chars, "Chars objects should be comparable to each other"
-      assert_equal "a".chars, "A".downcase, "Chars objects should be comparable to strings coming from elsewhere"
+      @chars.__string_for_multibyte_testing
     end
-    
-    assert !@s[:utf8].eql?(@s[:utf8].chars), "Strict comparison is not supported"
-    assert_equal @s[:utf8], @s[:utf8].chars, "Chars should be compared by their enclosed string"
-
-    other_string = @s[:utf8].dup
-    assert_equal other_string, @s[:utf8].chars, "Chars should be compared by their enclosed string"
-    assert_equal other_string.chars, @s[:utf8].chars, "Chars should be compared by their enclosed string"
-    
-    strings = ['builder'.chars, 'armor'.chars, 'zebra'.chars]
-    strings.sort!
-    assert_equal ['armor', 'builder', 'zebra'], strings, "Chars should be sortable based on their enclosed string"
-
-    # This leads to a StackLevelTooDeep exception if the comparison is not wired properly
-    assert_raise(NameError) do
-      Chars
+    assert_raises NoMethodError do
+      @chars.__unknown_method
     end
   end
-  
-  def test_utf8?
-    assert @s[:utf8].is_utf8?, "UTF-8 strings are UTF-8"
-    assert @s[:ascii].is_utf8?, "All ASCII strings are also valid UTF-8"
-    assert !@s[:bytes].is_utf8?, "This bytestring isn't UTF-8"
-  end
-  
-  # The test for the following methods are defined here because they can only be defined on the Chars class for
-  # various reasons 
-  
-  def test_gsub
-    assert_equal 'éxa', 'éda'.chars.gsub(/d/, 'x')
-    with_kcode('none') do
-      assert_equal 'éxa', 'éda'.chars.gsub(/d/, 'x')
-    end
+
+  def test_forwarded_method_calls_should_return_new_chars_instance
+    assert @chars.__string_for_multibyte_testing.kind_of?(ActiveSupport::Multibyte::Chars)
+    assert_not_equal @chars.object_id, @chars.__string_for_multibyte_testing.object_id
   end
-  
-  def test_split
-    word = "efficient"
-    chars = ["e", "ffi", "c", "i", "e", "n", "t"]
-    assert_equal chars, word.split(//)
-    assert_equal chars, word.chars.split(//)
-    assert_kind_of ActiveSupport::Multibyte::Chars, word.chars.split(//).first, "Split should return Chars instances"
-  end
-  
-  def test_regexp
-    with_kcode('none') do
-      assert_equal 12, (@s[:utf8].chars =~ /ffi/),
-        "Regex matching should be bypassed to String"
+
+  def test_forwarded_bang_method_calls_should_return_the_original_chars_instance
+    assert @chars.__string_for_multibyte_testing!.kind_of?(ActiveSupport::Multibyte::Chars)
+    assert_equal @chars.object_id, @chars.__string_for_multibyte_testing!.object_id
+  end
+
+  def test_methods_are_forwarded_to_wrapped_string_for_byte_strings
+    assert_equal BYTE_STRING.class, BYTE_STRING.mb_chars.class
+  end
+
+  def test_should_concatenate
+    assert_equal 'ab', 'a'.mb_chars + 'b'
+    assert_equal 'ab', 'a' + 'b'.mb_chars
+    assert_equal 'ab', 'a'.mb_chars + 'b'.mb_chars
+
+    assert_equal 'ab', 'a'.mb_chars << 'b'
+    assert_equal 'ab', 'a' << 'b'.mb_chars
+    assert_equal 'ab', 'a'.mb_chars << 'b'.mb_chars
+  end
+
+  if RUBY_VERSION < '1.9'
+    def test_concatenation_should_return_a_proxy_class_instance
+      assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars + 'b').class
+      assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars << 'b').class
     end
-    with_kcode('UTF8') do
-      assert_equal 9, (@s[:utf8].chars =~ /ffi/),
-        "Regex matching should be unicode aware"
-      assert_nil((''.chars =~ /\d+/),
-        "Non-matching regular expressions should return nil")
+
+    def test_ascii_strings_are_treated_at_utf8_strings
+      assert_equal ActiveSupport::Multibyte.proxy_class, ASCII_STRING.mb_chars.class
+    end
+
+    def test_concatenate_should_return_proxy_instance
+      assert(('a'.mb_chars + 'b').kind_of?(ActiveSupport::Multibyte::Chars))
+      assert(('a'.mb_chars + 'b'.mb_chars).kind_of?(ActiveSupport::Multibyte::Chars))
+      assert(('a'.mb_chars << 'b').kind_of?(ActiveSupport::Multibyte::Chars))
+      assert(('a'.mb_chars << 'b'.mb_chars).kind_of?(ActiveSupport::Multibyte::Chars))
     end
   end
+end
 
-  def test_pragma
-    if RUBY_VERSION < '1.9'
-      with_kcode('UTF8') do
-        assert " ".chars.send(:utf8_pragma?), "UTF8 pragma should be on because KCODE is UTF8"
-      end
-      with_kcode('none') do
-        assert !" ".chars.send(:utf8_pragma?), "UTF8 pragma should be off because KCODE is not UTF8"
+class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase
+  include MultibyteTest
+
+  def setup
+    @chars = UNICODE_STRING.dup.mb_chars
+
+    # NEWLINE, SPACE, EM SPACE
+    @whitespace = "\n#{[32, 8195].pack('U*')}"
+    @whitespace.force_encoding(Encoding::UTF_8) if @whitespace.respond_to?(:force_encoding)
+    @byte_order_mark = [65279].pack('U')
+  end
+
+  if RUBY_VERSION < '1.9'
+    def test_split_should_return_an_array_of_chars_instances
+      @chars.split(//).each do |character|
+        assert character.kind_of?(ActiveSupport::Multibyte::Chars)
       end
-    else
-      assert !" ".chars.send(:utf8_pragma?), "UTF8 pragma should be off in Ruby 1.9"
     end
-  end
 
-  def test_handler_setting
-    handler = ''.chars.handler
-    
-    ActiveSupport::Multibyte::Chars.handler = :first
-    assert_equal :first, ''.chars.handler
-    ActiveSupport::Multibyte::Chars.handler = :second
-    assert_equal :second, ''.chars.handler
-    assert_raise(NoMethodError) do
-      ''.chars.handler.split
+    def test_indexed_insert_accepts_fixnums
+      @chars[2] = 32
+      assert_equal 'こに わ', @chars
     end
-    
-    ActiveSupport::Multibyte::Chars.handler = handler
-  end
-  
-  def test_method_chaining
-    assert_kind_of ActiveSupport::Multibyte::Chars, ''.chars.downcase
-    assert_kind_of ActiveSupport::Multibyte::Chars, ''.chars.strip, "Strip should return a Chars object"
-    assert_kind_of ActiveSupport::Multibyte::Chars, ''.chars.downcase.strip, "The Chars object should be " +
-        "forwarded down the call path for chaining"
-    assert_equal 'foo', "  FOO   ".chars.normalize.downcase.strip, "The Chars that results from the " +
-      " operations should be comparable to the string value of the result"
-  end
-  
-  def test_passthrough_on_kcode
-    # The easiest way to check if the passthrough is in place is through #size
-    with_kcode('none') do
-      assert_equal 26, @s[:utf8].chars.size
+
+    def test_overridden_bang_methods_return_self
+      [:rstrip!, :lstrip!, :strip!, :reverse!, :upcase!, :downcase!, :capitalize!].each do |method|
+        assert_equal @chars.object_id, @chars.send(method).object_id
+      end
+      assert_equal @chars.object_id, @chars.slice!(1).object_id
     end
-    with_kcode('UTF8') do
-      assert_equal 17, @s[:utf8].chars.size
+
+    def test_overridden_bang_methods_change_wrapped_string
+      [:rstrip!, :lstrip!, :strip!, :reverse!, :upcase!, :downcase!].each do |method|
+        original = ' Café '
+        proxy = chars(original.dup)
+        proxy.send(method)
+        assert_not_equal original, proxy.to_s
+      end
+      proxy = chars('Café')
+      proxy.slice!(3)
+      assert_equal 'é', proxy.to_s
+
+      proxy = chars('òu')
+      proxy.capitalize!
+      assert_equal 'Òu', proxy.to_s
     end
   end
-    
-  def test_destructiveness  
-    # Note that we're testing the destructiveness here and not the correct behaviour of the methods
-    str = 'ac'
-    str.chars.insert(1, 'b')
-    assert_equal 'abc', str, 'Insert should be destructive for a string'
-    
-    str = 'ac'
-    str.chars.reverse!
-    assert_equal 'ca', str, 'reverse! should be destructive for a string'
-  end
-  
-  def test_resilience
-    assert_nothing_raised do
-      assert_equal 5, @s[:bytes].chars.size, "The sequence contains five interpretable bytes"
-    end
-    reversed = [0xb8, 0x17e, 0x8, 0x2c6, 0xa5].reverse.pack('U*')
-    assert_nothing_raised do
-      assert_equal reversed, @s[:bytes].chars.reverse.to_s, "Reversing the string should only yield interpretable bytes"
+
+  if RUBY_VERSION >= '1.9'
+    def test_unicode_string_should_have_utf8_encoding
+      assert_equal Encoding::UTF_8, UNICODE_STRING.encoding
     end
-    assert_nothing_raised do
-      @s[:bytes].chars.reverse!
-      assert_equal reversed, @s[:bytes].to_s, "Reversing the string should only yield interpretable bytes"
+  end
+
+  def test_should_be_equal_to_the_wrapped_string
+    assert_equal UNICODE_STRING, @chars
+    assert_equal @chars, UNICODE_STRING
+  end
+
+  def test_should_not_be_equal_to_an_other_string
+    assert_not_equal @chars, 'other'
+    assert_not_equal 'other', @chars
+  end
+
+  def test_should_return_character_offset_for_regexp_matches
+    assert_nil(@chars =~ /wrong/u)
+    assert_equal 0, (@chars =~ /こ/u)
+    assert_equal 1, (@chars =~ /に/u)
+    assert_equal 3, (@chars =~ /わ/u)
+  end
+
+  def test_should_use_character_offsets_for_insert_offsets
+    assert_equal '', ''.mb_chars.insert(0, '')
+    assert_equal 'こわにちわ', @chars.insert(1, 'わ')
+    assert_equal 'こわわわにちわ', @chars.insert(2, 'わわ')
+    assert_equal 'わこわわわにちわ', @chars.insert(0, 'わ')
+    assert_equal 'わこわわわにちわ', @chars.wrapped_string if RUBY_VERSION < '1.9'
+  end
+
+  def test_insert_should_be_destructive
+    @chars.insert(1, 'わ')
+    assert_equal 'こわにちわ', @chars
+  end
+
+  def test_insert_throws_index_error
+    assert_raises(IndexError) { @chars.insert(12, 'わ') }
+  end
+
+  def test_should_know_if_one_includes_the_other
+    assert @chars.include?('')
+    assert @chars.include?('ち')
+    assert @chars.include?('わ')
+    assert !@chars.include?('こちわ')
+    assert !@chars.include?('a')
+  end
+
+  def test_include_raises_type_error_when_nil_is_passed
+    assert_raises(TypeError) do
+      @chars.include?(nil)
     end
   end
-  
-  def test_duck_typing
-    assert_equal true,  'test'.chars.respond_to?(:strip)
-    assert_equal true,  'test'.chars.respond_to?(:normalize)
-    assert_equal true,  'test'.chars.respond_to?(:normalize!)
-    assert_equal false, 'test'.chars.respond_to?(:a_method_that_doesnt_exist)
+
+  def test_index_should_return_character_offset
+    assert_nil @chars.index('u')
+    assert_equal 0, @chars.index('こに')
+    assert_equal 2, @chars.index('ち')
+    assert_equal 3, @chars.index('わ')
+  end
+
+  def test_indexed_insert_should_take_character_offsets
+    @chars[2] = 'a'
+    assert_equal 'こにaわ', @chars
+    @chars[2] = 'ηη'
+    assert_equal 'こにηηわ', @chars
+    @chars[3, 2] = 'λλλ'
+    assert_equal 'こにηλλλ', @chars
+    @chars[1, 0] = "λ"
+    assert_equal 'こλにηλλλ', @chars
+    @chars[4..6] = "ηη"
+    assert_equal 'こλにηηη', @chars
+    @chars[/ηη/] = "λλλ"
+    assert_equal 'こλにλλλη', @chars
+    @chars[/(λλ)(.)/, 2] = "α"
+    assert_equal 'こλにλλαη', @chars
+    @chars["α"] = "¢"
+    assert_equal 'こλにλλ¢η', @chars
+    @chars["λλ"] = "ααα"
+    assert_equal 'こλにααα¢η', @chars
+  end
+
+  def test_indexed_insert_should_raise_on_index_overflow
+    before = @chars.to_s
+    assert_raises(IndexError) { @chars[10] = 'a' }
+    assert_raises(IndexError) { @chars[10, 4] = 'a' }
+    assert_raises(IndexError) { @chars[/ii/] = 'a' }
+    assert_raises(IndexError) { @chars[/()/, 10] = 'a' }
+    assert_equal before, @chars
+  end
+
+  def test_indexed_insert_should_raise_on_range_overflow
+    before = @chars.to_s
+    assert_raises(RangeError) { @chars[10..12] = 'a' }
+    assert_equal before, @chars
   end
-  
+
+  def test_rjust_should_raise_argument_errors_on_bad_arguments
+    assert_raises(ArgumentError) { @chars.rjust(10, '') }
+    assert_raises(ArgumentError) { @chars.rjust }
+  end
+
+  def test_rjust_should_count_characters_instead_of_bytes
+    assert_equal UNICODE_STRING, @chars.rjust(-3)
+    assert_equal UNICODE_STRING, @chars.rjust(0)
+    assert_equal UNICODE_STRING, @chars.rjust(4)
+    assert_equal " #{UNICODE_STRING}", @chars.rjust(5)
+    assert_equal "   #{UNICODE_STRING}", @chars.rjust(7)
+    assert_equal "---#{UNICODE_STRING}", @chars.rjust(7, '-')
+    assert_equal "ααα#{UNICODE_STRING}", @chars.rjust(7, 'α')
+    assert_equal "aba#{UNICODE_STRING}", @chars.rjust(7, 'ab')
+    assert_equal "αηα#{UNICODE_STRING}", @chars.rjust(7, 'αη')
+    assert_equal "αηαη#{UNICODE_STRING}", @chars.rjust(8, 'αη')
+  end
+
+  def test_ljust_should_raise_argument_errors_on_bad_arguments
+    assert_raises(ArgumentError) { @chars.ljust(10, '') }
+    assert_raises(ArgumentError) { @chars.ljust }
+  end
+
+  def test_ljust_should_count_characters_instead_of_bytes
+    assert_equal UNICODE_STRING, @chars.ljust(-3)
+    assert_equal UNICODE_STRING, @chars.ljust(0)
+    assert_equal UNICODE_STRING, @chars.ljust(4)
+    assert_equal "#{UNICODE_STRING} ", @chars.ljust(5)
+    assert_equal "#{UNICODE_STRING}   ", @chars.ljust(7)
+    assert_equal "#{UNICODE_STRING}---", @chars.ljust(7, '-')
+    assert_equal "#{UNICODE_STRING}ααα", @chars.ljust(7, 'α')
+    assert_equal "#{UNICODE_STRING}aba", @chars.ljust(7, 'ab')
+    assert_equal "#{UNICODE_STRING}αηα", @chars.ljust(7, 'αη')
+    assert_equal "#{UNICODE_STRING}αηαη", @chars.ljust(8, 'αη')
+  end
+
+  def test_center_should_raise_argument_errors_on_bad_arguments
+    assert_raises(ArgumentError) { @chars.center(10, '') }
+    assert_raises(ArgumentError) { @chars.center }
+  end
+
+  def test_center_should_count_charactes_instead_of_bytes
+    assert_equal UNICODE_STRING, @chars.center(-3)
+    assert_equal UNICODE_STRING, @chars.center(0)
+    assert_equal UNICODE_STRING, @chars.center(4)
+    assert_equal "#{UNICODE_STRING} ", @chars.center(5)
+    assert_equal " #{UNICODE_STRING} ", @chars.center(6)
+    assert_equal " #{UNICODE_STRING}  ", @chars.center(7)
+    assert_equal "--#{UNICODE_STRING}--", @chars.center(8, '-')
+    assert_equal "--#{UNICODE_STRING}---", @chars.center(9, '-')
+    assert_equal "αα#{UNICODE_STRING}αα", @chars.center(8, 'α')
+    assert_equal "αα#{UNICODE_STRING}ααα", @chars.center(9, 'α')
+    assert_equal "a#{UNICODE_STRING}ab", @chars.center(7, 'ab')
+    assert_equal "ab#{UNICODE_STRING}ab", @chars.center(8, 'ab')
+    assert_equal "abab#{UNICODE_STRING}abab", @chars.center(12, 'ab')
+    assert_equal "α#{UNICODE_STRING}αη", @chars.center(7, 'αη')
+    assert_equal "αη#{UNICODE_STRING}αη", @chars.center(8, 'αη')
+  end
+
+  def test_lstrip_strips_whitespace_from_the_left_of_the_string
+    assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.lstrip
+    assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING).mb_chars.lstrip
+    assert_equal UNICODE_STRING + @whitespace, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.lstrip
+  end
+
+  def test_rstrip_strips_whitespace_from_the_right_of_the_string
+    assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.rstrip
+    assert_equal UNICODE_STRING, (UNICODE_STRING + @whitespace).mb_chars.rstrip
+    assert_equal @whitespace + UNICODE_STRING, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.rstrip
+  end
+
+  def test_strip_strips_whitespace
+    assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.strip
+    assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING).mb_chars.strip
+    assert_equal UNICODE_STRING, (UNICODE_STRING + @whitespace).mb_chars.strip
+    assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.strip
+  end
+
+  def test_stripping_whitespace_leaves_whitespace_within_the_string_intact
+    string_with_whitespace = UNICODE_STRING + @whitespace + UNICODE_STRING
+    assert_equal string_with_whitespace, string_with_whitespace.mb_chars.strip
+    assert_equal string_with_whitespace, string_with_whitespace.mb_chars.lstrip
+    assert_equal string_with_whitespace, string_with_whitespace.mb_chars.rstrip
+  end
+
+  def test_size_returns_characters_instead_of_bytes
+    assert_equal 0, ''.mb_chars.size
+    assert_equal 4, @chars.size
+    assert_equal 4, @chars.length
+    assert_equal 5, ASCII_STRING.mb_chars.size
+  end
+
+  def test_reverse_reverses_characters
+    assert_equal '', ''.mb_chars.reverse
+    assert_equal 'わちにこ', @chars.reverse
+  end
+
+  def test_slice_should_take_character_offsets
+    assert_equal nil, ''.mb_chars.slice(0)
+    assert_equal 'こ', @chars.slice(0)
+    assert_equal 'わ', @chars.slice(3)
+    assert_equal nil, ''.mb_chars.slice(-1..1)
+    assert_equal '', ''.mb_chars.slice(0..10)
+    assert_equal 'にちわ', @chars.slice(1..3)
+    assert_equal 'にちわ', @chars.slice(1, 3)
+    assert_equal 'こ', @chars.slice(0, 1)
+    assert_equal 'ちわ', @chars.slice(2..10)
+    assert_equal '', @chars.slice(4..10)
+    assert_equal 'に', @chars.slice(/に/u)
+    assert_equal 'にち', @chars.slice(/に\w/u)
+    assert_equal nil, @chars.slice(/unknown/u)
+    assert_equal 'にち', @chars.slice(/(にち)/u, 1)
+    assert_equal nil, @chars.slice(/(にち)/u, 2)
+    assert_equal nil, @chars.slice(7..6)
+  end
+
+  def test_slice_should_throw_exceptions_on_invalid_arguments
+    assert_raise(TypeError) { @chars.slice(2..3, 1) }
+    assert_raise(TypeError) { @chars.slice(1, 2..3) }
+    assert_raise(ArgumentError) { @chars.slice(1, 1, 1) }
+  end
+
+  def test_upcase_should_upcase_ascii_characters
+    assert_equal '', ''.mb_chars.upcase
+    assert_equal 'ABC', 'aBc'.mb_chars.upcase
+  end
+
+  def test_downcase_should_downcase_ascii_characters
+    assert_equal '', ''.mb_chars.downcase
+    assert_equal 'abc', 'aBc'.mb_chars.downcase
+  end
+
+  def test_capitalize_should_work_on_ascii_characters
+    assert_equal '', ''.mb_chars.capitalize
+    assert_equal 'Abc', 'abc'.mb_chars.capitalize
+  end
+
+  def test_respond_to_knows_which_methods_the_proxy_responds_to
+    assert ''.mb_chars.respond_to?(:slice) # Defined on Chars
+    assert ''.mb_chars.respond_to?(:capitalize!) # Defined on Chars
+    assert ''.mb_chars.respond_to?(:gsub) # Defined on String
+    assert !''.mb_chars.respond_to?(:undefined_method) # Not defined
+  end
+
   def test_acts_like_string
-    assert 'Bambi'.chars.acts_like_string?
+    assert 'Bambi'.mb_chars.acts_like_string?
   end
+end
 
-  protected
+# The default Multibyte Chars proxy has more features than the normal string implementation. Tests
+# for the implementation of these features should run on all Ruby versions and shouldn't be tested
+# through the proxy methods.
+class MultibyteCharsExtrasTest < Test::Unit::TestCase
+  include MultibyteTest
 
-  def with_kcode(kcode)
-    old_kcode, $KCODE = $KCODE, kcode
-    begin
-      yield
-    ensure
-      $KCODE = old_kcode
+  if RUBY_VERSION >= '1.9'
+    def test_tidy_bytes_is_broken_on_1_9_0
+      assert_raises(ArgumentError) do
+        assert_equal_codepoints [0xfffd].pack('U'), chars("\xef\xbf\xbd").tidy_bytes
+      end
     end
   end
-end
 
+  def test_upcase_should_be_unicode_aware
+    assert_equal "АБВГД\0F", chars("аБвгд\0f").upcase
+    assert_equal 'こにちわ', chars('こにちわ').upcase
+  end
+
+  def test_downcase_should_be_unicode_aware
+    assert_equal "абвгд\0f", chars("аБвгд\0f").downcase
+    assert_equal 'こにちわ', chars('こにちわ').downcase
+  end
+
+  def test_capitalize_should_be_unicode_aware
+    { 'аБвг аБвг' => 'Абвг абвг',
+      'аБвг АБВГ' => 'Абвг абвг',
+      'АБВГ АБВГ' => 'Абвг абвг',
+      '' => '' }.each do |f,t|
+        assert_equal t, chars(f).capitalize
+    end
+  end
+
+  def test_composition_exclusion_is_set_up_properly
+    # Normalization of DEVANAGARI LETTER QA breaks when composition exclusion isn't used correctly
+    qa = [0x915, 0x93c].pack('U*')
+    assert_equal qa, chars(qa).normalize(:c)
+  end
+
+  # Test for the Public Review Issue #29, bad explanation of composition might lead to a
+  # bad implementation: http://www.unicode.org/review/pr-29.html
+  def test_normalization_C_pri_29
+    [
+      [0x0B47, 0x0300, 0x0B3E],
+      [0x1100, 0x0300, 0x1161]
+    ].map { |c| c.pack('U*') }.each do |c|
+      assert_equal_codepoints c, chars(c).normalize(:c)
+    end
+  end
+
+  def test_normalization_shouldnt_strip_null_bytes
+    null_byte_str = "Test\0test"
+
+    assert_equal null_byte_str, chars(null_byte_str).normalize(:kc)
+    assert_equal null_byte_str, chars(null_byte_str).normalize(:c)
+    assert_equal null_byte_str, chars(null_byte_str).normalize(:d)
+    assert_equal null_byte_str, chars(null_byte_str).normalize(:kd)
+    assert_equal null_byte_str, chars(null_byte_str).decompose
+    assert_equal null_byte_str, chars(null_byte_str).compose
+  end
+
+  def test_simple_normalization
+    comp_str = [
+      44,  # LATIN CAPITAL LETTER D
+      307, # COMBINING DOT ABOVE
+      328, # COMBINING OGONEK
+      323 # COMBINING DOT BELOW
+    ].pack("U*")
+
+    assert_equal_codepoints '', chars('').normalize
+    assert_equal_codepoints [44,105,106,328,323].pack("U*"), chars(comp_str).normalize(:kc).to_s
+    assert_equal_codepoints [44,307,328,323].pack("U*"), chars(comp_str).normalize(:c).to_s
+    assert_equal_codepoints [44,307,110,780,78,769].pack("U*"), chars(comp_str).normalize(:d).to_s
+    assert_equal_codepoints [44,105,106,110,780,78,769].pack("U*"), chars(comp_str).normalize(:kd).to_s
+  end
+
+  def test_should_compute_grapheme_length
+    [
+      ['', 0],
+      ['abc', 3],
+      ['こにちわ', 4],
+      [[0x0924, 0x094D, 0x0930].pack('U*'), 2],
+      [%w(cr lf), 1],
+      [%w(l l), 1],
+      [%w(l v), 1],
+      [%w(l lv), 1],
+      [%w(l lvt), 1],
+      [%w(lv v), 1],
+      [%w(lv t), 1],
+      [%w(v v), 1],
+      [%w(v t), 1],
+      [%w(lvt t), 1],
+      [%w(t t), 1],
+      [%w(n extend), 1],
+      [%w(n n), 2],
+      [%w(n cr lf n), 3],
+      [%w(n l v t), 2]
+    ].each do |input, expected_length|
+      if input.kind_of?(Array)
+        str = string_from_classes(input)
+      else
+        str = input
+      end
+      assert_equal expected_length, chars(str).g_length
+    end
+  end
+
+  def test_tidy_bytes_should_tidy_bytes
+    byte_string = "\270\236\010\210\245"
+    tidy_string = [0xb8, 0x17e, 0x8, 0x2c6, 0xa5].pack('U*')
+    ascii_padding = 'aa'
+    utf8_padding = 'éé'
+
+    assert_equal_codepoints tidy_string, chars(byte_string).tidy_bytes
+
+    assert_equal_codepoints ascii_padding.dup.insert(1, tidy_string),
+      chars(ascii_padding.dup.insert(1, byte_string)).tidy_bytes
+    assert_equal_codepoints utf8_padding.dup.insert(2, tidy_string),
+      chars(utf8_padding.dup.insert(2, byte_string)).tidy_bytes
+    assert_nothing_raised { chars(byte_string).tidy_bytes.to_s.unpack('U*') }
+
+    assert_equal_codepoints "\xC3\xA7", chars("\xE7").tidy_bytes # iso_8859_1: small c cedilla
+    assert_equal_codepoints "\xE2\x80\x9C", chars("\x93").tidy_bytes # win_1252: left smart quote
+    assert_equal_codepoints "\xE2\x82\xAC", chars("\x80").tidy_bytes # win_1252: euro
+    assert_equal_codepoints "\x00", chars("\x00").tidy_bytes # null char
+    assert_equal_codepoints [0xfffd].pack('U'), chars("\xef\xbf\xbd").tidy_bytes # invalid char
+  rescue ArgumentError => e
+    raise e if RUBY_VERSION < '1.9'
+  end
+
+  private
+
+  def string_from_classes(classes)
+    # Characters from the character classes as described in UAX #29
+    character_from_class = {
+      :l => 0x1100, :v => 0x1160, :t => 0x11A8, :lv => 0xAC00, :lvt => 0xAC01, :cr => 0x000D, :lf => 0x000A,
+      :extend => 0x094D, :n => 0x64
+    }
+    classes.collect do |k|
+      character_from_class[k.intern]
+    end.pack('U*')
+  end
 end