diff options
| -rw-r--r-- | activesupport/lib/active_support/multibyte.rb | 21 | ||||
| -rw-r--r-- | activesupport/lib/active_support/multibyte/utils.rb | 27 | ||||
| -rw-r--r-- | activesupport/test/multibyte_utils_test.rb | 93 |
3 files changed, 1 insertions, 140 deletions
diff --git a/activesupport/lib/active_support/multibyte.rb b/activesupport/lib/active_support/multibyte.rb index 57e8e24bf4..cabe073616 100644 --- a/activesupport/lib/active_support/multibyte.rb +++ b/activesupport/lib/active_support/multibyte.rb @@ -21,24 +21,5 @@ module ActiveSupport #:nodoc: def self.proxy_class @proxy_class ||= ActiveSupport::Multibyte::Chars end - - # Regular expressions that describe valid byte sequences for a character - VALID_CHARACTER = { - # Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site) - 'UTF-8' => /\A(?: - [\x00-\x7f] | - [\xc2-\xdf] [\x80-\xbf] | - \xe0 [\xa0-\xbf] [\x80-\xbf] | - [\xe1-\xef] [\x80-\xbf] [\x80-\xbf] | - \xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] | - [\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] | - \xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf])\z /xn, - # Quick check for valid Shift-JIS characters, disregards the odd-even pairing - 'Shift_JIS' => /\A(?: - [\x00-\x7e\xa1-\xdf] | - [\x81-\x9f\xe0-\xef] [\x40-\x7e\x80-\x9e\x9f-\xfc])\z /xn - } end -end - -require 'active_support/multibyte/utils'
\ No newline at end of file +end
\ No newline at end of file diff --git a/activesupport/lib/active_support/multibyte/utils.rb b/activesupport/lib/active_support/multibyte/utils.rb deleted file mode 100644 index bd6d4bad41..0000000000 --- a/activesupport/lib/active_support/multibyte/utils.rb +++ /dev/null @@ -1,27 +0,0 @@ -# encoding: utf-8 - -module ActiveSupport #:nodoc: - module Multibyte #:nodoc: - # Returns a regular expression that matches valid characters in the current encoding - def self.valid_character - VALID_CHARACTER[Encoding.default_external.to_s] - end - - # Verifies the encoding of a string - def self.verify(string) - string.valid_encoding? - end - - # Verifies the encoding of the string and raises an exception when it's not valid - def self.verify!(string) - raise EncodingError.new("Found characters with invalid encoding") unless verify(string) - end - - # Removes all invalid characters from the string. - # - # Note: this method is a no-op in Ruby 1.9 - def self.clean(string) - string - end - end -end diff --git a/activesupport/test/multibyte_utils_test.rb b/activesupport/test/multibyte_utils_test.rb deleted file mode 100644 index f807492be0..0000000000 --- a/activesupport/test/multibyte_utils_test.rb +++ /dev/null @@ -1,93 +0,0 @@ -# encoding: utf-8 - -require 'abstract_unit' -require 'multibyte_test_helpers' - -class MultibyteUtilsTest < ActiveSupport::TestCase - include MultibyteTestHelpers - - test "valid_character returns an expression for the current encoding" do - with_encoding('None') do - assert_nil ActiveSupport::Multibyte.valid_character - end - with_encoding('UTF8') do - assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['UTF-8'], ActiveSupport::Multibyte.valid_character - end - with_encoding('SJIS') do - assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['Shift_JIS'], ActiveSupport::Multibyte.valid_character - end - end - - test "verify verifies ASCII strings are properly encoded" do - with_encoding('None') do - examples.each do |example| - assert ActiveSupport::Multibyte.verify(example) - end - end - end - - test "verify verifies UTF-8 strings are properly encoded" do - with_encoding('UTF8') do - assert ActiveSupport::Multibyte.verify(example('valid UTF-8')) - assert !ActiveSupport::Multibyte.verify(example('invalid UTF-8')) - end - end - - test "verify verifies Shift-JIS strings are properly encoded" do - with_encoding('SJIS') do - assert ActiveSupport::Multibyte.verify(example('valid Shift-JIS')) - assert !ActiveSupport::Multibyte.verify(example('invalid Shift-JIS')) - end - end - - test "verify! raises an exception when it finds an invalid character" do - with_encoding('UTF8') do - assert_raises(ActiveSupport::Multibyte::EncodingError) do - ActiveSupport::Multibyte.verify!(example('invalid UTF-8')) - end - end - end - - test "verify! doesn't raise an exception when the encoding is valid" do - with_encoding('UTF8') do - assert_nothing_raised do - ActiveSupport::Multibyte.verify!(example('valid UTF-8')) - end - end - end - - test "clean is a no-op" do - with_encoding('UTF8') do - assert_equal example('invalid Shift-JIS'), ActiveSupport::Multibyte.clean(example('invalid Shift-JIS')) - end - end - - private - - STRINGS = { - 'valid ASCII' => [65, 83, 67, 73, 73].pack('C*'), - 'invalid ASCII' => [128].pack('C*'), - 'valid UTF-8' => [227, 129, 147, 227, 129, 171, 227, 129, 161, 227, 130, 143].pack('C*'), - 'invalid UTF-8' => [184, 158, 8, 136, 165].pack('C*'), - 'valid Shift-JIS' => [131, 122, 129, 91, 131, 128].pack('C*'), - 'invalid Shift-JIS' => [184, 158, 8, 0, 255, 136, 165].pack('C*') - } - - def example(key) - STRINGS[key].force_encoding(Encoding.default_external) - end - - def examples - STRINGS.values.map { |s| s.force_encoding(Encoding.default_external) } - end - - KCODE_TO_ENCODING = Hash.new(Encoding::BINARY). - update('UTF8' => Encoding::UTF_8, 'SJIS' => Encoding::Shift_JIS) - - def with_encoding(enc) - before = Encoding.default_external - silence_warnings { Encoding.default_external = KCODE_TO_ENCODING[enc] } - yield - silence_warnings { Encoding.default_external = before } - end -end |