From 9a73630d935e360f3dc896e50dd673afb97cf3b5 Mon Sep 17 00:00:00 2001 From: Michael Koziarski Date: Mon, 31 Aug 2009 12:16:22 -0700 Subject: Add verify and clean methods to ActiveSupport::Multibyte. When accepting character input from outside of your application you can't blindly trust that all strings are properly encoded. With these methods you can check incoming strings and clean them up if necessary. Signed-off-by: Michael Koziarski Conflicts: activesupport/lib/active_support/multibyte.rb --- activesupport/test/multibyte_utils_test.rb | 141 +++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 activesupport/test/multibyte_utils_test.rb (limited to 'activesupport/test') diff --git a/activesupport/test/multibyte_utils_test.rb b/activesupport/test/multibyte_utils_test.rb new file mode 100644 index 0000000000..d8ac5ff139 --- /dev/null +++ b/activesupport/test/multibyte_utils_test.rb @@ -0,0 +1,141 @@ +# encoding: utf-8 + +require 'abstract_unit' +require 'multibyte_test_helpers' + +class MultibyteUtilsTest < ActiveSupport::TestCase + include MultibyteTestHelpers + + test "valid_character returns an expression for the current encoding" do + with_encoding('None') do + assert_nil ActiveSupport::Multibyte.valid_character + end + with_encoding('UTF8') do + assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['UTF-8'], ActiveSupport::Multibyte.valid_character + end + with_encoding('SJIS') do + assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['Shift_JIS'], ActiveSupport::Multibyte.valid_character + end + end + + test "verify verifies ASCII strings are properly encoded" do + with_encoding('None') do + examples.each do |example| + assert ActiveSupport::Multibyte.verify(example) + end + end + end + + test "verify verifies UTF-8 strings are properly encoded" do + with_encoding('UTF8') do + assert ActiveSupport::Multibyte.verify(example('valid UTF-8')) + assert !ActiveSupport::Multibyte.verify(example('invalid UTF-8')) + end + end + + test "verify verifies Shift-JIS strings are properly encoded" do + with_encoding('SJIS') do + assert ActiveSupport::Multibyte.verify(example('valid Shift-JIS')) + assert !ActiveSupport::Multibyte.verify(example('invalid Shift-JIS')) + end + end + + test "verify! raises an exception when it finds an invalid character" do + with_encoding('UTF8') do + assert_raises(ActiveSupport::Multibyte::EncodingError) do + ActiveSupport::Multibyte.verify!(example('invalid UTF-8')) + end + end + end + + test "verify! doesn't raise an exception when the encoding is valid" do + with_encoding('UTF8') do + assert_nothing_raised do + ActiveSupport::Multibyte.verify!(example('valid UTF-8')) + end + end + end + + if RUBY_VERSION < '1.9' + test "clean leaves ASCII strings intact" do + with_encoding('None') do + [ + 'word', "\270\236\010\210\245" + ].each do |string| + assert_equal string, ActiveSupport::Multibyte.clean(string) + end + end + end + + test "clean cleans invalid characters from UTF-8 encoded strings" do + with_encoding('UTF8') do + cleaned_utf8 = [8].pack('C*') + assert_equal example('valid UTF-8'), ActiveSupport::Multibyte.clean(example('valid UTF-8')) + assert_equal cleaned_utf8, ActiveSupport::Multibyte.clean(example('invalid UTF-8')) + end + end + + test "clean cleans invalid characters from Shift-JIS encoded strings" do + with_encoding('SJIS') do + cleaned_sjis = [184, 0, 136, 165].pack('C*') + assert_equal example('valid Shift-JIS'), ActiveSupport::Multibyte.clean(example('valid Shift-JIS')) + assert_equal cleaned_sjis, ActiveSupport::Multibyte.clean(example('invalid Shift-JIS')) + end + end + else + test "clean is a no-op" do + with_encoding('UTF8') do + assert_equal example('invalid Shift-JIS'), ActiveSupport::Multibyte.clean(example('invalid Shift-JIS')) + end + end + end + + private + + STRINGS = { + 'valid ASCII' => [65, 83, 67, 73, 73].pack('C*'), + 'invalid ASCII' => [128].pack('C*'), + 'valid UTF-8' => [227, 129, 147, 227, 129, 171, 227, 129, 161, 227, 130, 143].pack('C*'), + 'invalid UTF-8' => [184, 158, 8, 136, 165].pack('C*'), + 'valid Shift-JIS' => [131, 122, 129, 91, 131, 128].pack('C*'), + 'invalid Shift-JIS' => [184, 158, 8, 0, 255, 136, 165].pack('C*') + } + + if Kernel.const_defined?(:Encoding) + def example(key) + STRINGS[key].force_encoding(Encoding.default_internal) + end + + def examples + STRINGS.values.map { |s| s.force_encoding(Encoding.default_internal) } + end + else + def example(key) + STRINGS[key] + end + + def examples + STRINGS.values + end + end + + if 'string'.respond_to?(:encoding) + def with_encoding(enc) + before = Encoding.default_internal + + case enc + when 'UTF8' + Encoding.default_internal = Encoding::UTF_8 + when 'SJIS' + Encoding.default_internal = Encoding::Shift_JIS + else + Encoding.default_internal = Encoding::BINARY + end + yield + + Encoding.default_internal = before + end + else + alias with_encoding with_kcode + end +end \ No newline at end of file -- cgit v1.2.3 From fe68cf2784d4b00cdceafc9a22a6343fa67e8bee Mon Sep 17 00:00:00 2001 From: Joshua Peek Date: Sat, 5 Sep 2009 18:31:10 -0500 Subject: Fix failing AS isolated tests --- activesupport/test/core_ext/boolean_ext_test.rb | 5 ++++- activesupport/test/core_ext/nil_ext_test.rb | 5 ++++- activesupport/test/core_ext/object_ext_test.rb | 2 ++ activesupport/test/core_ext/regexp_ext_test.rb | 5 ++++- 4 files changed, 14 insertions(+), 3 deletions(-) (limited to 'activesupport/test') diff --git a/activesupport/test/core_ext/boolean_ext_test.rb b/activesupport/test/core_ext/boolean_ext_test.rb index 751f703745..9439716efb 100644 --- a/activesupport/test/core_ext/boolean_ext_test.rb +++ b/activesupport/test/core_ext/boolean_ext_test.rb @@ -1,3 +1,6 @@ +require 'abstract_unit' +require 'active_support/core_ext/boolean/conversions' + class BooleanExtAccessTests < Test::Unit::TestCase def test_to_param_on_true assert_equal true, true.to_param @@ -6,4 +9,4 @@ class BooleanExtAccessTests < Test::Unit::TestCase def test_to_param_on_false assert_equal false, false.to_param end -end \ No newline at end of file +end diff --git a/activesupport/test/core_ext/nil_ext_test.rb b/activesupport/test/core_ext/nil_ext_test.rb index 945d3af239..1062676d65 100644 --- a/activesupport/test/core_ext/nil_ext_test.rb +++ b/activesupport/test/core_ext/nil_ext_test.rb @@ -1,5 +1,8 @@ +require 'abstract_unit' +require 'active_support/core_ext/nil/conversions' + class NilExtAccessTests < Test::Unit::TestCase def test_to_param assert_nil nil.to_param end -end \ No newline at end of file +end diff --git a/activesupport/test/core_ext/object_ext_test.rb b/activesupport/test/core_ext/object_ext_test.rb index 72e3bffa4c..484eecaab6 100644 --- a/activesupport/test/core_ext/object_ext_test.rb +++ b/activesupport/test/core_ext/object_ext_test.rb @@ -1,4 +1,6 @@ require 'abstract_unit' +require 'active_support/core_ext/object/metaclass' +require 'active_support/core_ext/object/conversions' class ObjectExtTest < Test::Unit::TestCase def test_tap_yields_and_returns_self diff --git a/activesupport/test/core_ext/regexp_ext_test.rb b/activesupport/test/core_ext/regexp_ext_test.rb index e2d9140bca..cc3f07d5c5 100644 --- a/activesupport/test/core_ext/regexp_ext_test.rb +++ b/activesupport/test/core_ext/regexp_ext_test.rb @@ -1,3 +1,6 @@ +require 'abstract_unit' +require 'active_support/core_ext/regexp' + class RegexpExtAccessTests < Test::Unit::TestCase def test_number_of_captures assert_equal 0, //.number_of_captures @@ -23,4 +26,4 @@ class RegexpExtAccessTests < Test::Unit::TestCase assert_equal "foo", Regexp.unoptionalize("(?:foo)?") assert_equal "", Regexp.unoptionalize("") end -end \ No newline at end of file +end -- cgit v1.2.3