From f238d495b70a264abdb864fe8107e02766b285b4 Mon Sep 17 00:00:00 2001 From: Michael Koziarski Date: Tue, 3 Oct 2006 23:45:32 +0000 Subject: Add ActiveSupport::Multibyte. Provides String#chars which lets you deal with strings as a sequence of chars, not of bytes. Closes #6242 [Julian Tarkhanov, Manfred Stienstra & Jan Behrens] git-svn-id: http://svn-commit.rubyonrails.org/rails/trunk@5223 5ecf4fe2-1ee6-0310-87b1-e25e094e27de --- activesupport/test/multibyte_conformance.rb | 141 ++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 activesupport/test/multibyte_conformance.rb (limited to 'activesupport/test/multibyte_conformance.rb') diff --git a/activesupport/test/multibyte_conformance.rb b/activesupport/test/multibyte_conformance.rb new file mode 100644 index 0000000000..dfd77bd6e1 --- /dev/null +++ b/activesupport/test/multibyte_conformance.rb @@ -0,0 +1,141 @@ +require File.dirname(__FILE__) + '/abstract_unit' +require 'open-uri' + +$KCODE = 'UTF8' +UNIDATA_URL = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::UNICODE_VERSION}/ucd" +UNIDATA_FILE = '/NormalizationTest.txt' +CACHE_DIR = File.dirname(__FILE__) + '/cache' + +class Downloader + def self.download(from, to) + unless File.exist?(to) + $stderr.puts "Downloading #{from} to #{to}" + unless File.exists?(File.dirname(to)) + system "mkdir -p #{File.dirname(to)}" + end + open(from) do |source| + File.open(to, 'w') do |target| + source.each_line do |l| + target.write l + end + end + end + end + end +end + +class String + # Unicode Inspect returns the codepoints of the string in hex + def ui + "#{self} " + ("[%s]" % unpack("U*").map{|cp| cp.to_s(16) }.join(' ')) + end unless ''.respond_to?(:ui) +end + +Dir.mkdir(CACHE_DIR) unless File.exists?(CACHE_DIR) +Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE) + +module ConformanceTest + def test_normalizations_C + each_line_of_norm_tests do |*cols| + col1, col2, col3, col4, col5, comment = *cols + + # CONFORMANCE: + # 1. The following invariants must be true for all conformant implementations + # + # NFC + # c2 == NFC(c1) == NFC(c2) == NFC(c3) + assert_equal col2.ui, @handler.normalize(col1, :c).ui, "Form C - Col 2 has to be NFC(1) - #{comment}" + assert_equal col2.ui, @handler.normalize(col2, :c).ui, "Form C - Col 2 has to be NFC(2) - #{comment}" + assert_equal col2.ui, @handler.normalize(col3, :c).ui, "Form C - Col 2 has to be NFC(3) - #{comment}" + # + # c4 == NFC(c4) == NFC(c5) + assert_equal col4.ui, @handler.normalize(col4, :c).ui, "Form C - Col 4 has to be C(4) - #{comment}" + assert_equal col4.ui, @handler.normalize(col5, :c).ui, "Form C - Col 4 has to be C(5) - #{comment}" + end + end + + def test_normalizations_D + each_line_of_norm_tests do |*cols| + col1, col2, col3, col4, col5, comment = *cols + # + # NFD + # c3 == NFD(c1) == NFD(c2) == NFD(c3) + assert_equal col3.ui, @handler.normalize(col1, :d).ui, "Form D - Col 3 has to be NFD(1) - #{comment}" + assert_equal col3.ui, @handler.normalize(col2, :d).ui, "Form D - Col 3 has to be NFD(2) - #{comment}" + assert_equal col3.ui, @handler.normalize(col3, :d).ui, "Form D - Col 3 has to be NFD(3) - #{comment}" + # c5 == NFD(c4) == NFD(c5) + assert_equal col5.ui, @handler.normalize(col4, :d).ui, "Form D - Col 5 has to be NFD(4) - #{comment}" + assert_equal col5.ui, @handler.normalize(col5, :d).ui, "Form D - Col 5 has to be NFD(5) - #{comment}" + end + end + + def test_normalizations_KC + each_line_of_norm_tests do | *cols | + col1, col2, col3, col4, col5, comment = *cols + # + # NFKC + # c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5) + assert_equal col4.ui, @handler.normalize(col1, :kc).ui, "Form D - Col 4 has to be NFKC(1) - #{comment}" + assert_equal col4.ui, @handler.normalize(col2, :kc).ui, "Form D - Col 4 has to be NFKC(2) - #{comment}" + assert_equal col4.ui, @handler.normalize(col3, :kc).ui, "Form D - Col 4 has to be NFKC(3) - #{comment}" + assert_equal col4.ui, @handler.normalize(col4, :kc).ui, "Form D - Col 4 has to be NFKC(4) - #{comment}" + assert_equal col4.ui, @handler.normalize(col5, :kc).ui, "Form D - Col 4 has to be NFKC(5) - #{comment}" + end + end + + def test_normalizations_KD + each_line_of_norm_tests do | *cols | + col1, col2, col3, col4, col5, comment = *cols + # + # NFKD + # c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5) + assert_equal col5.ui, @handler.normalize(col1, :kd).ui, "Form KD - Col 5 has to be NFKD(1) - #{comment}" + assert_equal col5.ui, @handler.normalize(col2, :kd).ui, "Form KD - Col 5 has to be NFKD(2) - #{comment}" + assert_equal col5.ui, @handler.normalize(col3, :kd).ui, "Form KD - Col 5 has to be NFKD(3) - #{comment}" + assert_equal col5.ui, @handler.normalize(col4, :kd).ui, "Form KD - Col 5 has to be NFKD(4) - #{comment}" + assert_equal col5.ui, @handler.normalize(col5, :kd).ui, "Form KD - Col 5 has to be NFKD(5) - #{comment}" + end + end + + protected + def each_line_of_norm_tests(&block) + lines = 0 + max_test_lines = 0 # Don't limit below 38, because that's the header of the testfile + File.open(File.dirname(__FILE__) + '/cache' + UNIDATA_FILE, 'r') do | f | + until f.eof? || (max_test_lines > 38 and lines > max_test_lines) + lines += 1 + line = f.gets.chomp! + next if (line.empty? || line =~ /^\#/) + + cols, comment = line.split("#") + cols = cols.split(";").map{|e| e.strip}.reject{|e| e.empty? } + next unless cols.length == 5 + + # codepoints are in hex in the test suite, pack wants them as integers + cols.map!{|c| c.split.map{|codepoint| codepoint.to_i(16)}.pack("U*") } + cols << comment + + yield(*cols) + end + end + end +end + +begin + require_library_or_gem('utf8proc_native') + require 'active_record/multibyte/handlers/utf8_handler_proc' + class ConformanceTestProc < Test::Unit::TestCase + include ConformanceTest + def setup + @handler = ::ActiveSupport::Multibyte::Handlers::UTF8HandlerProc + end + end +rescue LoadError +end + +class ConformanceTestPure < Test::Unit::TestCase + include ConformanceTest + def setup + @handler = ::ActiveSupport::Multibyte::Handlers::UTF8Handler + end +end \ No newline at end of file -- cgit v1.2.3