From 75b199753c1a8d29f8405165d76011e73d886291 Mon Sep 17 00:00:00 2001 From: Akira Matsuda Date: Sun, 24 Jan 2016 23:09:03 +0900 Subject: Test files has to be named *_test.rb --- .../test/multibyte_grapheme_break_conformance.rb | 76 ------------ .../multibyte_grapheme_break_conformance_test.rb | 76 ++++++++++++ .../test/multibyte_normalization_conformance.rb | 129 --------------------- .../multibyte_normalization_conformance_test.rb | 129 +++++++++++++++++++++ 4 files changed, 205 insertions(+), 205 deletions(-) delete mode 100644 activesupport/test/multibyte_grapheme_break_conformance.rb create mode 100644 activesupport/test/multibyte_grapheme_break_conformance_test.rb delete mode 100644 activesupport/test/multibyte_normalization_conformance.rb create mode 100644 activesupport/test/multibyte_normalization_conformance_test.rb (limited to 'activesupport') diff --git a/activesupport/test/multibyte_grapheme_break_conformance.rb b/activesupport/test/multibyte_grapheme_break_conformance.rb deleted file mode 100644 index 7d185e2cae..0000000000 --- a/activesupport/test/multibyte_grapheme_break_conformance.rb +++ /dev/null @@ -1,76 +0,0 @@ -# encoding: utf-8 - -require 'abstract_unit' - -require 'fileutils' -require 'open-uri' -require 'tmpdir' - -class Downloader - def self.download(from, to) - unless File.exist?(to) - $stderr.puts "Downloading #{from} to #{to}" - unless File.exist?(File.dirname(to)) - system "mkdir -p #{File.dirname(to)}" - end - open(from) do |source| - File.open(to, 'w') do |target| - source.each_line do |l| - target.write l - end - end - end - end - end -end - -class MultibyteGraphemeBreakConformanceTest < ActiveSupport::TestCase - TEST_DATA_URL = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::Unicode::UNICODE_VERSION}/ucd/auxiliary" - TEST_DATA_FILE = '/GraphemeBreakTest.txt' - CACHE_DIR = File.join(Dir.tmpdir, 'cache') - - def setup - FileUtils.mkdir_p(CACHE_DIR) - Downloader.download(TEST_DATA_URL + TEST_DATA_FILE, CACHE_DIR + TEST_DATA_FILE) - end - - def test_breaks - each_line_of_break_tests do |*cols| - *clusters, comment = *cols - packed = ActiveSupport::Multibyte::Unicode.pack_graphemes(clusters) - assert_equal clusters, ActiveSupport::Multibyte::Unicode.unpack_graphemes(packed), comment - end - end - - protected - def each_line_of_break_tests(&block) - lines = 0 - max_test_lines = 0 # Don't limit below 21, because that's the header of the testfile - File.open(File.join(CACHE_DIR, TEST_DATA_FILE), 'r') do | f | - until f.eof? || (max_test_lines > 21 and lines > max_test_lines) - lines += 1 - line = f.gets.chomp! - next if (line.empty? || line =~ /^\#/) - - cols, comment = line.split("#") - # Cluster breaks are represented by ÷ - clusters = cols.split("÷").map{|e| e.strip}.reject{|e| e.empty? } - clusters = clusters.map do |cluster| - # Codepoints within each cluster are separated by × - codepoints = cluster.split("×").map{|e| e.strip}.reject{|e| e.empty? } - # codepoints are in hex in the test suite, pack wants them as integers - codepoints.map{|codepoint| codepoint.to_i(16)} - end - - # The tests contain a solitary U+D800 character, which Ruby does not allow to stand - # alone in a UTF-8 string. So we'll just skip it. - next if clusters.flatten.include?(0xd800) - - clusters << comment.strip - - yield(*clusters) - end - end - end -end diff --git a/activesupport/test/multibyte_grapheme_break_conformance_test.rb b/activesupport/test/multibyte_grapheme_break_conformance_test.rb new file mode 100644 index 0000000000..7d185e2cae --- /dev/null +++ b/activesupport/test/multibyte_grapheme_break_conformance_test.rb @@ -0,0 +1,76 @@ +# encoding: utf-8 + +require 'abstract_unit' + +require 'fileutils' +require 'open-uri' +require 'tmpdir' + +class Downloader + def self.download(from, to) + unless File.exist?(to) + $stderr.puts "Downloading #{from} to #{to}" + unless File.exist?(File.dirname(to)) + system "mkdir -p #{File.dirname(to)}" + end + open(from) do |source| + File.open(to, 'w') do |target| + source.each_line do |l| + target.write l + end + end + end + end + end +end + +class MultibyteGraphemeBreakConformanceTest < ActiveSupport::TestCase + TEST_DATA_URL = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::Unicode::UNICODE_VERSION}/ucd/auxiliary" + TEST_DATA_FILE = '/GraphemeBreakTest.txt' + CACHE_DIR = File.join(Dir.tmpdir, 'cache') + + def setup + FileUtils.mkdir_p(CACHE_DIR) + Downloader.download(TEST_DATA_URL + TEST_DATA_FILE, CACHE_DIR + TEST_DATA_FILE) + end + + def test_breaks + each_line_of_break_tests do |*cols| + *clusters, comment = *cols + packed = ActiveSupport::Multibyte::Unicode.pack_graphemes(clusters) + assert_equal clusters, ActiveSupport::Multibyte::Unicode.unpack_graphemes(packed), comment + end + end + + protected + def each_line_of_break_tests(&block) + lines = 0 + max_test_lines = 0 # Don't limit below 21, because that's the header of the testfile + File.open(File.join(CACHE_DIR, TEST_DATA_FILE), 'r') do | f | + until f.eof? || (max_test_lines > 21 and lines > max_test_lines) + lines += 1 + line = f.gets.chomp! + next if (line.empty? || line =~ /^\#/) + + cols, comment = line.split("#") + # Cluster breaks are represented by ÷ + clusters = cols.split("÷").map{|e| e.strip}.reject{|e| e.empty? } + clusters = clusters.map do |cluster| + # Codepoints within each cluster are separated by × + codepoints = cluster.split("×").map{|e| e.strip}.reject{|e| e.empty? } + # codepoints are in hex in the test suite, pack wants them as integers + codepoints.map{|codepoint| codepoint.to_i(16)} + end + + # The tests contain a solitary U+D800 character, which Ruby does not allow to stand + # alone in a UTF-8 string. So we'll just skip it. + next if clusters.flatten.include?(0xd800) + + clusters << comment.strip + + yield(*clusters) + end + end + end +end diff --git a/activesupport/test/multibyte_normalization_conformance.rb b/activesupport/test/multibyte_normalization_conformance.rb deleted file mode 100644 index 839aec7fa8..0000000000 --- a/activesupport/test/multibyte_normalization_conformance.rb +++ /dev/null @@ -1,129 +0,0 @@ -# encoding: utf-8 - -require 'abstract_unit' -require 'multibyte_test_helpers' - -require 'fileutils' -require 'open-uri' -require 'tmpdir' - -class Downloader - def self.download(from, to) - unless File.exist?(to) - $stderr.puts "Downloading #{from} to #{to}" - unless File.exist?(File.dirname(to)) - system "mkdir -p #{File.dirname(to)}" - end - open(from) do |source| - File.open(to, 'w') do |target| - source.each_line do |l| - target.write l - end - end - end - end - end -end - -class MultibyteNormalizationConformanceTest < ActiveSupport::TestCase - include MultibyteTestHelpers - - UNIDATA_URL = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::Unicode::UNICODE_VERSION}/ucd" - UNIDATA_FILE = '/NormalizationTest.txt' - CACHE_DIR = File.join(Dir.tmpdir, 'cache') - - def setup - FileUtils.mkdir_p(CACHE_DIR) - Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE) - @proxy = ActiveSupport::Multibyte::Chars - end - - def test_normalizations_C - each_line_of_norm_tests do |*cols| - col1, col2, col3, col4, col5, comment = *cols - - # CONFORMANCE: - # 1. The following invariants must be true for all conformant implementations - # - # NFC - # c2 == NFC(c1) == NFC(c2) == NFC(c3) - assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}" - assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}" - assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}" - # - # c4 == NFC(c4) == NFC(c5) - assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}" - end - end - - def test_normalizations_D - each_line_of_norm_tests do |*cols| - col1, col2, col3, col4, col5, comment = *cols - # - # NFD - # c3 == NFD(c1) == NFD(c2) == NFD(c3) - assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}" - assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}" - assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}" - # c5 == NFD(c4) == NFD(c5) - assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}" - end - end - - def test_normalizations_KC - each_line_of_norm_tests do | *cols | - col1, col2, col3, col4, col5, comment = *cols - # - # NFKC - # c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5) - assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}" - end - end - - def test_normalizations_KD - each_line_of_norm_tests do | *cols | - col1, col2, col3, col4, col5, comment = *cols - # - # NFKD - # c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5) - assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}" - end - end - - protected - def each_line_of_norm_tests(&block) - lines = 0 - max_test_lines = 0 # Don't limit below 38, because that's the header of the testfile - File.open(File.join(CACHE_DIR, UNIDATA_FILE), 'r') do | f | - until f.eof? || (max_test_lines > 38 and lines > max_test_lines) - lines += 1 - line = f.gets.chomp! - next if (line.empty? || line =~ /^\#/) - - cols, comment = line.split("#") - cols = cols.split(";").map{|e| e.strip}.reject{|e| e.empty? } - next unless cols.length == 5 - - # codepoints are in hex in the test suite, pack wants them as integers - cols.map!{|c| c.split.map{|codepoint| codepoint.to_i(16)}.pack("U*") } - cols << comment - - yield(*cols) - end - end - end - - def inspect_codepoints(str) - str.to_s.unpack("U*").map{|cp| cp.to_s(16) }.join(' ') - end -end diff --git a/activesupport/test/multibyte_normalization_conformance_test.rb b/activesupport/test/multibyte_normalization_conformance_test.rb new file mode 100644 index 0000000000..839aec7fa8 --- /dev/null +++ b/activesupport/test/multibyte_normalization_conformance_test.rb @@ -0,0 +1,129 @@ +# encoding: utf-8 + +require 'abstract_unit' +require 'multibyte_test_helpers' + +require 'fileutils' +require 'open-uri' +require 'tmpdir' + +class Downloader + def self.download(from, to) + unless File.exist?(to) + $stderr.puts "Downloading #{from} to #{to}" + unless File.exist?(File.dirname(to)) + system "mkdir -p #{File.dirname(to)}" + end + open(from) do |source| + File.open(to, 'w') do |target| + source.each_line do |l| + target.write l + end + end + end + end + end +end + +class MultibyteNormalizationConformanceTest < ActiveSupport::TestCase + include MultibyteTestHelpers + + UNIDATA_URL = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::Unicode::UNICODE_VERSION}/ucd" + UNIDATA_FILE = '/NormalizationTest.txt' + CACHE_DIR = File.join(Dir.tmpdir, 'cache') + + def setup + FileUtils.mkdir_p(CACHE_DIR) + Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE) + @proxy = ActiveSupport::Multibyte::Chars + end + + def test_normalizations_C + each_line_of_norm_tests do |*cols| + col1, col2, col3, col4, col5, comment = *cols + + # CONFORMANCE: + # 1. The following invariants must be true for all conformant implementations + # + # NFC + # c2 == NFC(c1) == NFC(c2) == NFC(c3) + assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}" + assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}" + assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}" + # + # c4 == NFC(c4) == NFC(c5) + assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}" + assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}" + end + end + + def test_normalizations_D + each_line_of_norm_tests do |*cols| + col1, col2, col3, col4, col5, comment = *cols + # + # NFD + # c3 == NFD(c1) == NFD(c2) == NFD(c3) + assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}" + assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}" + assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}" + # c5 == NFD(c4) == NFD(c5) + assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}" + assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}" + end + end + + def test_normalizations_KC + each_line_of_norm_tests do | *cols | + col1, col2, col3, col4, col5, comment = *cols + # + # NFKC + # c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5) + assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}" + assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}" + assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}" + assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}" + assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}" + end + end + + def test_normalizations_KD + each_line_of_norm_tests do | *cols | + col1, col2, col3, col4, col5, comment = *cols + # + # NFKD + # c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5) + assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}" + assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}" + assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}" + assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}" + assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}" + end + end + + protected + def each_line_of_norm_tests(&block) + lines = 0 + max_test_lines = 0 # Don't limit below 38, because that's the header of the testfile + File.open(File.join(CACHE_DIR, UNIDATA_FILE), 'r') do | f | + until f.eof? || (max_test_lines > 38 and lines > max_test_lines) + lines += 1 + line = f.gets.chomp! + next if (line.empty? || line =~ /^\#/) + + cols, comment = line.split("#") + cols = cols.split(";").map{|e| e.strip}.reject{|e| e.empty? } + next unless cols.length == 5 + + # codepoints are in hex in the test suite, pack wants them as integers + cols.map!{|c| c.split.map{|codepoint| codepoint.to_i(16)}.pack("U*") } + cols << comment + + yield(*cols) + end + end + end + + def inspect_codepoints(str) + str.to_s.unpack("U*").map{|cp| cp.to_s(16) }.join(' ') + end +end -- cgit v1.2.3