aboutsummaryrefslogtreecommitdiffstats
path: root/activesupport/lib/active_support/multibyte
diff options
context:
space:
mode:
Diffstat (limited to 'activesupport/lib/active_support/multibyte')
-rw-r--r--activesupport/lib/active_support/multibyte/chars.rb77
-rw-r--r--activesupport/lib/active_support/multibyte/unicode.rb60
2 files changed, 69 insertions, 68 deletions
diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb
index 8152b8fd22..a1e23aeaca 100644
--- a/activesupport/lib/active_support/multibyte/chars.rb
+++ b/activesupport/lib/active_support/multibyte/chars.rb
@@ -4,7 +4,6 @@ require "active_support/json"
require "active_support/core_ext/string/access"
require "active_support/core_ext/string/behavior"
require "active_support/core_ext/module/delegation"
-require "active_support/core_ext/regexp"
module ActiveSupport #:nodoc:
module Multibyte #:nodoc:
@@ -18,7 +17,7 @@ module ActiveSupport #:nodoc:
# through the +mb_chars+ method. Methods which would normally return a
# String object now return a Chars object so methods can be chained.
#
- # 'The Perfect String '.mb_chars.downcase.strip.normalize
+ # 'The Perfect String '.mb_chars.downcase.strip
# # => #<ActiveSupport::Multibyte::Chars:0x007fdc434ccc10 @wrapped_string="the perfect string">
#
# Chars objects are perfectly interchangeable with String objects as long as
@@ -77,6 +76,11 @@ module ActiveSupport #:nodoc:
# Returns +true+ when the proxy class can handle the string. Returns
# +false+ otherwise.
def self.consumes?(string)
+ ActiveSupport::Deprecation.warn(<<-MSG.squish)
+ ActiveSupport::Multibyte::Chars.consumes? is deprecated and will be
+ removed from Rails 6.1. Use string.is_utf8? instead.
+ MSG
+
string.encoding == Encoding::UTF_8
end
@@ -109,7 +113,7 @@ module ActiveSupport #:nodoc:
#
# 'Café'.mb_chars.reverse.to_s # => 'éfaC'
def reverse
- chars(Unicode.unpack_graphemes(@wrapped_string).reverse.flatten.pack("U*"))
+ chars(@wrapped_string.scan(/\X/).reverse.join)
end
# Limits the byte size of the string to a number of bytes without breaking
@@ -118,35 +122,7 @@ module ActiveSupport #:nodoc:
#
# 'こんにちは'.mb_chars.limit(7).to_s # => "こん"
def limit(limit)
- slice(0...translate_offset(limit))
- end
-
- # Converts characters in the string to uppercase.
- #
- # 'Laurent, où sont les tests ?'.mb_chars.upcase.to_s # => "LAURENT, OÙ SONT LES TESTS ?"
- def upcase
- chars Unicode.upcase(@wrapped_string)
- end
-
- # Converts characters in the string to lowercase.
- #
- # 'VĚDA A VÝZKUM'.mb_chars.downcase.to_s # => "věda a výzkum"
- def downcase
- chars Unicode.downcase(@wrapped_string)
- end
-
- # Converts characters in the string to the opposite case.
- #
- # 'El Cañón'.mb_chars.swapcase.to_s # => "eL cAÑÓN"
- def swapcase
- chars Unicode.swapcase(@wrapped_string)
- end
-
- # Converts the first character to uppercase and the remainder to lowercase.
- #
- # 'über'.mb_chars.capitalize.to_s # => "Über"
- def capitalize
- (slice(0) || chars("")).upcase + (slice(1..-1) || chars("")).downcase
+ truncate_bytes(limit, omission: nil)
end
# Capitalizes the first letter of every word, when possible.
@@ -154,7 +130,7 @@ module ActiveSupport #:nodoc:
# "ÉL QUE SE ENTERÓ".mb_chars.titleize.to_s # => "Él Que Se Enteró"
# "日本語".mb_chars.titleize.to_s # => "日本語"
def titleize
- chars(downcase.to_s.gsub(/\b('?\S)/u) { Unicode.upcase($1) })
+ chars(downcase.to_s.gsub(/\b('?\S)/u) { $1.upcase })
end
alias_method :titlecase, :titleize
@@ -166,7 +142,24 @@ module ActiveSupport #:nodoc:
# <tt>:c</tt>, <tt>:kc</tt>, <tt>:d</tt>, or <tt>:kd</tt>. Default is
# ActiveSupport::Multibyte::Unicode.default_normalization_form
def normalize(form = nil)
- chars(Unicode.normalize(@wrapped_string, form))
+ form ||= Unicode.default_normalization_form
+
+ # See https://www.unicode.org/reports/tr15, Table 1
+ if alias_form = Unicode::NORMALIZATION_FORM_ALIASES[form]
+ ActiveSupport::Deprecation.warn(<<-MSG.squish)
+ ActiveSupport::Multibyte::Chars#normalize is deprecated and will be
+ removed from Rails 6.1. Use #unicode_normalize(:#{alias_form}) instead.
+ MSG
+
+ send(:unicode_normalize, alias_form)
+ else
+ ActiveSupport::Deprecation.warn(<<-MSG.squish)
+ ActiveSupport::Multibyte::Chars#normalize is deprecated and will be
+ removed from Rails 6.1. Use #unicode_normalize instead.
+ MSG
+
+ raise ArgumentError, "#{form} is not a valid normalization variant", caller
+ end
end
# Performs canonical decomposition on all the characters.
@@ -190,7 +183,7 @@ module ActiveSupport #:nodoc:
# 'क्षि'.mb_chars.length # => 4
# 'क्षि'.mb_chars.grapheme_length # => 3
def grapheme_length
- Unicode.unpack_graphemes(@wrapped_string).length
+ @wrapped_string.scan(/\X/).length
end
# Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent
@@ -206,7 +199,7 @@ module ActiveSupport #:nodoc:
to_s.as_json(options)
end
- %w(capitalize downcase reverse tidy_bytes upcase).each do |method|
+ %w(reverse tidy_bytes).each do |method|
define_method("#{method}!") do |*args|
@wrapped_string = send(method, *args).to_s
self
@@ -215,18 +208,6 @@ module ActiveSupport #:nodoc:
private
- def translate_offset(byte_offset)
- return nil if byte_offset.nil?
- return 0 if @wrapped_string == ""
-
- begin
- @wrapped_string.byteslice(0...byte_offset).unpack("U*").length
- rescue ArgumentError
- byte_offset -= 1
- retry
- end
- end
-
def chars(string)
self.class.new(string)
end
diff --git a/activesupport/lib/active_support/multibyte/unicode.rb b/activesupport/lib/active_support/multibyte/unicode.rb
index 4f0e1165ef..ce8ecece69 100644
--- a/activesupport/lib/active_support/multibyte/unicode.rb
+++ b/activesupport/lib/active_support/multibyte/unicode.rb
@@ -6,10 +6,17 @@ module ActiveSupport
extend self
# A list of all available normalization forms.
- # See http://www.unicode.org/reports/tr15/tr15-29.html for more
+ # See https://www.unicode.org/reports/tr15/tr15-29.html for more
# information about normalization.
NORMALIZATION_FORMS = [:c, :kc, :d, :kd]
+ NORMALIZATION_FORM_ALIASES = { # :nodoc:
+ c: :nfc,
+ d: :nfd,
+ kc: :nfkc,
+ kd: :nfkd
+ }
+
# The Unicode version that is supported by the implementation
UNICODE_VERSION = RbConfig::CONFIG["UNICODE_VERSION"]
@@ -27,6 +34,11 @@ module ActiveSupport
# Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]]
# Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]]
def unpack_graphemes(string)
+ ActiveSupport::Deprecation.warn(<<-MSG.squish)
+ ActiveSupport::Multibyte::Unicode#unpack_graphemes is deprecated and will be
+ removed from Rails 6.1. Use string.scan(/\X/).map(&:codepoints) instead.
+ MSG
+
string.scan(/\X/).map(&:codepoints)
end
@@ -34,6 +46,11 @@ module ActiveSupport
#
# Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि'
def pack_graphemes(unpacked)
+ ActiveSupport::Deprecation.warn(<<-MSG.squish)
+ ActiveSupport::Multibyte::Unicode#pack_graphemes is deprecated and will be
+ removed from Rails 6.1. Use array.flatten.pack("U*") instead.
+ MSG
+
unpacked.flatten.pack("U*")
end
@@ -100,31 +117,34 @@ module ActiveSupport
# Default is ActiveSupport::Multibyte::Unicode.default_normalization_form.
def normalize(string, form = nil)
form ||= @default_normalization_form
- # See http://www.unicode.org/reports/tr15, Table 1
- case form
- when :d
- string.unicode_normalize(:nfd)
- when :c
- string.unicode_normalize(:nfc)
- when :kd
- string.unicode_normalize(:nfkd)
- when :kc
- string.unicode_normalize(:nfkc)
+
+ # See https://www.unicode.org/reports/tr15, Table 1
+ if alias_form = NORMALIZATION_FORM_ALIASES[form]
+ ActiveSupport::Deprecation.warn(<<-MSG.squish)
+ ActiveSupport::Multibyte::Unicode#normalize is deprecated and will be
+ removed from Rails 6.1. Use String#unicode_normalize(:#{alias_form}) instead.
+ MSG
+
+ string.unicode_normalize(alias_form)
else
+ ActiveSupport::Deprecation.warn(<<-MSG.squish)
+ ActiveSupport::Multibyte::Unicode#normalize is deprecated and will be
+ removed from Rails 6.1. Use String#unicode_normalize instead.
+ MSG
+
raise ArgumentError, "#{form} is not a valid normalization variant", caller
end
end
- def downcase(string)
- string.downcase
- end
+ %w(downcase upcase swapcase).each do |method|
+ define_method(method) do |string|
+ ActiveSupport::Deprecation.warn(<<-MSG.squish)
+ ActiveSupport::Multibyte::Unicode##{method} is deprecated and
+ will be removed from Rails 6.1. Use String methods directly.
+ MSG
- def upcase(string)
- string.upcase
- end
-
- def swapcase(string)
- string.swapcase
+ string.send(method)
+ end
end
private