1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
|
# Contains all the handlers and helper classes
module ActiveSupport::Multibyte::Handlers
class EncodingError < ArgumentError; end
class Codepoint #:nodoc:
attr_accessor :code, :combining_class, :decomp_type, :decomp_mapping, :uppercase_mapping, :lowercase_mapping
end
class UnicodeDatabase #:nodoc:
attr_writer :codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252
# self-expiring methods that lazily load the Unicode database and then return the value.
[:codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252].each do |attr_name|
class_eval(<<-EOS, __FILE__, __LINE__)
def #{attr_name}
load
@#{attr_name}
end
EOS
end
# Shortcut to ucd.codepoints[]
def [](index); codepoints[index]; end
# Returns the directory in which the data files are stored
def self.dirname
File.dirname(__FILE__) + '/../../values/'
end
# Returns the filename for the data file for this version
def self.filename
File.expand_path File.join(dirname, "unicode_tables.dat")
end
# Loads the unicode database and returns all the internal objects of UnicodeDatabase
# Once the values have been loaded, define attr_reader methods for the instance variables.
def load
begin
@codepoints, @composition_exclusion, @composition_map, @boundary, @cp1252 = File.open(self.class.filename, 'rb') { |f| Marshal.load f.read }
rescue Exception => e
raise IOError.new("Couldn't load the unicode tables for UTF8Handler (#{e.message}), handler is unusable")
end
@codepoints ||= Hash.new(Codepoint.new)
@composition_exclusion ||= []
@composition_map ||= {}
@boundary ||= {}
@cp1252 ||= {}
# Redefine the === method so we can write shorter rules for grapheme cluster breaks
@boundary.each do |k,_|
@boundary[k].instance_eval do
def ===(other)
detect { |i| i === other } ? true : false
end
end if @boundary[k].kind_of?(Array)
end
# define attr_reader methods for the instance variables
class << self
attr_reader :codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252
end
end
end
# UTF8Handler implements Unicode aware operations for strings, these operations will be used by the Chars
# proxy when $KCODE is set to 'UTF8'.
class UTF8Handler
# Hangul character boundaries and properties
HANGUL_SBASE = 0xAC00
HANGUL_LBASE = 0x1100
HANGUL_VBASE = 0x1161
HANGUL_TBASE = 0x11A7
HANGUL_LCOUNT = 19
HANGUL_VCOUNT = 21
HANGUL_TCOUNT = 28
HANGUL_NCOUNT = HANGUL_VCOUNT * HANGUL_TCOUNT
HANGUL_SCOUNT = 11172
HANGUL_SLAST = HANGUL_SBASE + HANGUL_SCOUNT
HANGUL_JAMO_FIRST = 0x1100
HANGUL_JAMO_LAST = 0x11FF
# All the unicode whitespace
UNICODE_WHITESPACE = [
(0x0009..0x000D).to_a, # White_Space # Cc [5] <control-0009>..<control-000D>
0x0020, # White_Space # Zs SPACE
0x0085, # White_Space # Cc <control-0085>
0x00A0, # White_Space # Zs NO-BREAK SPACE
0x1680, # White_Space # Zs OGHAM SPACE MARK
0x180E, # White_Space # Zs MONGOLIAN VOWEL SEPARATOR
(0x2000..0x200A).to_a, # White_Space # Zs [11] EN QUAD..HAIR SPACE
0x2028, # White_Space # Zl LINE SEPARATOR
0x2029, # White_Space # Zp PARAGRAPH SEPARATOR
0x202F, # White_Space # Zs NARROW NO-BREAK SPACE
0x205F, # White_Space # Zs MEDIUM MATHEMATICAL SPACE
0x3000, # White_Space # Zs IDEOGRAPHIC SPACE
].flatten.freeze
# BOM (byte order mark) can also be seen as whitespace, it's a non-rendering character used to distinguish
# between little and big endian. This is not an issue in utf-8, so it must be ignored.
UNICODE_LEADERS_AND_TRAILERS = UNICODE_WHITESPACE + [65279] # ZERO-WIDTH NO-BREAK SPACE aka BOM
# Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site)
UTF8_PAT = /\A(?:
[\x00-\x7f] |
[\xc2-\xdf] [\x80-\xbf] |
\xe0 [\xa0-\xbf] [\x80-\xbf] |
[\xe1-\xef] [\x80-\xbf] [\x80-\xbf] |
\xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] |
[\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] |
\xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf]
)*\z/xn
# Returns a regular expression pattern that matches the passed Unicode codepoints
def self.codepoints_to_pattern(array_of_codepoints) #:nodoc:
array_of_codepoints.collect{ |e| [e].pack 'U*' }.join('|')
end
UNICODE_TRAILERS_PAT = /(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+\Z/
UNICODE_LEADERS_PAT = /\A(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+/
class << self
# ///
# /// BEGIN String method overrides
# ///
# Inserts the passed string at specified codepoint offsets
def insert(str, offset, fragment)
str.replace(
u_unpack(str).insert(
offset,
u_unpack(fragment)
).flatten.pack('U*')
)
end
# Returns the position of the passed argument in the string, counting in codepoints
def index(str, *args)
bidx = str.index(*args)
bidx ? (u_unpack(str.slice(0...bidx)).size) : nil
end
# Does Unicode-aware rstrip
def rstrip(str)
str.gsub(UNICODE_TRAILERS_PAT, '')
end
# Does Unicode-aware lstrip
def lstrip(str)
str.gsub(UNICODE_LEADERS_PAT, '')
end
# Removed leading and trailing whitespace
def strip(str)
str.gsub(UNICODE_LEADERS_PAT, '').gsub(UNICODE_TRAILERS_PAT, '')
end
# Returns the number of codepoints in the string
def size(str)
u_unpack(str).size
end
alias_method :length, :size
# Reverses codepoints in the string.
def reverse(str)
u_unpack(str).reverse.pack('U*')
end
# Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that
# character.
def slice(str, *args)
if (args.size == 2 && args.first.is_a?(Range))
raise TypeError, 'cannot convert Range into Integer' # Do as if we were native
elsif args[0].kind_of? Range
cps = u_unpack(str).slice(*args)
cps.nil? ? nil : cps.pack('U*')
elsif args.size == 1 && args[0].kind_of?(Numeric)
u_unpack(str)[args[0]]
else
u_unpack(str).slice(*args).pack('U*')
end
end
alias_method :[], :slice
# Convert characters in the string to uppercase
def upcase(str); to_case :uppercase_mapping, str; end
# Convert characters in the string to lowercase
def downcase(str); to_case :lowercase_mapping, str; end
# Returns a copy of +str+ with the first character converted to uppercase and the remainder to lowercase
def capitalize(str)
upcase(slice(str, 0..0)) + downcase(slice(str, 1..-1) || '')
end
# ///
# /// Extra String methods for unicode operations
# ///
# Returns the KC normalization of the string by default. NFKC is considered the best normalization form for
# passing strings to databases and validations.
#
# * <tt>str</tt>: The string to perform normalization on.
# * <tt>form</tt>: The form you want to normalize in. Should be one of the following: :c, :kc, :d or :kd.
def normalize(str, form=ActiveSupport::Multibyte::DEFAULT_NORMALIZATION_FORM)
# See http://www.unicode.org/reports/tr15, Table 1
codepoints = u_unpack(str)
case form
when :d
reorder_characters(decompose_codepoints(:canonical, codepoints))
when :c
compose_codepoints reorder_characters(decompose_codepoints(:canonical, codepoints))
when :kd
reorder_characters(decompose_codepoints(:compatability, codepoints))
when :kc
compose_codepoints reorder_characters(decompose_codepoints(:compatability, codepoints))
else
raise ArgumentError, "#{form} is not a valid normalization variant", caller
end.pack('U*')
end
# Perform decomposition on the characters in the string
def decompose(str)
decompose_codepoints(:canonical, u_unpack(str)).pack('U*')
end
# Perform composition on the characters in the string
def compose(str)
compose_codepoints u_unpack(str).pack('U*')
end
# ///
# /// BEGIN Helper methods for unicode operation
# ///
# Used to translate an offset from bytes to characters, for instance one received from a regular expression match
def translate_offset(str, byte_offset)
return 0 if str == ''
return nil if byte_offset.nil?
chunk = str[0..byte_offset]
begin
begin
chunk.unpack('U*').length - 1
rescue ArgumentError => e
chunk = str[0..(byte_offset+=1)]
# Stop retrying at the end of the string
raise e unless byte_offset < chunk.length
# We damaged a character, retry
retry
end
# Catch the ArgumentError so we can throw our own
rescue ArgumentError
raise EncodingError.new('malformed UTF-8 character')
end
end
# Checks if the string is valid UTF8.
def consumes?(str)
# Unpack is a little bit faster than regular expressions
begin
str.unpack('U*')
true
rescue ArgumentError
false
end
end
# Returns the number of grapheme clusters in the string. This method is very likely to be moved or renamed
# in future versions.
def g_length(str)
g_unpack(str).length
end
# Replaces all the non-utf-8 bytes by their iso-8859-1 or cp1252 equivalent resulting in a valid utf-8 string
def tidy_bytes(str)
str.split(//u).map do |c|
if !UTF8_PAT.match(c)
n = c.unpack('C')[0]
n < 128 ? n.chr :
n < 160 ? [UCD.cp1252[n] || n].pack('U') :
n < 192 ? "\xC2" + n.chr : "\xC3" + (n-64).chr
else
c
end
end.join
end
protected
# Detect whether the codepoint is in a certain character class. Primarily used by the
# grapheme cluster support.
def in_char_class?(codepoint, classes)
classes.detect { |c| UCD.boundary[c] === codepoint } ? true : false
end
# Unpack the string at codepoints boundaries
def u_unpack(str)
begin
str.unpack 'U*'
rescue ArgumentError
raise EncodingError.new('malformed UTF-8 character')
end
end
# Unpack the string at grapheme boundaries instead of codepoint boundaries
def g_unpack(str)
codepoints = u_unpack(str)
unpacked = []
pos = 0
marker = 0
eoc = codepoints.length
while(pos < eoc)
pos += 1
previous = codepoints[pos-1]
current = codepoints[pos]
if (
# CR X LF
one = ( previous == UCD.boundary[:cr] and current == UCD.boundary[:lf] ) or
# L X (L|V|LV|LVT)
two = ( UCD.boundary[:l] === previous and in_char_class?(current, [:l,:v,:lv,:lvt]) ) or
# (LV|V) X (V|T)
three = ( in_char_class?(previous, [:lv,:v]) and in_char_class?(current, [:v,:t]) ) or
# (LVT|T) X (T)
four = ( in_char_class?(previous, [:lvt,:t]) and UCD.boundary[:t] === current ) or
# X Extend
five = (UCD.boundary[:extend] === current)
)
else
unpacked << codepoints[marker..pos-1]
marker = pos
end
end
unpacked
end
# Reverse operation of g_unpack
def g_pack(unpacked)
unpacked.flatten
end
# Convert characters to a different case
def to_case(way, str)
u_unpack(str).map do |codepoint|
cp = UCD[codepoint]
unless cp.nil?
ncp = cp.send(way)
ncp > 0 ? ncp : codepoint
else
codepoint
end
end.pack('U*')
end
# Re-order codepoints so the string becomes canonical
def reorder_characters(codepoints)
length = codepoints.length- 1
pos = 0
while pos < length do
cp1, cp2 = UCD[codepoints[pos]], UCD[codepoints[pos+1]]
if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
codepoints[pos..pos+1] = cp2.code, cp1.code
pos += (pos > 0 ? -1 : 1)
else
pos += 1
end
end
codepoints
end
# Decompose composed characters to the decomposed form
def decompose_codepoints(type, codepoints)
codepoints.inject([]) do |decomposed, cp|
# if it's a hangul syllable starter character
if HANGUL_SBASE <= cp and cp < HANGUL_SLAST
sindex = cp - HANGUL_SBASE
ncp = [] # new codepoints
ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
tindex = sindex % HANGUL_TCOUNT
ncp << (HANGUL_TBASE + tindex) unless tindex == 0
decomposed.concat ncp
# if the codepoint is decomposable in with the current decomposition type
elsif (ncp = UCD[cp].decomp_mapping) and (!UCD[cp].decomp_type || type == :compatability)
decomposed.concat decompose_codepoints(type, ncp.dup)
else
decomposed << cp
end
end
end
# Compose decomposed characters to the composed form
def compose_codepoints(codepoints)
pos = 0
eoa = codepoints.length - 1
starter_pos = 0
starter_char = codepoints[0]
previous_combining_class = -1
while pos < eoa
pos += 1
lindex = starter_char - HANGUL_LBASE
# -- Hangul
if 0 <= lindex and lindex < HANGUL_LCOUNT
vindex = codepoints[starter_pos+1] - HANGUL_VBASE rescue vindex = -1
if 0 <= vindex and vindex < HANGUL_VCOUNT
tindex = codepoints[starter_pos+2] - HANGUL_TBASE rescue tindex = -1
if 0 <= tindex and tindex < HANGUL_TCOUNT
j = starter_pos + 2
eoa -= 2
else
tindex = 0
j = starter_pos + 1
eoa -= 1
end
codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
end
starter_pos += 1
starter_char = codepoints[starter_pos]
# -- Other characters
else
current_char = codepoints[pos]
current = UCD[current_char]
if current.combining_class > previous_combining_class
if ref = UCD.composition_map[starter_char]
composition = ref[current_char]
else
composition = nil
end
unless composition.nil?
codepoints[starter_pos] = composition
starter_char = composition
codepoints.delete_at pos
eoa -= 1
pos -= 1
previous_combining_class = -1
else
previous_combining_class = current.combining_class
end
else
previous_combining_class = current.combining_class
end
if current.combining_class == 0
starter_pos = pos
starter_char = codepoints[pos]
end
end
end
codepoints
end
# UniCode Database
UCD = UnicodeDatabase.new
end
end
end
|