|
1 |
| -module RSpec |
2 |
| - module Support |
3 |
| - # @private |
4 |
| - class EncodedString |
5 |
| - # Reduce allocations by storing constants. |
6 |
| - UTF_8 = "UTF-8" |
7 |
| - US_ASCII = "US-ASCII" |
8 |
| - # |
9 |
| - # In MRI 2.1 'invalid: :replace' changed to also replace an invalid byte sequence |
10 |
| - # see https://github.com/ruby/ruby/blob/v2_1_0/NEWS#L176 |
11 |
| - # https://www.ruby-forum.com/topic/6861247 |
12 |
| - # https://twitter.com/nalsh/status/553413844685438976 |
13 |
| - # |
14 |
| - # For example, given: |
15 |
| - # "\x80".force_encoding("Emacs-Mule").encode(:invalid => :replace).bytes.to_a |
16 |
| - # |
17 |
| - # On MRI 2.1 or above: 63 # '?' |
18 |
| - # else : 128 # "\x80" |
19 |
| - # |
20 |
| - # Ruby's default replacement string is: |
21 |
| - # U+FFFD ("\xEF\xBF\xBD"), for Unicode encoding forms, else |
22 |
| - # ? ("\x3F") |
23 |
| - REPLACE = "?" |
24 |
| - ENCODE_UNCONVERTABLE_BYTES = { |
25 |
| - :invalid => :replace, |
26 |
| - :undef => :replace, |
27 |
| - :replace => REPLACE |
28 |
| - } |
29 |
| - ENCODE_NO_CONVERTER = { |
30 |
| - :invalid => :replace, |
31 |
| - :replace => REPLACE |
32 |
| - } |
| 1 | +class EncodedString |
| 2 | + # Reduce allocations by storing constants. |
| 3 | + UTF_8 = "UTF-8" |
| 4 | + US_ASCII = "US-ASCII" |
| 5 | + # |
| 6 | + # In MRI 2.1 'invalid: :replace' changed to also replace an invalid byte sequence |
| 7 | + # see https://github.com/ruby/ruby/blob/v2_1_0/NEWS#L176 |
| 8 | + # https://www.ruby-forum.com/topic/6861247 |
| 9 | + # https://twitter.com/nalsh/status/553413844685438976 |
| 10 | + # |
| 11 | + # For example, given: |
| 12 | + # "\x80".force_encoding("Emacs-Mule").encode(:invalid => :replace).bytes.to_a |
| 13 | + # |
| 14 | + # On MRI 2.1 or above: 63 # '?' |
| 15 | + # else : 128 # "\x80" |
| 16 | + # |
| 17 | + # Ruby's default replacement string is: |
| 18 | + # U+FFFD ("\xEF\xBF\xBD"), for Unicode encoding forms, else |
| 19 | + # ? ("\x3F") |
| 20 | + REPLACE = "?" |
| 21 | + ENCODE_UNCONVERTABLE_BYTES = { |
| 22 | + :invalid => :replace, |
| 23 | + :undef => :replace, |
| 24 | + :replace => REPLACE |
| 25 | + } |
| 26 | + ENCODE_NO_CONVERTER = { |
| 27 | + :invalid => :replace, |
| 28 | + :replace => REPLACE |
| 29 | + } |
33 | 30 |
|
34 |
| - def initialize(string, encoding=nil) |
35 |
| - @encoding = encoding |
36 |
| - @source_encoding = detect_source_encoding(string) |
37 |
| - @string = matching_encoding(string) |
38 |
| - end |
39 |
| - attr_reader :source_encoding |
| 31 | + def initialize(string, encoding=nil) |
| 32 | + @encoding = encoding |
| 33 | + @source_encoding = detect_source_encoding(string) |
| 34 | + @string = matching_encoding(string) |
| 35 | + end |
| 36 | + attr_reader :source_encoding |
40 | 37 |
|
41 |
| - delegated_methods = String.instance_methods.map(&:to_s) & %w[eql? lines == encoding empty?] |
42 |
| - delegated_methods.each do |name| |
43 |
| - define_method(name) { |*args, &block| @string.__send__(name, *args, &block) } |
44 |
| - end |
| 38 | + delegated_methods = String.instance_methods.map(&:to_s) & %w[eql? lines == encoding empty?] |
| 39 | + delegated_methods.each do |name| |
| 40 | + define_method(name) { |*args, &block| @string.__send__(name, *args, &block) } |
| 41 | + end |
45 | 42 |
|
46 |
| - def <<(string) |
47 |
| - @string << matching_encoding(string) |
48 |
| - end |
| 43 | + def <<(string) |
| 44 | + @string << matching_encoding(string) |
| 45 | + end |
49 | 46 |
|
50 |
| - def split(regex_or_string) |
51 |
| - @string.split(matching_encoding(regex_or_string)) |
52 |
| - end |
| 47 | + def split(regex_or_string) |
| 48 | + @string.split(matching_encoding(regex_or_string)) |
| 49 | + end |
53 | 50 |
|
54 |
| - def to_s |
55 |
| - @string |
56 |
| - end |
57 |
| - alias :to_str :to_s |
| 51 | + def to_s |
| 52 | + @string |
| 53 | + end |
| 54 | + alias :to_str :to_s |
58 | 55 |
|
59 |
| - if String.method_defined?(:encoding) |
| 56 | + if String.method_defined?(:encoding) |
60 | 57 |
|
61 |
| - private |
| 58 | + private |
62 | 59 |
|
63 |
| - # Encoding Exceptions: |
64 |
| - # |
65 |
| - # Raised by Encoding and String methods: |
66 |
| - # Encoding::UndefinedConversionError: |
67 |
| - # when a transcoding operation fails |
68 |
| - # if the String contains characters invalid for the target encoding |
69 |
| - # e.g. "\x80".encode('UTF-8','ASCII-8BIT') |
70 |
| - # vs "\x80".encode('UTF-8','ASCII-8BIT', undef: :replace, replace: '<undef>') |
71 |
| - # # => '<undef>' |
72 |
| - # Encoding::CompatibilityError |
73 |
| - # when Encoding.compatibile?(str1, str2) is nil |
74 |
| - # e.g. utf_16le_emoji_string.split("\n") |
75 |
| - # e.g. valid_unicode_string.encode(utf8_encoding) << ascii_string |
76 |
| - # Encoding::InvalidByteSequenceError: |
77 |
| - # when the string being transcoded contains a byte invalid for |
78 |
| - # either the source or target encoding |
79 |
| - # e.g. "\x80".encode('UTF-8','US-ASCII') |
80 |
| - # vs "\x80".encode('UTF-8','US-ASCII', invalid: :replace, replace: '<byte>') |
81 |
| - # # => '<byte>' |
82 |
| - # ArgumentError |
83 |
| - # when operating on a string with invalid bytes |
84 |
| - # e.g."\x80".split("\n") |
85 |
| - # TypeError |
86 |
| - # when a symbol is passed as an encoding |
87 |
| - # Encoding.find(:"UTF-8") |
88 |
| - # when calling force_encoding on an object |
89 |
| - # that doesn't respond to #to_str |
90 |
| - # |
91 |
| - # Raised by transcoding methods: |
92 |
| - # Encoding::ConverterNotFoundError: |
93 |
| - # when a named encoding does not correspond with a known converter |
94 |
| - # e.g. 'abc'.force_encoding('UTF-8').encode('foo') |
95 |
| - # or a converter path cannot be found |
96 |
| - # e.g. "\x80".force_encoding('ASCII-8BIT').encode('Emacs-Mule') |
97 |
| - # |
98 |
| - # Raised by byte <-> char conversions |
99 |
| - # RangeError: out of char range |
100 |
| - # e.g. the UTF-16LE emoji: 128169.chr |
101 |
| - def matching_encoding(string) |
102 |
| - string = remove_invalid_bytes(string) |
103 |
| - string.encode(@encoding) |
104 |
| - rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError |
105 |
| - string.encode(@encoding, ENCODE_UNCONVERTABLE_BYTES) |
106 |
| - rescue Encoding::ConverterNotFoundError |
107 |
| - string.dup.force_encoding(@encoding).encode(ENCODE_NO_CONVERTER) |
108 |
| - end |
| 60 | + # Encoding Exceptions: |
| 61 | + # |
| 62 | + # Raised by Encoding and String methods: |
| 63 | + # Encoding::UndefinedConversionError: |
| 64 | + # when a transcoding operation fails |
| 65 | + # if the String contains characters invalid for the target encoding |
| 66 | + # e.g. "\x80".encode('UTF-8','ASCII-8BIT') |
| 67 | + # vs "\x80".encode('UTF-8','ASCII-8BIT', undef: :replace, replace: '<undef>') |
| 68 | + # # => '<undef>' |
| 69 | + # Encoding::CompatibilityError |
| 70 | + # when Encoding.compatibile?(str1, str2) is nil |
| 71 | + # e.g. utf_16le_emoji_string.split("\n") |
| 72 | + # e.g. valid_unicode_string.encode(utf8_encoding) << ascii_string |
| 73 | + # Encoding::InvalidByteSequenceError: |
| 74 | + # when the string being transcoded contains a byte invalid for |
| 75 | + # either the source or target encoding |
| 76 | + # e.g. "\x80".encode('UTF-8','US-ASCII') |
| 77 | + # vs "\x80".encode('UTF-8','US-ASCII', invalid: :replace, replace: '<byte>') |
| 78 | + # # => '<byte>' |
| 79 | + # ArgumentError |
| 80 | + # when operating on a string with invalid bytes |
| 81 | + # e.g."\x80".split("\n") |
| 82 | + # TypeError |
| 83 | + # when a symbol is passed as an encoding |
| 84 | + # Encoding.find(:"UTF-8") |
| 85 | + # when calling force_encoding on an object |
| 86 | + # that doesn't respond to #to_str |
| 87 | + # |
| 88 | + # Raised by transcoding methods: |
| 89 | + # Encoding::ConverterNotFoundError: |
| 90 | + # when a named encoding does not correspond with a known converter |
| 91 | + # e.g. 'abc'.force_encoding('UTF-8').encode('foo') |
| 92 | + # or a converter path cannot be found |
| 93 | + # e.g. "\x80".force_encoding('ASCII-8BIT').encode('Emacs-Mule') |
| 94 | + # |
| 95 | + # Raised by byte <-> char conversions |
| 96 | + # RangeError: out of char range |
| 97 | + # e.g. the UTF-16LE emoji: 128169.chr |
| 98 | + def matching_encoding(string) |
| 99 | + string = remove_invalid_bytes(string) |
| 100 | + string.encode(@encoding) |
| 101 | + rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError |
| 102 | + string.encode(@encoding, ENCODE_UNCONVERTABLE_BYTES) |
| 103 | + rescue Encoding::ConverterNotFoundError |
| 104 | + string.dup.force_encoding(@encoding).encode(ENCODE_NO_CONVERTER) |
| 105 | + end |
109 | 106 |
|
110 |
| - # Prevents raising ArgumentError |
111 |
| - if String.method_defined?(:scrub) |
112 |
| - # https://github.com/ruby/ruby/blob/eeb05e8c11/doc/NEWS-2.1.0#L120-L123 |
113 |
| - # https://github.com/ruby/ruby/blob/v2_1_0/string.c#L8242 |
114 |
| - # https://github.com/hsbt/string-scrub |
115 |
| - # https://github.com/rubinius/rubinius/blob/v2.5.2/kernel/common/string.rb#L1913-L1972 |
116 |
| - def remove_invalid_bytes(string) |
117 |
| - string.scrub(REPLACE) |
118 |
| - end |
119 |
| - else |
120 |
| - # http://stackoverflow.com/a/8711118/879854 |
121 |
| - # Loop over chars in a string replacing chars |
122 |
| - # with invalid encoding, which is a pretty good proxy |
123 |
| - # for the invalid byte sequence that causes an ArgumentError |
124 |
| - def remove_invalid_bytes(string) |
125 |
| - string.chars.map do |char| |
126 |
| - char.valid_encoding? ? char : REPLACE |
127 |
| - end.join |
128 |
| - end |
129 |
| - end |
| 107 | + # Prevents raising ArgumentError |
| 108 | + if String.method_defined?(:scrub) |
| 109 | + # https://github.com/ruby/ruby/blob/eeb05e8c11/doc/NEWS-2.1.0#L120-L123 |
| 110 | + # https://github.com/ruby/ruby/blob/v2_1_0/string.c#L8242 |
| 111 | + # https://github.com/hsbt/string-scrub |
| 112 | + # https://github.com/rubinius/rubinius/blob/v2.5.2/kernel/common/string.rb#L1913-L1972 |
| 113 | + def remove_invalid_bytes(string) |
| 114 | + string.scrub(REPLACE) |
| 115 | + end |
| 116 | + else |
| 117 | + # http://stackoverflow.com/a/8711118/879854 |
| 118 | + # Loop over chars in a string replacing chars |
| 119 | + # with invalid encoding, which is a pretty good proxy |
| 120 | + # for the invalid byte sequence that causes an ArgumentError |
| 121 | + def remove_invalid_bytes(string) |
| 122 | + string.chars.map do |char| |
| 123 | + char.valid_encoding? ? char : REPLACE |
| 124 | + end.join |
| 125 | + end |
| 126 | + end |
130 | 127 |
|
131 |
| - def detect_source_encoding(string) |
132 |
| - string.encoding |
133 |
| - end |
| 128 | + def detect_source_encoding(string) |
| 129 | + string.encoding |
| 130 | + end |
134 | 131 |
|
135 |
| - def self.pick_encoding(source_a, source_b) |
136 |
| - Encoding.compatible?(source_a, source_b) || Encoding.default_external |
137 |
| - end |
138 |
| - else |
| 132 | + def self.pick_encoding(source_a, source_b) |
| 133 | + Encoding.compatible?(source_a, source_b) || Encoding.default_external |
| 134 | + end |
| 135 | + else |
139 | 136 |
|
140 |
| - def self.pick_encoding(_source_a, _source_b) |
141 |
| - end |
| 137 | + def self.pick_encoding(_source_a, _source_b) |
| 138 | + end |
142 | 139 |
|
143 |
| - private |
| 140 | + private |
144 | 141 |
|
145 |
| - def matching_encoding(string) |
146 |
| - string |
147 |
| - end |
| 142 | + def matching_encoding(string) |
| 143 | + string |
| 144 | + end |
148 | 145 |
|
149 |
| - def detect_source_encoding(_string) |
150 |
| - US_ASCII |
151 |
| - end |
152 |
| - end |
| 146 | + def detect_source_encoding(_string) |
| 147 | + US_ASCII |
153 | 148 | end
|
154 | 149 | end
|
155 | 150 | end
|
0 commit comments