@@ -44,13 +44,11 @@ pub struct StringReader<'a> {
44
44
pub next_pos : BytePos ,
45
45
/// The absolute offset within the codemap of the current character
46
46
pub pos : BytePos ,
47
- /// The column of the next character to read
48
- pub col : CharPos ,
49
47
/// The current character (which has been read from self.pos)
50
48
pub ch : Option < char > ,
51
49
pub filemap : Lrc < syntax_pos:: FileMap > ,
52
- /// If Some, stop reading the source at this position (inclusive) .
53
- pub terminator : Option < BytePos > ,
50
+ /// Stop reading src at this index .
51
+ pub end_src_index : usize ,
54
52
/// Whether to record new-lines and multibyte chars in filemap.
55
53
/// This is only necessary the first time a filemap is lexed.
56
54
/// If part of a filemap is being re-lexed, this should be set to false.
@@ -61,7 +59,7 @@ pub struct StringReader<'a> {
61
59
pub fatal_errs : Vec < DiagnosticBuilder < ' a > > ,
62
60
// cache a direct reference to the source text, so that we don't have to
63
61
// retrieve it via `self.filemap.src.as_ref().unwrap()` all the time.
64
- source_text : Lrc < String > ,
62
+ src : Lrc < String > ,
65
63
/// Stack of open delimiters and their spans. Used for error message.
66
64
token : token:: Token ,
67
65
span : Span ,
@@ -113,14 +111,7 @@ impl<'a> StringReader<'a> {
113
111
self . unwrap_or_abort ( res)
114
112
}
115
113
fn is_eof ( & self ) -> bool {
116
- if self . ch . is_none ( ) {
117
- return true ;
118
- }
119
-
120
- match self . terminator {
121
- Some ( t) => self . next_pos > t,
122
- None => false ,
123
- }
114
+ self . ch . is_none ( )
124
115
}
125
116
/// Return the next token. EFFECT: advances the string_reader.
126
117
pub fn try_next_token ( & mut self ) -> Result < TokenAndSpan , ( ) > {
@@ -176,21 +167,20 @@ impl<'a> StringReader<'a> {
176
167
filemap. name) ) ;
177
168
}
178
169
179
- let source_text = ( * filemap. src . as_ref ( ) . unwrap ( ) ) . clone ( ) ;
170
+ let src = ( * filemap. src . as_ref ( ) . unwrap ( ) ) . clone ( ) ;
180
171
181
172
StringReader {
182
173
sess,
183
174
next_pos : filemap. start_pos ,
184
175
pos : filemap. start_pos ,
185
- col : CharPos ( 0 ) ,
186
176
ch : Some ( '\n' ) ,
187
177
filemap,
188
- terminator : None ,
178
+ end_src_index : src . len ( ) ,
189
179
save_new_lines_and_multibyte : true ,
190
180
// dummy values; not read
191
181
peek_tok : token:: Eof ,
192
182
peek_span : syntax_pos:: DUMMY_SP ,
193
- source_text ,
183
+ src ,
194
184
fatal_errs : Vec :: new ( ) ,
195
185
token : token:: Eof ,
196
186
span : syntax_pos:: DUMMY_SP ,
@@ -222,7 +212,7 @@ impl<'a> StringReader<'a> {
222
212
// Seek the lexer to the right byte range.
223
213
sr. save_new_lines_and_multibyte = false ;
224
214
sr. next_pos = span. lo ( ) ;
225
- sr. terminator = Some ( span. hi ( ) ) ;
215
+ sr. end_src_index = sr . src_index ( span. hi ( ) ) ;
226
216
227
217
sr. bump ( ) ;
228
218
@@ -326,9 +316,7 @@ impl<'a> StringReader<'a> {
326
316
/// offending string to the error message
327
317
fn fatal_span_verbose ( & self , from_pos : BytePos , to_pos : BytePos , mut m : String ) -> FatalError {
328
318
m. push_str ( ": " ) ;
329
- let from = self . byte_offset ( from_pos) . to_usize ( ) ;
330
- let to = self . byte_offset ( to_pos) . to_usize ( ) ;
331
- m. push_str ( & self . source_text [ from..to] ) ;
319
+ m. push_str ( & self . src [ self . src_index ( from_pos) ..self . src_index ( to_pos) ] ) ;
332
320
self . fatal_span_ ( from_pos, to_pos, & m[ ..] )
333
321
}
334
322
@@ -354,8 +342,9 @@ impl<'a> StringReader<'a> {
354
342
Ok ( ( ) )
355
343
}
356
344
357
- fn byte_offset ( & self , pos : BytePos ) -> BytePos {
358
- ( pos - self . filemap . start_pos )
345
+ #[ inline]
346
+ fn src_index ( & self , pos : BytePos ) -> usize {
347
+ ( pos - self . filemap . start_pos ) . to_usize ( )
359
348
}
360
349
361
350
/// Calls `f` with a string slice of the source text spanning from `start`
@@ -386,7 +375,7 @@ impl<'a> StringReader<'a> {
386
375
fn with_str_from_to < T , F > ( & self , start : BytePos , end : BytePos , f : F ) -> T
387
376
where F : FnOnce ( & str ) -> T
388
377
{
389
- f ( & self . source_text [ self . byte_offset ( start) . to_usize ( ) .. self . byte_offset ( end) . to_usize ( ) ] )
378
+ f ( & self . src [ self . src_index ( start) .. self . src_index ( end) ] )
390
379
}
391
380
392
381
/// Converts CRLF to LF in the given string, raising an error on bare CR.
@@ -438,47 +427,39 @@ impl<'a> StringReader<'a> {
438
427
}
439
428
}
440
429
441
-
442
430
/// Advance the StringReader by one character. If a newline is
443
431
/// discovered, add it to the FileMap's list of line start offsets.
444
432
pub fn bump ( & mut self ) {
445
- let new_pos = self . next_pos ;
446
- let new_byte_offset = self . byte_offset ( new_pos) . to_usize ( ) ;
447
- let end = self . terminator . map_or ( self . source_text . len ( ) , |t| {
448
- self . byte_offset ( t) . to_usize ( )
449
- } ) ;
450
- if new_byte_offset < end {
451
- let old_ch_is_newline = self . ch . unwrap ( ) == '\n' ;
452
- let new_ch = char_at ( & self . source_text , new_byte_offset) ;
453
- let new_ch_len = new_ch. len_utf8 ( ) ;
454
-
455
- self . ch = Some ( new_ch) ;
456
- self . pos = new_pos;
457
- self . next_pos = new_pos + Pos :: from_usize ( new_ch_len) ;
458
- if old_ch_is_newline {
433
+ let next_src_index = self . src_index ( self . next_pos ) ;
434
+ if next_src_index < self . end_src_index {
435
+ let next_ch = char_at ( & self . src , next_src_index) ;
436
+ let next_ch_len = next_ch. len_utf8 ( ) ;
437
+
438
+ if self . ch . unwrap ( ) == '\n' {
459
439
if self . save_new_lines_and_multibyte {
460
- self . filemap . next_line ( self . pos ) ;
440
+ self . filemap . next_line ( self . next_pos ) ;
461
441
}
462
- self . col = CharPos ( 0 ) ;
463
- } else {
464
- self . col = self . col + CharPos ( 1 ) ;
465
442
}
466
- if new_ch_len > 1 {
443
+ if next_ch_len > 1 {
467
444
if self . save_new_lines_and_multibyte {
468
- self . filemap . record_multibyte_char ( self . pos , new_ch_len ) ;
445
+ self . filemap . record_multibyte_char ( self . next_pos , next_ch_len ) ;
469
446
}
470
447
}
471
- self . filemap . record_width ( self . pos , new_ch) ;
448
+ self . filemap . record_width ( self . next_pos , next_ch) ;
449
+
450
+ self . ch = Some ( next_ch) ;
451
+ self . pos = self . next_pos ;
452
+ self . next_pos = self . next_pos + Pos :: from_usize ( next_ch_len) ;
472
453
} else {
473
454
self . ch = None ;
474
- self . pos = new_pos ;
455
+ self . pos = self . next_pos ;
475
456
}
476
457
}
477
458
478
459
pub fn nextch ( & self ) -> Option < char > {
479
- let offset = self . byte_offset ( self . next_pos ) . to_usize ( ) ;
480
- if offset < self . source_text . len ( ) {
481
- Some ( char_at ( & self . source_text , offset ) )
460
+ let next_src_index = self . src_index ( self . next_pos ) ;
461
+ if next_src_index < self . end_src_index {
462
+ Some ( char_at ( & self . src , next_src_index ) )
482
463
} else {
483
464
None
484
465
}
@@ -489,17 +470,15 @@ impl<'a> StringReader<'a> {
489
470
}
490
471
491
472
pub fn nextnextch ( & self ) -> Option < char > {
492
- let offset = self . byte_offset ( self . next_pos ) . to_usize ( ) ;
493
- let s = & self . source_text [ ..] ;
494
- if offset >= s. len ( ) {
495
- return None ;
496
- }
497
- let next = offset + char_at ( s, offset) . len_utf8 ( ) ;
498
- if next < s. len ( ) {
499
- Some ( char_at ( s, next) )
500
- } else {
501
- None
473
+ let next_src_index = self . src_index ( self . next_pos ) ;
474
+ if next_src_index < self . end_src_index {
475
+ let next_next_src_index =
476
+ next_src_index + char_at ( & self . src , next_src_index) . len_utf8 ( ) ;
477
+ if next_next_src_index < self . end_src_index {
478
+ return Some ( char_at ( & self . src , next_next_src_index) ) ;
479
+ }
502
480
}
481
+ None
503
482
}
504
483
505
484
pub fn nextnextch_is ( & self , c : char ) -> bool {
@@ -1359,8 +1338,8 @@ impl<'a> StringReader<'a> {
1359
1338
loop {
1360
1339
self . bump ( ) ;
1361
1340
if self . ch_is ( '\'' ) {
1362
- let start = self . byte_offset ( start) . to_usize ( ) ;
1363
- let end = self . byte_offset ( self . pos ) . to_usize ( ) ;
1341
+ let start = self . src_index ( start) ;
1342
+ let end = self . src_index ( self . pos ) ;
1364
1343
self . bump ( ) ;
1365
1344
let span = self . mk_sp ( start_with_quote, self . pos ) ;
1366
1345
self . sess . span_diagnostic
@@ -1369,8 +1348,7 @@ impl<'a> StringReader<'a> {
1369
1348
. span_suggestion ( span,
1370
1349
"if you meant to write a `str` literal, \
1371
1350
use double quotes",
1372
- format ! ( "\" {}\" " ,
1373
- & self . source_text[ start..end] ) )
1351
+ format ! ( "\" {}\" " , & self . src[ start..end] ) )
1374
1352
. emit ( ) ;
1375
1353
return Ok ( token:: Literal ( token:: Str_ ( Symbol :: intern ( "??" ) ) , None ) )
1376
1354
}
0 commit comments