Skip to content

Commit b3e8c8b

Browse files
matkladmatklad
committed
adapt rustdoc to infailable lexer
1 parent 58ac81a commit b3e8c8b

File tree

3 files changed

+177
-50
lines changed

3 files changed

+177
-50
lines changed

‎src/librustdoc/html/highlight.rs

+14-13
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ pub fn render_with_highlighting(
4444

4545
let mut highlighted_source = vec![];
4646
if classifier.write_source(&mut highlighted_source).is_err() {
47-
Err(classifier.lexer.buffer_fatal_errors())
47+
Err(())
4848
} else {
4949
Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
5050
}
@@ -59,14 +59,9 @@ pub fn render_with_highlighting(
5959
}
6060
write_footer(&mut out).unwrap();
6161
}
62-
Err(errors) => {
63-
// If errors are encountered while trying to highlight, cancel the errors and just emit
64-
// the unhighlighted source. The errors will have already been reported in the
65-
// `check-code-block-syntax` pass.
66-
for mut error in errors {
67-
error.cancel();
68-
}
69-
62+
Err(()) => {
63+
// If errors are encountered while trying to highlight, just emit
64+
// the unhighlighted source.
7065
write!(out, "<pre><code>{}</code></pre>", src).unwrap();
7166
}
7267
}
@@ -192,14 +187,20 @@ impl<'a> Classifier<'a> {
192187
if let Some(token) = self.peek_token.take() {
193188
return Ok(token);
194189
}
195-
self.lexer.try_next_token().map_err(|()| HighlightError::LexError)
190+
let token = self.lexer.next_token();
191+
if let token::Unknown(..) = &token.kind {
192+
return Err(HighlightError::LexError);
193+
}
194+
Ok(token)
196195
}
197196

198197
fn peek(&mut self) -> Result<&Token, HighlightError> {
199198
if self.peek_token.is_none() {
200-
self.peek_token = Some(
201-
self.lexer.try_next_token().map_err(|()| HighlightError::LexError)?
202-
);
199+
let token = self.lexer.next_token();
200+
if let token::Unknown(..) = &token.kind {
201+
return Err(HighlightError::LexError);
202+
}
203+
self.peek_token = Some(token);
203204
}
204205
Ok(self.peek_token.as_ref().unwrap())
205206
}

‎src/librustdoc/passes/check_code_block_syntax.rs

+9-23
Original file line numberDiff line numberDiff line change
@@ -32,24 +32,20 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
3232
dox[code_block.code].to_owned(),
3333
);
3434

35-
let errors = {
35+
let has_errors = {
36+
let mut has_errors = false;
3637
let mut lexer = Lexer::new(&sess, source_file, None);
37-
while let Ok(token::Token { kind, .. }) = lexer.try_next_token() {
38-
if kind == token::Eof {
39-
break;
38+
loop {
39+
match lexer.next_token().kind {
40+
token::Eof => break,
41+
token::Unknown(..) => has_errors = true,
42+
_ => (),
4043
}
4144
}
42-
43-
let errors = lexer.buffer_fatal_errors();
44-
45-
if !errors.is_empty() {
46-
Err(errors)
47-
} else {
48-
Ok(())
49-
}
45+
has_errors
5046
};
5147

52-
if let Err(errors) = errors {
48+
if has_errors {
5349
let mut diag = if let Some(sp) =
5450
super::source_span_for_markdown_range(self.cx, &dox, &code_block.range, &item.attrs)
5551
{
@@ -58,11 +54,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
5854
.sess()
5955
.struct_span_warn(sp, "could not parse code block as Rust code");
6056

61-
for mut err in errors {
62-
diag.note(&format!("error from rustc: {}", err.message()));
63-
err.cancel();
64-
}
65-
6657
if code_block.syntax.is_none() && code_block.is_fenced {
6758
let sp = sp.from_inner(InnerSpan::new(0, 3));
6859
diag.span_suggestion(
@@ -82,11 +73,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
8273
"doc comment contains an invalid Rust code block",
8374
);
8475

85-
for mut err in errors {
86-
// Don't bother reporting the error, because we can't show where it happened.
87-
err.cancel();
88-
}
89-
9076
if code_block.syntax.is_none() && code_block.is_fenced {
9177
diag.help("mark blocks that do not contain Rust code as text: ```text");
9278
}

‎src/test/rustdoc-ui/invalid-syntax.stderr

+154-14
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,21 @@
1+
error: unknown start of token: \
2+
--> <doctest>:1:1
3+
|
4+
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
5+
| ^
6+
7+
error: unknown start of token: \
8+
--> <doctest>:1:43
9+
|
10+
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
11+
| ^
12+
13+
error: unknown start of token: \
14+
--> <doctest>:1:60
15+
|
16+
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
17+
| ^
18+
119
warning: could not parse code block as Rust code
220
--> $DIR/invalid-syntax.rs:3:5
321
|
@@ -6,13 +24,31 @@ LL | /// ```
624
LL | | /// \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
725
LL | | /// ```
826
| |_______^
9-
|
10-
= note: error from rustc: unknown start of token: \
1127
help: mark blocks that do not contain Rust code as text
1228
|
1329
LL | /// ```text
1430
| ^^^^^^^
1531

32+
error: unknown start of token: `
33+
--> <doctest>:3:30
34+
|
35+
3 | | ^^^^^^ did you mean `baz::foobar`?
36+
| ^
37+
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
38+
|
39+
3 | | ^^^^^^ did you mean 'baz::foobar`?
40+
| ^
41+
42+
error: unknown start of token: `
43+
--> <doctest>:3:42
44+
|
45+
3 | | ^^^^^^ did you mean `baz::foobar`?
46+
| ^
47+
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
48+
|
49+
3 | | ^^^^^^ did you mean `baz::foobar'?
50+
| ^
51+
1652
warning: could not parse code block as Rust code
1753
--> $DIR/invalid-syntax.rs:8:5
1854
|
@@ -23,13 +59,17 @@ LL | | /// LL | use foobar::Baz;
2359
LL | | /// | ^^^^^^ did you mean `baz::foobar`?
2460
LL | | /// ```
2561
| |_______^
26-
|
27-
= note: error from rustc: unknown start of token: `
2862
help: mark blocks that do not contain Rust code as text
2963
|
3064
LL | /// ```text
3165
| ^^^^^^^
3266

67+
error: unknown start of token: \
68+
--> <doctest>:1:1
69+
|
70+
1 | \_
71+
| ^
72+
3373
warning: could not parse code block as Rust code
3474
--> $DIR/invalid-syntax.rs:19:5
3575
|
@@ -38,13 +78,17 @@ LL | /// ```
3878
LL | | /// \_
3979
LL | | /// ```
4080
| |_______^
41-
|
42-
= note: error from rustc: unknown start of token: \
4381
help: mark blocks that do not contain Rust code as text
4482
|
4583
LL | /// ```text
4684
| ^^^^^^^
4785

86+
error: unknown start of token: \
87+
--> <doctest>:1:1
88+
|
89+
1 | \_
90+
| ^
91+
4892
warning: could not parse code block as Rust code
4993
--> $DIR/invalid-syntax.rs:32:5
5094
|
@@ -53,8 +97,12 @@ LL | /// ```rust
5397
LL | | /// \_
5498
LL | | /// ```
5599
| |_______^
56-
|
57-
= note: error from rustc: unknown start of token: \
100+
101+
error: unknown start of token: \
102+
--> <doctest>:2:5
103+
|
104+
2 | \_
105+
| ^
58106

59107
warning: could not parse code block as Rust code
60108
--> $DIR/invalid-syntax.rs:41:9
@@ -63,16 +111,48 @@ LL | /// code with bad syntax
63111
| _________^
64112
LL | | /// \_
65113
| |__________^
66-
|
67-
= note: error from rustc: unknown start of token: \
114+
115+
error: unknown start of token: `
116+
--> <doctest>:1:1
117+
|
118+
1 | ```
119+
| ^
120+
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
121+
|
122+
1 | '``
123+
| ^
124+
125+
error: unknown start of token: `
126+
--> <doctest>:1:2
127+
|
128+
1 | ```
129+
| ^
130+
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
131+
|
132+
1 | `'`
133+
| ^
134+
135+
error: unknown start of token: `
136+
--> <doctest>:1:3
137+
|
138+
1 | ```
139+
| ^
140+
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
141+
|
142+
1 | ``'
143+
| ^
68144

69145
warning: could not parse code block as Rust code
70146
--> $DIR/invalid-syntax.rs:55:9
71147
|
72148
LL | /// ```
73149
| ^^^
74-
|
75-
= note: error from rustc: unknown start of token: `
150+
151+
error: unknown start of token: \
152+
--> <doctest>:1:1
153+
|
154+
1 | \_
155+
| ^
76156

77157
warning: could not parse code block as Rust code
78158
--> $DIR/invalid-syntax.rs:58:5
@@ -82,8 +162,12 @@ LL | /// ```edition2018
82162
LL | | /// \_
83163
LL | | /// ```
84164
| |_______^
85-
|
86-
= note: error from rustc: unknown start of token: \
165+
166+
error: unknown start of token: \
167+
--> <doctest>:1:1
168+
|
169+
1 | \_
170+
| ^
87171

88172
warning: doc comment contains an invalid Rust code block
89173
--> $DIR/invalid-syntax.rs:63:1
@@ -95,3 +179,59 @@ LL | | #[doc = "```"]
95179
|
96180
= help: mark blocks that do not contain Rust code as text: ```text
97181

182+
error: unknown start of token: \
183+
--> <rustdoc-highlighting>:1:1
184+
|
185+
1 | \_
186+
| ^
187+
188+
error: unknown start of token: \
189+
--> <rustdoc-highlighting>:1:1
190+
|
191+
1 | \_
192+
| ^
193+
194+
error: unknown start of token: `
195+
--> <rustdoc-highlighting>:1:1
196+
|
197+
1 | ```
198+
| ^
199+
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
200+
|
201+
1 | '``
202+
| ^
203+
204+
error: unknown start of token: \
205+
--> <rustdoc-highlighting>:2:1
206+
|
207+
2 | \_
208+
| ^
209+
210+
error: unknown start of token: \
211+
--> <rustdoc-highlighting>:1:1
212+
|
213+
1 | \_
214+
| ^
215+
216+
error: unknown start of token: \
217+
--> <rustdoc-highlighting>:1:1
218+
|
219+
1 | \_
220+
| ^
221+
222+
error: unknown start of token: `
223+
--> <rustdoc-highlighting>:3:30
224+
|
225+
3 | | ^^^^^^ did you mean `baz::foobar`?
226+
| ^
227+
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
228+
|
229+
3 | | ^^^^^^ did you mean 'baz::foobar`?
230+
| ^
231+
232+
error: unknown start of token: \
233+
--> <rustdoc-highlighting>:1:1
234+
|
235+
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
236+
| ^
237+

0 commit comments

Comments
 (0)