diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs index b9781581ff75d..5bb8d91234180 100644 --- a/compiler/rustc_lexer/src/lib.rs +++ b/compiler/rustc_lexer/src/lib.rs @@ -66,6 +66,9 @@ pub enum TokenKind { Ident, /// "r#ident" RawIdent, + /// "k#ident" + /// Unstable feature handling is done in rustc_parse/src/lexer/mod.rs + RawKeyword, /// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details. Literal { kind: LiteralKind, suffix_start: usize }, /// "'a" @@ -361,6 +364,9 @@ impl Cursor<'_> { _ => self.ident(), }, + // Raw keyword + 'k' if self.first() == '#' && is_id_start(self.second()) => self.raw_keyword(), + // Identifier (this should be checked after other variant that can // start as identifier). c if is_id_start(c) => self.ident(), @@ -487,6 +493,15 @@ impl Cursor<'_> { RawIdent } + fn raw_keyword(&mut self) -> TokenKind { + debug_assert!(self.prev() == 'k' && self.first() == '#' && is_id_start(self.second())); + // Eat "#" symbol. + self.bump(); + // Eat the identifier part of the keyword. + self.eat_identifier(); + RawKeyword + } + fn ident(&mut self) -> TokenKind { debug_assert!(is_id_start(self.prev())); // Start is already eaten, eat the rest of identifier. diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index bd8dfd678a97b..851708bb7bf94 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -187,6 +187,25 @@ impl<'a> StringReader<'a> { } token::Ident(sym, is_raw_ident) } + rustc_lexer::TokenKind::RawKeyword => { + let span = self.mk_sp(start, self.pos); + self.sess + .span_diagnostic + .struct_span_err(span, "raw keyword syntax is reserved for future use") + // With whitespace, it tokenizes as three tokens, just like before this reservation. + .span_suggestion( + span, + "insert a whitespace", + format!("k #{}", self.str_from(start + BytePos(2))), + Applicability::MachineApplicable, + ) + .emit(); + + let sym = nfc_normalize(self.str_from(start + BytePos(2))); + // For recovery, treat it as a raw ident. + // FIXME: Add a flag to Ident to mark it as a raw keyword. + token::Ident(sym, true) + } rustc_lexer::TokenKind::Literal { kind, suffix_start } => { let suffix_start = start + BytePos(suffix_start as u32); let (kind, symbol) = self.cook_lexer_literal(start, suffix_start, kind); diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 0030ef67c075d..32899677c12f6 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -408,6 +408,7 @@ impl<'a> Classifier<'a> { c => c, }, TokenKind::RawIdent => Class::Ident, + TokenKind::RawKeyword => Class::KeyWord, TokenKind::Lifetime { .. } => Class::Lifetime, }; // Anything that didn't return above is the simple case where we the diff --git a/src/test/ui/parser/raw/raw_keywords.rs b/src/test/ui/parser/raw/raw_keywords.rs new file mode 100644 index 0000000000000..038f6da108282 --- /dev/null +++ b/src/test/ui/parser/raw/raw_keywords.rs @@ -0,0 +1,82 @@ +// check-fail + +macro_rules! one_token { ( $a:tt ) => { }; } +macro_rules! two_tokens { ( $a:tt $b:tt ) => { }; } +macro_rules! three_tokens { ( $a:tt $b:tt $c:tt) => { }; } +macro_rules! four_tokens { ( $a:tt $b:tt $c:tt $d:tt ) => { }; } + +fn main() { + // Intended raw keyword syntax + one_token!(k#ident); + //~^ ERROR raw keyword syntax is reserved for future use + one_token!(k#fn); + //~^ ERROR raw keyword syntax is reserved for future use + one_token!(k#Self); + //~^ ERROR raw keyword syntax is reserved for future use + one_token!(k#dyn); + //~^ ERROR raw keyword syntax is reserved for future use + one_token!(k#await); + //~^ ERROR raw keyword syntax is reserved for future use + + // This was previously valid in `quote!` macros, where `k` was a variable. + two_tokens!(#k#other_variable); + //~^ ERROR raw keyword syntax is reserved for future use + + // weird interactions with raw literals. + two_tokens!(k#r"raw string"); + //~^ ERROR raw keyword syntax is reserved for future use + two_tokens!(k#r "raw string"); + //~^ ERROR raw keyword syntax is reserved for future use + two_tokens!(k#br"raw byte string"); + //~^ ERROR raw keyword syntax is reserved for future use + three_tokens!(k#r"raw string"#); + //~^ ERROR raw keyword syntax is reserved for future use + four_tokens!(k#r#"raw string"#); + //~^ ERROR raw keyword syntax is reserved for future use + two_tokens!(k#b'-'); + //~^ ERROR raw keyword syntax is reserved for future use + two_tokens!(k#b '-'); + //~^ ERROR raw keyword syntax is reserved for future use + + // weird raw idents + one_token!(r#k); + two_tokens!(r#k#); + three_tokens!(r#k#ident); + two_tokens!(r#k ident); + three_tokens!(r#k#r); + three_tokens!(r#k#r#ident); + + // Inserting a whitespace gets you the previous lexing behaviour + three_tokens!(k# ident); + three_tokens!(k #ident); + three_tokens!(k # ident); + + // These should work anyway + two_tokens!(k#); + two_tokens!(k #); + three_tokens!(k#123); + three_tokens!(k#"normal string"); + three_tokens!(k#'c'); + three_tokens!(k#'lifetime); + three_tokens!(k#[doc("doc attribute")]); + + #[cfg(False)] + mod inside_cfg_false { + const k#SOMETHING: i32 = 42; + //~^ ERROR raw keyword syntax is reserved for future use + } + + fn returns_k() -> i32 { + let k = 42; + k#k + //~^ ERROR raw keyword syntax is reserved for future use + } + + + // Test that the token counting macros actually count the right amount of tokens. + two_tokens!(k#knampf); + //~^ ERROR unexpected end of macro invocation + //~^^ ERROR raw keyword syntax is reserved for future use + two_tokens!(k # knampf); + //~^ ERROR no rules expected the token `knampf` +} diff --git a/src/test/ui/parser/raw/raw_keywords.stderr b/src/test/ui/parser/raw/raw_keywords.stderr new file mode 100644 index 0000000000000..689d5d641f0c6 --- /dev/null +++ b/src/test/ui/parser/raw/raw_keywords.stderr @@ -0,0 +1,116 @@ +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:10:16 + | +LL | one_token!(k#ident); + | ^^^^^^^ help: insert a whitespace: `k #ident` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:12:16 + | +LL | one_token!(k#fn); + | ^^^^ help: insert a whitespace: `k #fn` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:14:16 + | +LL | one_token!(k#Self); + | ^^^^^^ help: insert a whitespace: `k #Self` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:16:16 + | +LL | one_token!(k#dyn); + | ^^^^^ help: insert a whitespace: `k #dyn` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:18:16 + | +LL | one_token!(k#await); + | ^^^^^^^ help: insert a whitespace: `k #await` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:22:18 + | +LL | two_tokens!(#k#other_variable); + | ^^^^^^^^^^^^^^^^ help: insert a whitespace: `k #other_variable` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:26:17 + | +LL | two_tokens!(k#r"raw string"); + | ^^^ help: insert a whitespace: `k #r` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:28:17 + | +LL | two_tokens!(k#r "raw string"); + | ^^^ help: insert a whitespace: `k #r` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:30:17 + | +LL | two_tokens!(k#br"raw byte string"); + | ^^^^ help: insert a whitespace: `k #br` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:32:19 + | +LL | three_tokens!(k#r"raw string"#); + | ^^^ help: insert a whitespace: `k #r` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:34:18 + | +LL | four_tokens!(k#r#"raw string"#); + | ^^^ help: insert a whitespace: `k #r` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:36:17 + | +LL | two_tokens!(k#b'-'); + | ^^^ help: insert a whitespace: `k #b` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:38:17 + | +LL | two_tokens!(k#b '-'); + | ^^^ help: insert a whitespace: `k #b` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:65:15 + | +LL | const k#SOMETHING: i32 = 42; + | ^^^^^^^^^^^ help: insert a whitespace: `k #SOMETHING` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:71:9 + | +LL | k#k + | ^^^ help: insert a whitespace: `k #k` + +error: raw keyword syntax is reserved for future use + --> $DIR/raw_keywords.rs:77:17 + | +LL | two_tokens!(k#knampf); + | ^^^^^^^^ help: insert a whitespace: `k #knampf` + +error: unexpected end of macro invocation + --> $DIR/raw_keywords.rs:77:25 + | +LL | macro_rules! two_tokens { ( $a:tt $b:tt ) => { }; } + | ----------------------- when calling this macro +... +LL | two_tokens!(k#knampf); + | ^ missing tokens in macro arguments + +error: no rules expected the token `knampf` + --> $DIR/raw_keywords.rs:80:21 + | +LL | macro_rules! two_tokens { ( $a:tt $b:tt ) => { }; } + | ----------------------- when calling this macro +... +LL | two_tokens!(k # knampf); + | ^^^^^^ no rules expected this token in macro call + +error: aborting due to 18 previous errors +