Skip to content

Commit bd3529a

Browse files
nnethercotennethercote
committed
XXX: ban/handle OpenDelim(Invis) wherever Interpolated/OpenDelim is allowed
1 parent dcdbb88 commit bd3529a

File tree

18 files changed

+179
-10
lines changed

18 files changed

+179
-10
lines changed

‎compiler/rustc_ast/src/attr/mod.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,12 @@ impl MetaItem {
347347
token::Nonterminal::NtPath(path) => (**path).clone(),
348348
_ => return None,
349349
},
350+
Some(TokenTree::Token(
351+
Token { kind: token::OpenDelim(Delimiter::Invisible(_)), .. },
352+
_,
353+
)) => {
354+
panic!("njn: invis-delim?");
355+
}
350356
_ => return None,
351357
};
352358
let list_closing_paren_pos = tokens.peek().map(|tt| tt.span().hi());

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -789,6 +789,9 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
789789
vis.visit_span(sp);
790790
visit_nonterminal(nt, vis);
791791
}
792+
token::OpenDelim(token::Delimiter::Invisible(_)) => {
793+
panic!("njn: invis-delim?");
794+
}
792795
_ => {}
793796
}
794797
vis.visit_span(span);

‎compiler/rustc_ast/src/token.rs

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ impl Lit {
139139
match token.uninterpolate().kind {
140140
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
141141
Literal(token_lit) => Some(token_lit),
142+
// njn: deal with later, with NtExpr/NtLiteral
142143
Interpolated(ref nt)
143144
if let NtExpr(expr) | NtLiteral(expr) = &nt.0
144145
&& let ast::ExprKind::Lit(token_lit) = expr.kind =>
@@ -380,6 +381,7 @@ impl Clone for TokenKind {
380381
// a copy. This is faster than the `derive(Clone)` version which has a
381382
// separate path for every variant.
382383
match self {
384+
// njn: nothing needed here
383385
Interpolated(nt) => Interpolated(nt.clone()),
384386
_ => unsafe { std::ptr::read(self) },
385387
}
@@ -470,6 +472,7 @@ impl Token {
470472
/// if they keep spans or perform edition checks.
471473
pub fn uninterpolated_span(&self) -> Span {
472474
match &self.kind {
475+
// njn: nothing needed here until Ident/Lifetime are done
473476
Interpolated(nt) => nt.0.use_span(),
474477
_ => self.span,
475478
}
@@ -487,6 +490,7 @@ impl Token {
487490
true
488491
}
489492

493+
// njn: no change needed
490494
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
491495
| Lifetime(..) | Interpolated(..) | Eof => false,
492496
}
@@ -513,13 +517,19 @@ impl Token {
513517
// DotDotDot is no longer supported, but we need some way to display the error
514518
DotDot | DotDotDot | DotDotEq | // range notation
515519
Lt | BinOp(Shl) | // associated path
516-
PathSep | // global path
520+
PathSep | // global path
517521
Lifetime(..) | // labeled loop
518522
Pound => true, // expression attributes
519523
Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) |
520524
NtExpr(..) |
521525
NtBlock(..) |
522526
NtPath(..)),
527+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
528+
NonterminalKind::Block |
529+
NonterminalKind::Expr |
530+
NonterminalKind::Literal |
531+
NonterminalKind::Path
532+
))) => true,
523533
_ => false,
524534
}
525535
}
@@ -539,11 +549,18 @@ impl Token {
539549
// DotDotDot is no longer supported
540550
| DotDot | DotDotDot | DotDotEq // ranges
541551
| Lt | BinOp(Shl) // associated path
542-
| PathSep => true, // global path
552+
| PathSep => true, // global path
543553
Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) |
544554
NtPat(..) |
545555
NtBlock(..) |
546556
NtPath(..)),
557+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
558+
NonterminalKind::Block |
559+
NonterminalKind::PatParam { .. } |
560+
NonterminalKind::PatWithOr |
561+
NonterminalKind::Path |
562+
NonterminalKind::Literal
563+
))) => true,
547564
_ => false,
548565
}
549566
}
@@ -564,6 +581,10 @@ impl Token {
564581
Lt | BinOp(Shl) | // associated path
565582
PathSep => true, // global path
566583
Interpolated(ref nt) => matches!(&nt.0, NtTy(..) | NtPath(..)),
584+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
585+
NonterminalKind::Ty |
586+
NonterminalKind::Path
587+
))) => true,
567588
// For anonymous structs or unions, which only appear in specific positions
568589
// (type of struct fields or union fields), we don't consider them as regular types
569590
_ => false,
@@ -575,6 +596,9 @@ impl Token {
575596
match self.kind {
576597
OpenDelim(Delimiter::Brace) => true,
577598
Interpolated(ref nt) => matches!(&nt.0, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
599+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
600+
NonterminalKind::Expr | NonterminalKind::Block | NonterminalKind::Literal,
601+
))) => true,
578602
_ => self.can_begin_literal_maybe_minus(),
579603
}
580604
}
@@ -630,6 +654,10 @@ impl Token {
630654
},
631655
_ => false,
632656
},
657+
// njn: too simple compared to what's above?
658+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
659+
NonterminalKind::Literal | NonterminalKind::Expr,
660+
))) => true,
633661
_ => false,
634662
}
635663
}
@@ -640,6 +668,7 @@ impl Token {
640668
/// otherwise returns the original token.
641669
pub fn uninterpolate(&self) -> Cow<'_, Token> {
642670
match &self.kind {
671+
// njn: nothing needed here, just change when NtIdent is removed
643672
Interpolated(nt) => match &nt.0 {
644673
NtIdent(ident, is_raw) => {
645674
Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span))
@@ -657,6 +686,7 @@ impl Token {
657686
// We avoid using `Token::uninterpolate` here because it's slow.
658687
match &self.kind {
659688
&Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
689+
// njn: nothing needed here, just change when NtIdent is removed
660690
Interpolated(nt) => match &nt.0 {
661691
NtIdent(ident, is_raw) => Some((*ident, *is_raw)),
662692
_ => None,
@@ -671,6 +701,7 @@ impl Token {
671701
// We avoid using `Token::uninterpolate` here because it's slow.
672702
match &self.kind {
673703
&Lifetime(name) => Some(Ident::new(name, self.span)),
704+
// njn: nothing needed here, just change when NtLifetime is removed
674705
Interpolated(nt) => match &nt.0 {
675706
NtLifetime(ident) => Some(*ident),
676707
_ => None,
@@ -697,6 +728,7 @@ impl Token {
697728

698729
/// Returns `true` if the token is an interpolated path.
699730
fn is_whole_path(&self) -> bool {
731+
// njn: nothing needed, just deal with NtPath later
700732
if let Interpolated(nt) = &self.kind
701733
&& let NtPath(..) = &nt.0
702734
{
@@ -710,6 +742,7 @@ impl Token {
710742
/// That is, is this a pre-parsed expression dropped into the token stream
711743
/// (which happens while parsing the result of macro expansion)?
712744
pub fn is_whole_expr(&self) -> bool {
745+
// njn: nothing needed, just deal with NtExpr/NtLiteral/NtPath/NtBlock later
713746
if let Interpolated(nt) = &self.kind
714747
&& let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = &nt.0
715748
{
@@ -721,6 +754,7 @@ impl Token {
721754

722755
/// Is the token an interpolated block (`$b:block`)?
723756
pub fn is_whole_block(&self) -> bool {
757+
// njn: nothing needed, just deal with NtBlock later
724758
if let Interpolated(nt) = &self.kind
725759
&& let NtBlock(..) = &nt.0
726760
{
@@ -861,6 +895,7 @@ impl Token {
861895
_ => return None,
862896
},
863897

898+
// njn: nothing needed here
864899
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
865900
| DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar
866901
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,9 @@ impl TokenStream {
500500
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
501501
TokenStream::from_nonterminal_ast(&nt.0).flattened(),
502502
),
503+
token::OpenDelim(Delimiter::Invisible(_)) => {
504+
panic!("njn: invis-delim?");
505+
}
503506
_ => TokenTree::Token(token.clone(), spacing),
504507
}
505508
}
@@ -517,7 +520,15 @@ impl TokenStream {
517520
pub fn flattened(&self) -> TokenStream {
518521
fn can_skip(stream: &TokenStream) -> bool {
519522
stream.trees().all(|tree| match tree {
520-
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
523+
TokenTree::Token(token, _) => {
524+
if let token::Interpolated(_) = token.kind {
525+
false
526+
} else if let token::OpenDelim(Delimiter::Invisible(_)) = token.kind {
527+
panic!("njn: invis-delim?");
528+
} else {
529+
true
530+
}
531+
}
521532
TokenTree::Delimited(.., inner) => can_skip(inner),
522533
})
523534
}

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -925,6 +925,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
925925
}
926926
token::Eof => "<eof>".into(),
927927

928+
// njn: handled above
928929
token::Interpolated(ref nt) => self.nonterminal_to_string(&nt.0).into(),
929930
}
930931
}

‎compiler/rustc_expand/src/config.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,11 @@ impl<'a> StripUnconfigured<'a> {
204204
{
205205
panic!("Nonterminal should have been flattened at {:?}: {:?}", token.span, nt);
206206
}
207+
AttrTokenTree::Token(ref token, _)
208+
if let TokenKind::OpenDelim(Delimiter::Invisible(_)) = &token.kind =>
209+
{
210+
panic!("njn: invis-delim?");
211+
}
207212
AttrTokenTree::Token(token, spacing) => {
208213
Some(AttrTokenTree::Token(token, spacing)).into_iter()
209214
}

‎compiler/rustc_expand/src/mbe/diagnostics.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,13 @@ pub(super) fn failed_to_match_macro<'cx>(
8484
}
8585
}
8686

87+
if let MatcherLoc::Token { token: expected_token } = &remaining_matcher
88+
&& (matches!(expected_token.kind, TokenKind::OpenDelim(token::Delimiter::Invisible(_)))
89+
|| matches!(token.kind, TokenKind::OpenDelim(token::Delimiter::Invisible(_))))
90+
{
91+
panic!("njn: invis-delim?");
92+
}
93+
8794
// Check whether there's a missing comma in this macro call, like `println!("{}" a);`
8895
if let Some((arg, comma_span)) = arg.add_comma() {
8996
for lhs in lhses {

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
286286
}
287287
}
288288

289+
// njn: covers invis-delim
289290
OpenDelim(..) | CloseDelim(..) => unreachable!(),
290291
Eof => unreachable!(),
291292
}

‎compiler/rustc_parse/src/lexer/tokentrees.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> {
4242
let mut buf = Vec::new();
4343
loop {
4444
match self.token.kind {
45+
token::OpenDelim(Delimiter::Invisible(_)) => {
46+
panic!("njn: invis-delim?");
47+
}
4548
token::OpenDelim(delim) => {
4649
buf.push(match self.parse_token_tree_open_delim(delim) {
4750
Ok(val) => val,

‎compiler/rustc_parse/src/parser/attr.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,9 @@ impl<'a> Parser<'a> {
375375
None => self.unexpected()?,
376376
}
377377
}
378+
if let token::OpenDelim(Delimiter::Invisible(_)) = &self.token.kind {
379+
panic!("njn: invis-delim?");
380+
}
378381

379382
let lo = self.token.span;
380383
let path = self.parse_path(PathStyle::Mod)?;

‎compiler/rustc_parse/src/parser/diagnostics.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2369,6 +2369,9 @@ impl<'a> Parser<'a> {
23692369
// in a subsequent macro invocation (#71039).
23702370
let mut tok = self.token.clone();
23712371
let mut labels = vec![];
2372+
if let token::OpenDelim(Delimiter::Invisible(_)) = &tok.kind {
2373+
panic!("njn: invis-delim?");
2374+
}
23722375
while let TokenKind::Interpolated(node) = &tok.kind {
23732376
let tokens = node.0.tokens();
23742377
labels.push(node.clone());
@@ -2378,6 +2381,9 @@ impl<'a> Parser<'a> {
23782381
&& let [AttrTokenTree::Token(token, _)] = &tokens[..]
23792382
{
23802383
tok = token.clone();
2384+
if let token::OpenDelim(Delimiter::Invisible(_)) = &tok.kind {
2385+
panic!("njn: invis-delim?");
2386+
}
23812387
} else {
23822388
break;
23832389
}

‎compiler/rustc_parse/src/parser/expr.rs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ use thin_vec::{thin_vec, ThinVec};
4444
/// `token::Interpolated` tokens.
4545
macro_rules! maybe_whole_expr {
4646
($p:expr) => {
47+
// njn: nothing needed here due to Nt* matches
4748
if let token::Interpolated(nt) = &$p.token.kind {
4849
match &nt.0 {
4950
token::NtExpr(e) | token::NtLiteral(e) => {
@@ -711,6 +712,9 @@ impl<'a> Parser<'a> {
711712
fn interpolated_or_expr_span(&self, expr: &Expr) -> Span {
712713
match self.prev_token.kind {
713714
TokenKind::Interpolated(..) => self.prev_token.span,
715+
TokenKind::OpenDelim(Delimiter::Invisible(_)) => {
716+
panic!("njn: invis-delim?");
717+
}
714718
_ => expr.span,
715719
}
716720
}
@@ -3539,7 +3543,12 @@ impl<'a> Parser<'a> {
35393543
&& !self.token.is_reserved_ident()
35403544
&& self.look_ahead(1, |t| {
35413545
AssocOp::from_token(t).is_some()
3542-
|| matches!(t.kind, token::OpenDelim(_))
3546+
|| {
3547+
if let token::OpenDelim(Delimiter::Invisible(_)) = t.kind {
3548+
panic!("njn: invis-delim?");
3549+
}
3550+
matches!(t.kind, token::OpenDelim(_))
3551+
}
35433552
|| t.kind == token::Dot
35443553
})
35453554
{

‎compiler/rustc_parse/src/parser/item.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2838,6 +2838,7 @@ impl<'a> Parser<'a> {
28382838

28392839
fn is_named_param(&self) -> bool {
28402840
let offset = match &self.token.kind {
2841+
// njn: nothing to do, just handle NtPat later
28412842
token::Interpolated(nt) => match &nt.0 {
28422843
token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon),
28432844
_ => 0,

‎compiler/rustc_parse/src/parser/mod.rs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ pub enum TrailingToken {
9292
#[macro_export]
9393
macro_rules! maybe_whole {
9494
($p:expr, $constructor:ident, |$x:ident| $e:expr) => {
95+
// njn: nothing needed here due to $constructor match
9596
if let token::Interpolated(nt) = &$p.token.kind
9697
&& let token::$constructor(x) = &nt.0
9798
{
@@ -117,6 +118,13 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath {
117118
$self.bump();
118119
return $self.maybe_recover_from_bad_qpath_stage_2($self.prev_token.span, ty);
119120
}
121+
if $allow_qpath_recovery
122+
&& $self.may_recover()
123+
&& $self.look_ahead(1, |t| t == &token::PathSep)
124+
&& let token::OpenDelim(Delimiter::Invisible(_)) = &$self.token.kind
125+
{
126+
panic!("njn: invis-delim?");
127+
}
120128
};
121129
}
122130

@@ -261,6 +269,7 @@ impl TokenCursor {
261269
&TokenTree::Token(ref token, spacing) => {
262270
debug_assert!(!matches!(
263271
token.kind,
272+
// njn: no change needed
264273
token::OpenDelim(_) | token::CloseDelim(_)
265274
));
266275
return (token.clone(), spacing);
@@ -716,6 +725,14 @@ impl<'a> Parser<'a> {
716725
}
717726

718727
fn check_inline_const(&self, dist: usize) -> bool {
728+
if self.is_keyword_ahead(dist, &[kw::Const]) {
729+
self.look_ahead(dist + 1, |t| match &t.kind {
730+
token::OpenDelim(Delimiter::Invisible(_)) => {
731+
panic!("njn: invis-delim?");
732+
}
733+
_ => {}
734+
})
735+
}
719736
self.is_keyword_ahead(dist, &[kw::Const])
720737
&& self.look_ahead(dist + 1, |t| match &t.kind {
721738
token::Interpolated(nt) => matches!(&nt.0, token::NtBlock(..)),

0 commit comments

Comments
 (0)