Skip to content

Commit dc2aecd

Browse files
VeykrilVeykril
committed
Fix the eager token maps by re-mapping the textranges between the input and input expansion
1 parent 2366c16 commit dc2aecd

File tree

18 files changed

+433
-157
lines changed

18 files changed

+433
-157
lines changed

‎crates/hir-def/src/macro_expansion_tests/builtin_fn_macro.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -235,10 +235,10 @@ macro_rules! format_args {
235235
236236
fn main() {
237237
/* error: no rule matches input tokens */;
238-
/* error: no rule matches input tokens */;
239-
/* error: no rule matches input tokens */;
240-
/* error: no rule matches input tokens */::core::fmt::Arguments::new_v1(&["", ], &[::core::fmt::ArgumentV1::new(&(), ::core::fmt::Display::fmt), ]);
241-
/* error: no rule matches input tokens */;
238+
/* error: expected expression */;
239+
/* error: expected expression, expected COMMA */;
240+
/* error: expected expression */::core::fmt::Arguments::new_v1(&["", ], &[::core::fmt::ArgumentV1::new(&(), ::core::fmt::Display::fmt), ]);
241+
/* error: expected expression, expected R_PAREN */;
242242
::core::fmt::Arguments::new_v1(&["", ], &[::core::fmt::ArgumentV1::new(&(5), ::core::fmt::Display::fmt), ]);
243243
}
244244
"##]],
@@ -364,7 +364,7 @@ macro_rules! format_args {
364364
365365
fn main() {
366366
let _ =
367-
/* error: no rule matches input tokens *//* parse error: expected field name or number */
367+
/* error: expected field name or number *//* parse error: expected field name or number */
368368
::core::fmt::Arguments::new_v1(&["", " ", ], &[::core::fmt::ArgumentV1::new(&(a.), ::core::fmt::Display::fmt), ::core::fmt::ArgumentV1::new(&(), ::core::fmt::Debug::fmt), ]);
369369
}
370370
"##]],

‎crates/hir-def/src/macro_expansion_tests/mbe.rs

+36
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,42 @@ fn#19 main#20(#21)#21 {#22
9898
"##]],
9999
);
100100
}
101+
102+
#[test]
103+
fn token_mapping_eager() {
104+
check(
105+
r#"
106+
#[rustc_builtin_macro]
107+
#[macro_export]
108+
macro_rules! format_args {}
109+
110+
macro_rules! identity {
111+
($expr:expr) => { $expr };
112+
}
113+
114+
fn main(foo: ()) {
115+
format_args/*+tokenids*/!("{} {} {}", format_args!("{}", 0), foo, identity!(10), "bar")
116+
}
117+
118+
"#,
119+
expect![[r##"
120+
#[rustc_builtin_macro]
121+
#[macro_export]
122+
macro_rules! format_args {}
123+
124+
macro_rules! identity {
125+
($expr:expr) => { $expr };
126+
}
127+
128+
fn main(foo: ()) {
129+
// format_args/*+tokenids*/!("{} {} {}"#1,#3 format_args!("{}", 0#10),#12 foo#13,#14 identity!(10#18),#21 "bar"#22)
130+
::core#4294967295::fmt#4294967295::Arguments#4294967295::new_v1#4294967295(&#4294967295[#4294967295""#4294967295,#4294967295 " "#4294967295,#4294967295 " "#4294967295,#4294967295 ]#4294967295,#4294967295 &#4294967295[::core#4294967295::fmt#4294967295::ArgumentV1#4294967295::new#4294967295(&#4294967295(::core#4294967295::fmt#4294967295::Arguments#4294967295::new_v1#4294967295(&#4294967295[#4294967295""#4294967295,#4294967295 ]#4294967295,#4294967295 &#4294967295[::core#4294967295::fmt#4294967295::ArgumentV1#4294967295::new#4294967295(&#4294967295(#42949672950#10)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::Display#4294967295::fmt#4294967295)#4294967295,#4294967295 ]#4294967295)#4294967295)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::Display#4294967295::fmt#4294967295)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::ArgumentV1#4294967295::new#4294967295(&#4294967295(#4294967295foo#13)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::Display#4294967295::fmt#4294967295)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::ArgumentV1#4294967295::new#4294967295(&#4294967295(#429496729510#18)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::Display#4294967295::fmt#4294967295)#4294967295,#4294967295 ]#4294967295)#4294967295
131+
}
132+
133+
"##]],
134+
);
135+
}
136+
101137
#[test]
102138
fn float_field_access_macro_input() {
103139
check(

‎crates/hir-def/src/macro_expansion_tests/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ pub fn identity_when_valid(_attr: TokenStream, item: TokenStream) -> TokenStream
187187
let range: Range<usize> = range.into();
188188

189189
if show_token_ids {
190-
if let Some((tree, map, _)) = arg.as_deref() {
190+
if let Some((tree, map, _)) = arg.value.as_deref() {
191191
let tt_range = call.token_tree().unwrap().syntax().text_range();
192192
let mut ranges = Vec::new();
193193
extract_id_ranges(&mut ranges, map, tree);

‎crates/hir-expand/src/builtin_fn_macro.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ pub(crate) fn include_arg_to_tt(
692692
arg_id: MacroCallId,
693693
) -> Result<(triomphe::Arc<(::tt::Subtree<::tt::TokenId>, TokenMap)>, FileId), ExpandError> {
694694
let loc = db.lookup_intern_macro_call(arg_id);
695-
let Some(EagerCallInfo { arg, arg_id: Some(arg_id), .. }) = loc.eager.as_deref() else {
695+
let Some(EagerCallInfo { arg,arg_id, .. }) = loc.eager.as_deref() else {
696696
panic!("include_arg_to_tt called on non include macro call: {:?}", &loc.eager);
697697
};
698698
let path = parse_string(&arg.0)?;

‎crates/hir-expand/src/db.rs

+105-46
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use base_db::{salsa, CrateId, Edition, SourceDatabase};
44
use either::Either;
55
use limit::Limit;
6-
use mbe::syntax_node_to_token_tree;
6+
use mbe::{syntax_node_to_token_tree, ValueResult};
77
use rustc_hash::FxHashSet;
88
use syntax::{
99
ast::{self, HasAttrs, HasDocComments},
@@ -124,16 +124,21 @@ pub trait ExpandDatabase: SourceDatabase {
124124
fn macro_arg(
125125
&self,
126126
id: MacroCallId,
127-
) -> Option<Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>>;
127+
) -> ValueResult<
128+
Option<Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>>,
129+
Arc<Box<[SyntaxError]>>,
130+
>;
128131
/// Extracts syntax node, corresponding to a macro call. That's a firewall
129132
/// query, only typing in the macro call itself changes the returned
130133
/// subtree.
131-
fn macro_arg_text(&self, id: MacroCallId) -> Option<GreenNode>;
132-
/// Gets the expander for this macro. This compiles declarative macros, and
133-
/// just fetches procedural ones.
134-
// FIXME: Rename this
134+
fn macro_arg_node(
135+
&self,
136+
id: MacroCallId,
137+
) -> ValueResult<Option<GreenNode>, Arc<Box<[SyntaxError]>>>;
138+
/// Fetches the expander for this macro.
135139
#[salsa::transparent]
136-
fn macro_def(&self, id: MacroDefId) -> TokenExpander;
140+
fn macro_expander(&self, id: MacroDefId) -> TokenExpander;
141+
/// Fetches (and compiles) the expander of this decl macro.
137142
fn decl_macro_expander(
138143
&self,
139144
def_crate: CrateId,
@@ -335,14 +340,20 @@ fn parse_macro_expansion_error(
335340
fn macro_arg(
336341
db: &dyn ExpandDatabase,
337342
id: MacroCallId,
338-
) -> Option<Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>> {
343+
) -> ValueResult<
344+
Option<Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>>,
345+
Arc<Box<[SyntaxError]>>,
346+
> {
339347
let loc = db.lookup_intern_macro_call(id);
340348

341349
if let Some(EagerCallInfo { arg, arg_id: _, error: _ }) = loc.eager.as_deref() {
342-
return Some(Arc::new((arg.0.clone(), arg.1.clone(), Default::default())));
350+
return ValueResult::ok(Some(Arc::new((arg.0.clone(), arg.1.clone(), Default::default()))));
343351
}
344352

345-
let arg = db.macro_arg_text(id)?;
353+
let ValueResult { value, err } = db.macro_arg_node(id);
354+
let Some(arg) = value else {
355+
return ValueResult { value: None, err };
356+
};
346357

347358
let node = SyntaxNode::new_root(arg);
348359
let censor = censor_for_macro_input(&loc, &node);
@@ -360,7 +371,11 @@ fn macro_arg(
360371
// proc macros expect their inputs without parentheses, MBEs expect it with them included
361372
tt.delimiter = tt::Delimiter::unspecified();
362373
}
363-
Some(Arc::new((tt, tmap, fixups.undo_info)))
374+
let val = Some(Arc::new((tt, tmap, fixups.undo_info)));
375+
match err {
376+
Some(err) => ValueResult::new(val, err),
377+
None => ValueResult::ok(val),
378+
}
364379
}
365380

366381
/// Certain macro calls expect some nodes in the input to be preprocessed away, namely:
@@ -402,9 +417,43 @@ fn censor_for_macro_input(loc: &MacroCallLoc, node: &SyntaxNode) -> FxHashSet<Sy
402417
.unwrap_or_default()
403418
}
404419

405-
fn macro_arg_text(db: &dyn ExpandDatabase, id: MacroCallId) -> Option<GreenNode> {
420+
fn macro_arg_node(
421+
db: &dyn ExpandDatabase,
422+
id: MacroCallId,
423+
) -> ValueResult<Option<GreenNode>, Arc<Box<[SyntaxError]>>> {
424+
let err = || -> Arc<Box<[_]>> {
425+
Arc::new(Box::new([SyntaxError::new_at_offset(
426+
"invalid macro call".to_owned(),
427+
syntax::TextSize::from(0),
428+
)]))
429+
};
406430
let loc = db.lookup_intern_macro_call(id);
407-
let arg = loc.kind.arg(db)?.value;
431+
let arg = if let MacroDefKind::BuiltInEager(..) = loc.def.kind {
432+
let res = if let Some(EagerCallInfo { arg, .. }) = loc.eager.as_deref() {
433+
Some(mbe::token_tree_to_syntax_node(&arg.0, mbe::TopEntryPoint::Expr).0)
434+
} else {
435+
loc.kind
436+
.arg(db)
437+
.and_then(|arg| ast::TokenTree::cast(arg.value))
438+
.map(|tt| tt.reparse_as_expr().to_syntax())
439+
};
440+
441+
match res {
442+
Some(res) if res.errors().is_empty() => res.syntax_node(),
443+
Some(res) => {
444+
return ValueResult::new(
445+
Some(res.syntax_node().green().into()),
446+
Arc::new(Box::from(res.errors())),
447+
)
448+
}
449+
None => return ValueResult::only_err(err()),
450+
}
451+
} else {
452+
match loc.kind.arg(db) {
453+
Some(res) => res.value,
454+
None => return ValueResult::only_err(err()),
455+
}
456+
};
408457
if matches!(loc.kind, MacroCallKind::FnLike { .. }) {
409458
let first = arg.first_child_or_token().map_or(T![.], |it| it.kind());
410459
let last = arg.last_child_or_token().map_or(T![.], |it| it.kind());
@@ -419,20 +468,13 @@ fn macro_arg_text(db: &dyn ExpandDatabase, id: MacroCallId) -> Option<GreenNode>
419468
// Some day, we'll have explicit recursion counters for all
420469
// recursive things, at which point this code might be removed.
421470
cov_mark::hit!(issue9358_bad_macro_stack_overflow);
422-
return None;
471+
return ValueResult::only_err(Arc::new(Box::new([SyntaxError::new(
472+
"unbalanced token tree".to_owned(),
473+
arg.text_range(),
474+
)])));
423475
}
424476
}
425-
if let Some(EagerCallInfo { arg, .. }) = loc.eager.as_deref() {
426-
Some(
427-
mbe::token_tree_to_syntax_node(&arg.0, mbe::TopEntryPoint::Expr)
428-
.0
429-
.syntax_node()
430-
.green()
431-
.into(),
432-
)
433-
} else {
434-
Some(arg.green().into())
435-
}
477+
ValueResult::ok(Some(arg.green().into()))
436478
}
437479

438480
fn decl_macro_expander(
@@ -474,7 +516,7 @@ fn decl_macro_expander(
474516
Arc::new(DeclarativeMacroExpander { mac, def_site_token_map })
475517
}
476518

477-
fn macro_def(db: &dyn ExpandDatabase, id: MacroDefId) -> TokenExpander {
519+
fn macro_expander(db: &dyn ExpandDatabase, id: MacroDefId) -> TokenExpander {
478520
match id.kind {
479521
MacroDefKind::Declarative(ast_id) => {
480522
TokenExpander::DeclarativeMacro(db.decl_macro_expander(id.krate, ast_id))
@@ -491,20 +533,10 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt
491533
let _p = profile::span("macro_expand");
492534
let loc = db.lookup_intern_macro_call(id);
493535

494-
// This might look a bit odd, but we do not expand the inputs to eager macros here.
495-
// Eager macros inputs are expanded, well, eagerly when we collect the macro calls.
496-
// That kind of expansion uses the ast id map of an eager macros input though which goes through
497-
// the HirFileId machinery. As eager macro inputs are assigned a macro file id that query
498-
// will end up going through here again, whereas we want to just want to inspect the raw input.
499-
// As such we just return the input subtree here.
500-
if let Some(EagerCallInfo { arg, arg_id: None, error }) = loc.eager.as_deref() {
501-
return ExpandResult { value: Arc::new(arg.0.clone()), err: error.clone() };
502-
}
503-
504-
let (ExpandResult { value: mut tt, mut err }, tmap) = match loc.def.kind {
536+
let ExpandResult { value: tt, mut err } = match loc.def.kind {
505537
MacroDefKind::ProcMacro(..) => return db.expand_proc_macro(id),
506538
MacroDefKind::BuiltInDerive(expander, ..) => {
507-
let arg = db.macro_arg_text(id).unwrap();
539+
let arg = db.macro_arg_node(id).value.unwrap();
508540

509541
let node = SyntaxNode::new_root(arg);
510542
let censor = censor_for_macro_input(&loc, &node);
@@ -520,10 +552,13 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt
520552

521553
// this cast is a bit sus, can we avoid losing the typedness here?
522554
let adt = ast::Adt::cast(node).unwrap();
523-
(expander.expand(db, id, &adt, &tmap), Some((tmap, fixups.undo_info)))
555+
let mut res = expander.expand(db, id, &adt, &tmap);
556+
fixup::reverse_fixups(&mut res.value, &tmap, &fixups.undo_info);
557+
res
524558
}
525559
_ => {
526-
let Some(macro_arg) = db.macro_arg(id) else {
560+
let ValueResult { value, err } = db.macro_arg(id);
561+
let Some(macro_arg) = value else {
527562
return ExpandResult {
528563
value: Arc::new(tt::Subtree {
529564
delimiter: tt::Delimiter::UNSPECIFIED,
@@ -534,18 +569,43 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt
534569
err: Some(ExpandError::other("invalid token tree")),
535570
};
536571
};
572+
537573
let (arg, arg_tm, undo_info) = &*macro_arg;
538574
let mut res = match loc.def.kind {
539575
MacroDefKind::Declarative(id) => {
540576
db.decl_macro_expander(loc.def.krate, id).expand(arg.clone())
541577
}
542578
MacroDefKind::BuiltIn(it, _) => it.expand(db, id, &arg).map_err(Into::into),
579+
// This might look a bit odd, but we do not expand the inputs to eager macros here.
580+
// Eager macros inputs are expanded, well, eagerly when we collect the macro calls.
581+
// That kind of expansion uses the ast id map of an eager macros input though which goes through
582+
// the HirFileId machinery. As eager macro inputs are assigned a macro file id that query
583+
// will end up going through here again, whereas we want to just want to inspect the raw input.
584+
// As such we just return the input subtree here.
585+
MacroDefKind::BuiltInEager(..) if loc.eager.is_none() => {
586+
let mut arg = arg.clone();
587+
fixup::reverse_fixups(&mut arg, arg_tm, undo_info);
588+
589+
return ExpandResult {
590+
value: Arc::new(arg),
591+
err: err.map(|err| {
592+
let mut buf = String::new();
593+
for err in &**err {
594+
use std::fmt::Write;
595+
_ = write!(buf, "{}, ", err);
596+
}
597+
buf.pop();
598+
buf.pop();
599+
ExpandError::other(buf)
600+
}),
601+
};
602+
}
543603
MacroDefKind::BuiltInEager(it, _) => it.expand(db, id, &arg).map_err(Into::into),
544604
MacroDefKind::BuiltInAttr(it, _) => it.expand(db, id, &arg),
545605
_ => unreachable!(),
546606
};
547607
fixup::reverse_fixups(&mut res.value, arg_tm, undo_info);
548-
(res, None)
608+
res
549609
}
550610
};
551611

@@ -559,24 +619,23 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt
559619
return value;
560620
}
561621

562-
if let Some((arg_tm, undo_info)) = &tmap {
563-
fixup::reverse_fixups(&mut tt, arg_tm, undo_info);
564-
}
565-
566622
ExpandResult { value: Arc::new(tt), err }
567623
}
568624

569625
fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt::Subtree>> {
570626
let loc = db.lookup_intern_macro_call(id);
571-
let Some(macro_arg) = db.macro_arg(id) else {
627+
let Some(macro_arg) = db.macro_arg(id).value else {
572628
return ExpandResult {
573629
value: Arc::new(tt::Subtree {
574630
delimiter: tt::Delimiter::UNSPECIFIED,
575631
token_trees: Vec::new(),
576632
}),
633+
// FIXME: We should make sure to enforce an invariant that invalid macro
634+
// calls do not reach this call path!
577635
err: Some(ExpandError::other("invalid token tree")),
578636
};
579637
};
638+
580639
let (arg_tt, arg_tm, undo_info) = &*macro_arg;
581640

582641
let expander = match loc.def.kind {

0 commit comments

Comments
 (0)