ilex: Make tokens carry an ilex::Context

ilex/src/file/mod.rs

@@ -85,11 +85,11 @@ impl<'ctx> File<'ctx> {
   }
 
   /// Tokenizes the this file according to `spec` and generates a token stream.
-  pub fn lex<'spec>(
+  pub fn lex(
     self,
-    spec: &'spec Spec,
+    spec: &'ctx Spec,
     report: &Report,
-  ) -> Result<token::Stream<'spec>, Fatal> {
+  ) -> Result<token::Stream<'ctx>, Fatal> {
     rt::lex(self, report, spec)
   }
 }

ilex/src/fp.rs

@@ -613,7 +613,7 @@ impl Digital<'_> {
         && (exp.is_none()
           || exp.is_some_and(|exp| {
             exp.radix() == 10
-              && (exp.has_prefix(ctx, "e") || exp.has_prefix(ctx, "E"))
+              && (exp.has_prefix("e") || exp.has_prefix("E"))
               && has_ordinary_sign(ctx, &exp)
           }))
         && (rule.separator.is_empty()

ilex/src/report/builtin.rs

@@ -9,7 +9,6 @@ use byteyarn::yarn;
 use byteyarn::YarnBox;
 
 use crate::f;
-use crate::file::Context;
 use crate::file::Spanned;
 use crate::plural;
 use crate::report::Diagnostic;
@@ -43,10 +42,10 @@ impl Builtins<'_> {
       .report
       .error(f!(
         "unexpected {} in {}",
-        found.for_user_diagnostic(self.spec, &self.report.ctx),
+        found.for_user_diagnostic(self.spec),
         unexpected_in
           .into()
-          .for_user_diagnostic(self.spec, &self.report.ctx),
+          .for_user_diagnostic(self.spec),
       ))
       .at(at)
       .reported_at(Location::caller());
@@ -85,7 +84,7 @@ impl Builtins<'_> {
         plural(found.chars().count()),
         unexpected_in
           .into()
-          .for_user_diagnostic(self.spec, &self.report.ctx),
+          .for_user_diagnostic(self.spec),
       ))
       .at(at)
       .remark(
@@ -109,14 +108,14 @@ impl Builtins<'_> {
     at: impl Spanned,
   ) -> Diagnostic {
     let expected = expected.into_iter().map(Into::into).collect::<Vec<_>>();
-    let alts = disjunction_to_string(self.spec, &self.report.ctx, &expected);
+    let alts = disjunction_to_string(self.spec, &expected);
     let found = found.into();
 
     let diagnostic = self
       .report
       .error(f!(
         "expected {alts}, but found {}",
-        found.for_user_diagnostic(self.spec, &self.report.ctx)
+        found.for_user_diagnostic(self.spec)
       ))
       .saying(at, f!("expected {alts}"))
       .reported_at(Location::caller());
@@ -140,7 +139,7 @@ impl Builtins<'_> {
       .report
       .error(f!(
         "unexpected closing {}",
-        found.for_user_diagnostic(self.spec, &self.report.ctx)
+        found.for_user_diagnostic(self.spec)
       ))
       .saying(at, f!("expected to be opened by `{expected}`"))
       .reported_at(Location::caller());
@@ -165,7 +164,7 @@ impl Builtins<'_> {
       .report
       .error(f!(
         "expected closing `{expected}`, but found {}",
-        found.for_user_diagnostic(self.spec, &self.report.ctx)
+        found.for_user_diagnostic(self.spec)
       ))
       .saying(at, f!("expected `{expected}` here"))
       .remark(open, "previously opened here")
@@ -189,7 +188,7 @@ impl Builtins<'_> {
         "unexpected non-ASCII characters in {}",
         expected
           .into()
-          .for_user_diagnostic(self.spec, &self.report.ctx)
+          .for_user_diagnostic(self.spec)
       ))
       .at(at)
       .reported_at(Location::caller())
@@ -263,7 +262,7 @@ impl Builtins<'_> {
       .report
       .error(f!(
         "{} out of span",
-        what.into().for_user_diagnostic(self.spec, &self.report.ctx)
+        what.into().for_user_diagnostic(self.spec)
       ))
       .at(at)
       .note(f!(
@@ -313,12 +312,11 @@ fn non_printable_note(found: Expected, diagnostic: Diagnostic) -> Diagnostic {
 
 fn disjunction_to_string<'a>(
   spec: &'a Spec,
-  ctx: &'a Context,
   lexemes: &'a [Expected],
 ) -> YarnBox<'a, str> {
   let mut names = lexemes
     .iter()
-    .map(|tok| tok.for_user_diagnostic(spec, ctx))
+    .map(|tok| tok.for_user_diagnostic(spec))
     .collect::<Vec<_>>();
   names.sort();
   names.dedup();
@@ -424,13 +422,12 @@ impl Expected<'_> {
   pub(crate) fn for_user_diagnostic<'a>(
     &'a self,
     spec: &'a Spec,
-    ctx: &Context,
   ) -> YarnBox<'a, str> {
     match self {
       Self::Literal(lit) => yarn!("`{lit}`"),
       Self::Name(name) => name.as_ref().to_box(),
       Self::Lexeme(lex) => lex.to_yarn(spec),
-      Self::Token(tok) => tok.to_yarn(ctx),
+      Self::Token(tok) => tok.to_yarn(),
     }
   }
 }

ilex/src/rt/emit2.rs

@@ -498,7 +498,7 @@ pub fn emit(lexer: &mut Lexer) {
           prefix,
           suffix,
         };
-        let token = Cursor::fake_token(lexer.spec(), &tok);
+        let token = Cursor::fake_token(lexer.file(), lexer.spec(), &tok);
 
         // This happens later so we have access to the full spans of
         // the digit blocks.
@@ -747,6 +747,7 @@ pub fn emit(lexer: &mut Lexer) {
   if match_.extra > 0 {
     let expected = if generated_token {
       Expected::Token(token::Cursor::fake_token(
+        lexer.file(),
         lexer.spec(),
         lexer.last_token(),
       ))
@@ -778,6 +779,7 @@ pub fn emit(lexer: &mut Lexer) {
 
       let expected = if generated_token {
         Expected::Token(token::Cursor::fake_token(
+          lexer.file(),
           lexer.spec(),
           lexer.last_token(),
         ))

ilex/src/rt/lexer.rs

@@ -24,9 +24,9 @@ use super::unicode::is_xid;
 
 /// The lexer state struct, that tracks everything going on during a lexing
 /// operation.
-pub struct Lexer<'a, 'spec, 'ctx> {
+pub struct Lexer<'a, 'ctx> {
   report: &'a Report,
-  spec: &'spec Spec,
+  spec: &'ctx Spec,
   file: File<'ctx>,
 
   cursor: usize,
@@ -46,9 +46,9 @@ pub struct Closer {
   close: Yarn,
 }
 
-impl<'a, 'spec, 'ctx> Lexer<'a, 'spec, 'ctx> {
+impl<'a, 'ctx> Lexer<'a, 'ctx> {
   /// Creates a new lexer.
-  pub fn new(file: File<'ctx>, report: &'a Report, spec: &'spec Spec) -> Self {
+  pub fn new(file: File<'ctx>, report: &'a Report, spec: &'ctx Spec) -> Self {
     Lexer {
       eof: file.span(file.len()..file.len()).intern(file.context()),
       cache: Cache::new(&spec.dfa().engine),
@@ -79,7 +79,7 @@ impl<'a, 'spec, 'ctx> Lexer<'a, 'spec, 'ctx> {
   }
 
   /// Returns the spec we're lexing against.
-  pub fn spec(&self) -> &'spec Spec {
+  pub fn spec(&self) -> &'ctx Spec {
     self.spec
   }
 
@@ -265,7 +265,7 @@ impl<'a, 'spec, 'ctx> Lexer<'a, 'spec, 'ctx> {
     }
   }
 
-  pub fn finish(mut self) -> token::Stream<'spec> {
+  pub fn finish(mut self) -> token::Stream<'ctx> {
     self.add_token(rt::Token {
       kind: rt::Kind::Eof,
       span: self.eof,
@@ -281,6 +281,6 @@ impl<'a, 'spec, 'ctx> Lexer<'a, 'spec, 'ctx> {
         .unclosed(open, &close.close, Lexeme::eof(), self.eof());
     }
 
-    token::Stream { spec: self.spec, toks: self.tokens }
+    token::Stream { file: self.file, spec: self.spec, toks: self.tokens }
   }
 }

ilex/src/rt/mod.rs

@@ -19,11 +19,11 @@ mod dfa;
 pub use dfa::compile;
 pub use dfa::Dfa;
 
-pub fn lex<'spec>(
-  file: File,
+pub fn lex<'ctx>(
+  file: File<'ctx>,
   report: &Report,
-  spec: &'spec Spec,
-) -> Result<token::Stream<'spec>, Fatal> {
+  spec: &'ctx Spec,
+) -> Result<token::Stream<'ctx>, Fatal> {
   let mut lexer = lexer::Lexer::new(file, report, spec);
 
   let mut unexpected_start = None;