Mark grammar roots inline in productions

We check that the list of grammar "roots" -- that is, productions that
are not used in any other production -- is what we expect it to be.

We had hard coded this list of roots in `mdbook-spec`.  Let's instead
add a way to specify this in our syntax for productions by prefixing
the production with `@root`.

Author: traviscross

Diff for: docs/grammar.md

@@ -16,6 +16,8 @@ The category is used to group similar productions on the grammar summary page in
 
 The syntax for the grammar itself is pretty close to what is described in the [Notation chapter](../src/notation.md), though there are some rendering differences.
 
+A "root" production, marked with `@root`, is one that is not used in any other production.
+
 The syntax for the grammar itself (written in itself, hopefully that's not too confusing) is:
 
 ```
@@ -25,7 +27,7 @@ BACKTICK -> U+0060
 
 LF -> U+000A
 
-Production -> Name ` ->` Expression
+Production -> `@root`? Name ` ->` Expression
 
 Name -> <Alphanumeric or `_`>+
 

Diff for: mdbook-spec/src/grammar.rs

@@ -28,6 +28,7 @@ pub struct Production {
     expression: Expression,
     /// The path to the chapter where this is defined.
     path: PathBuf,
+    is_root: bool,
 }
 
 #[derive(Debug)]
@@ -139,7 +140,7 @@ impl Expression {
 static GRAMMAR_RE: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r"(?ms)^```grammar,([^\n]+)\n(.*?)^```").unwrap());
 static NAMES_RE: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"(?m)^([A-Za-z0-9_]+)(?: \([^)]+\))? ->").unwrap());
+    LazyLock::new(|| Regex::new(r"(?m)^(?:@root )?([A-Za-z0-9_]+)(?: \([^)]+\))? ->").unwrap());
 
 /// Loads the [`Grammar`] from the book.
 pub fn load_grammar(book: &Book, diag: &mut Diagnostics) -> Grammar {
@@ -183,38 +184,30 @@ fn check_unexpected_roots(grammar: &Grammar, diag: &mut Diagnostics) {
     grammar.visit_nt(&mut |nt| {
         set.remove(nt);
     });
-    // TODO: We may want to rethink how some of these are structured.
-    let expected: HashSet<_> = [
-        "CfgAttrAttribute",
-        "CfgAttribute",
-        "CHAR",
-        "Crate",
-        "INNER_LINE_DOC",
-        "LINE_COMMENT",
-        "MetaListIdents",
-        "MetaListNameValueStr",
-        "MetaListPaths",
-        "MetaWord",
-        "OUTER_LINE_DOC",
-    ]
-    .into_iter()
-    .collect();
+    let expected: HashSet<_> = grammar
+        .productions
+        .values()
+        .filter_map(|p| p.is_root.then(|| p.name.as_str()))
+        .collect();
     if set != expected {
         let new: Vec<_> = set.difference(&expected).collect();
         let removed: Vec<_> = expected.difference(&set).collect();
         if !new.is_empty() {
             warn_or_err!(
                 diag,
-                "New grammar production detected that is not used in any other production.\n\
-                 If this is expected, add it to the `check_unexpected_roots` function.\n\
-                 If not, make sure it is spelled correctly and used in another production.\n\
+                "New grammar production detected that is not used in any other\n\
+                 production. If this is expected, mark the production with\n\
+                 `@root`. If not, make sure it is spelled correctly and used in\n\
+                 another production.\n\
+                 \n\
                  The new names are: {new:?}\n"
             );
         } else if !removed.is_empty() {
             warn_or_err!(
                 diag,
                 "Old grammar production root seems to have been removed.\n\
-                 If this is expected, remove it from the `check_unexpected_roots` function.\n\
+                 If this is expected, remove `@root` from the production.\n\
+                 \n\
                  The removed names are: {removed:?}\n"
             );
         } else {

Diff for: mdbook-spec/src/grammar/parser.rs

@@ -136,6 +136,8 @@ impl Parser<'_> {
     }
 
     fn parse_production(&mut self, category: &str, path: &Path) -> Result<Production> {
+        let is_root = self.parse_is_root();
+        self.space0();
         let name = self
             .parse_name()
             .ok_or_else(|| self.error("expected production name".to_string()))?;
@@ -148,9 +150,14 @@ impl Parser<'_> {
             category: category.to_string(),
             expression,
             path: path.to_owned(),
+            is_root,
         })
     }
 
+    fn parse_is_root(&mut self) -> bool {
+        self.take_str("@root")
+    }
+
     fn parse_name(&mut self) -> Option<String> {
         let name = self.take_while(&|c: char| c.is_alphanumeric() || c == '_');
         if name.is_empty() {

Diff for: src/attributes.md

@@ -164,19 +164,19 @@ forms:
 
 r[attributes.meta.builtin.syntax]
 ```grammar,attributes
-MetaWord ->
+@root MetaWord ->
     IDENTIFIER
 
 MetaNameValueStr ->
     IDENTIFIER `=` (STRING_LITERAL | RAW_STRING_LITERAL)
 
-MetaListPaths ->
+@root MetaListPaths ->
     IDENTIFIER `(` ( SimplePath (`,` SimplePath)* `,`? )? `)`
 
-MetaListIdents ->
+@root MetaListIdents ->
     IDENTIFIER `(` ( IDENTIFIER (`,` IDENTIFIER)* `,`? )? `)`
 
-MetaListNameValueStr ->
+@root MetaListNameValueStr ->
     IDENTIFIER `(` ( MetaNameValueStr (`,` MetaNameValueStr)* `,`? )? `)`
 ```
 

Diff for: src/comments.md

@@ -3,7 +3,7 @@ r[comments]
 
 r[comments.syntax]
 ```grammar,lexer
-LINE_COMMENT ->
+@root LINE_COMMENT ->
       `//` (~[`/` `!` LF] | `//`) ~LF*
     | `//`
 
@@ -15,13 +15,13 @@ BLOCK_COMMENT ->
     | `/**/`
     | `/***/`
 
-INNER_LINE_DOC ->
+@root INNER_LINE_DOC ->
     `//!` ~[LF CR]*
 
 INNER_BLOCK_DOC ->
     `/*!` ( BlockCommentOrDoc | ~[`*/` CR] )* `*/`
 
-OUTER_LINE_DOC ->
+@root OUTER_LINE_DOC ->
     `///` (~`/` ~[LF CR]*)?
 
 OUTER_BLOCK_DOC ->

Diff for: src/conditional-compilation.md

@@ -315,7 +315,7 @@ r[cfg.attr]
 
 r[cfg.attr.syntax]
 ```grammar,configuration
-CfgAttribute -> `cfg` `(` ConfigurationPredicate `)`
+@root CfgAttribute -> `cfg` `(` ConfigurationPredicate `)`
 ```
 
 <!-- should we say they're active attributes here? -->
@@ -382,7 +382,7 @@ r[cfg.cfg_attr]
 
 r[cfg.cfg_attr.syntax]
 ```grammar,configuration
-CfgAttrAttribute -> `cfg_attr` `(` ConfigurationPredicate `,` CfgAttrs? `)`
+@root CfgAttrAttribute -> `cfg_attr` `(` ConfigurationPredicate `,` CfgAttrs? `)`
 
 CfgAttrs -> Attr (`,` Attr)* `,`?
 ```

Diff for: src/crates-and-source-files.md

@@ -3,7 +3,7 @@ r[crate]
 
 r[crate.syntax]
 ```grammar,items
-Crate ->
+@root Crate ->
     InnerAttribute*
     Item*
 ```

Diff for: src/notation.md

@@ -52,7 +52,7 @@ The following are common definitions used in the grammar.
 
 r[input.syntax]
 ```grammar,lexer
-CHAR -> <a Unicode scalar value>
+@root CHAR -> <a Unicode scalar value>
 
 NUL -> U+0000