[NFC] Make DeclGroupSyntax.memberBlock an IUO

The special decls nested inside `@abi` attributes don’t have bodies, so it’s necessary to allow `DeclGroupSyntax.memberBlock` (and the nodes that conform to it, which include the nodes for extensions and nominal types) to be `nil`.

The plan is that `nil`s will *only* occur in new syntax, not elsewhere in Swift code—even if a DeclGroupSyntax node is written without a body, the parser will still create a MemberBlockSyntax for it with missing tokens.

Since `nil` will never occur in pre-existing nodes or code, I’ve chosen to make `memberBlock` an implicitly unwrapped optional. Alternatives would be to make it a normal optional (breaking source compatibility) or to make `memberBlock` a backwards compatibility shim for an optional property with a new name.

I have nevertheless chosen to write all use sites for `memberBlock` as though the property was a normal Optional, except for ones in tests.

Author: beccadax

CodeGeneration/Sources/SyntaxSupport/Child.swift

@@ -36,6 +36,16 @@ public enum TokenChoice: Equatable, IdentifierConvertible {
   }
 }
 
+/// If a child can be optional, what flavor of optional is it? Used to make
+/// some children optional without breaking source compatibility.
+public enum Optionality: String {
+  /// An ordinary `Optional` which must be explicitly unwrapped by users.
+  case normal = "?"
+  /// An implicitly-unwrapped `Optional` which maintains source compatibility
+  /// if the child's optionality has changed.
+  case implicitlyUnwrapped = "!"
+}
+
 public enum ChildKind {
   /// The child always contains a node of the given `kind`.
   case node(kind: SyntaxNodeKind)
@@ -95,7 +105,7 @@ public class Child: NodeChoiceConvertible {
   public let kind: ChildKind
 
   /// Whether this child is optional and can be `nil`.
-  public let isOptional: Bool
+  public let optionality: Optionality?
 
   public let experimentalFeature: ExperimentalFeature?
 
@@ -256,7 +266,7 @@ public class Child: NodeChoiceConvertible {
     experimentalFeature: ExperimentalFeature? = nil,
     nameForDiagnostics: String? = nil,
     documentation: String? = nil,
-    isOptional: Bool = false
+    optionality: Optionality? = nil
   ) {
     precondition(name.first?.isLowercase ?? true, "The first letter of a child’s name should be lowercase")
     precondition(
@@ -270,6 +280,31 @@ public class Child: NodeChoiceConvertible {
     self.nameForDiagnostics = nameForDiagnostics
     self.documentationSummary = SwiftSyntax.Trivia.docCommentTrivia(from: documentation)
     self.documentationAbstract = String(documentation?.split(whereSeparator: \.isNewline).first ?? "")
-    self.isOptional = isOptional
+    self.optionality = optionality
+  }
+
+  /// If a classification is passed, it specifies the color identifiers in
+  /// that subtree should inherit for syntax coloring. Must be a member of
+  /// ``SyntaxClassification``.
+  /// If `forceClassification` is also set to true, all child nodes (not only
+  /// identifiers) inherit the syntax classification.
+  convenience init(
+    name: String,
+    deprecatedName: String? = nil,
+    kind: ChildKind,
+    experimentalFeature: ExperimentalFeature? = nil,
+    nameForDiagnostics: String? = nil,
+    documentation: String? = nil,
+    isOptional: Bool
+  ) {
+    self.init(
+      name: name,
+      deprecatedName: deprecatedName,
+      kind: kind,
+      experimentalFeature: experimentalFeature,
+      nameForDiagnostics: nameForDiagnostics,
+      documentation: documentation,
+      optionality: isOptional ? .normal : nil
+    )
   }
 }

CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift

@@ -227,7 +227,8 @@ public let DECL_NODES: [Node] = [
       ),
       Child(
         name: "memberBlock",
-        kind: .node(kind: .memberBlock)
+        kind: .node(kind: .memberBlock),
+        optionality: .implicitlyUnwrapped
       ),
     ]
   ),
@@ -401,7 +402,8 @@ public let DECL_NODES: [Node] = [
         name: "memberBlock",
         kind: .node(kind: .memberBlock),
         documentation:
-          "The members of the class declaration. As class extension declarations may declare additional members, the contents of this member block isn't guaranteed to be a complete list of members for this type."
+          "The members of the class declaration. As class extension declarations may declare additional members, the contents of this member block isn't guaranteed to be a complete list of members for this type.",
+        optionality: .implicitlyUnwrapped
       ),
     ]
   ),
@@ -827,7 +829,8 @@ public let DECL_NODES: [Node] = [
         name: "memberBlock",
         kind: .node(kind: .memberBlock),
         documentation:
-          "The cases and other members associated with this enum declaration. Because enum extension declarations may declare additional members the contents of this member block isn't guaranteed to be a complete list of members for this type."
+          "The cases and other members associated with this enum declaration. Because enum extension declarations may declare additional members the contents of this member block isn't guaranteed to be a complete list of members for this type.",
+        optionality: .implicitlyUnwrapped
       ),
     ]
   ),
@@ -906,7 +909,8 @@ public let DECL_NODES: [Node] = [
         name: "memberBlock",
         kind: .node(kind: .memberBlock),
         documentation:
-          "The members of the extension declaration. As this is an extension, the contents of this member block isn't guaranteed to be a complete list of members for this type."
+          "The members of the extension declaration. As this is an extension, the contents of this member block isn't guaranteed to be a complete list of members for this type.",
+        optionality: .implicitlyUnwrapped
       ),
     ]
   ),
@@ -2011,7 +2015,8 @@ public let DECL_NODES: [Node] = [
       Child(
         name: "memberBlock",
         kind: .node(kind: .memberBlock),
-        documentation: "The members of the protocol declaration."
+        documentation: "The members of the protocol declaration.",
+        optionality: .implicitlyUnwrapped
       ),
     ]
   ),
@@ -2182,7 +2187,8 @@ public let DECL_NODES: [Node] = [
         name: "memberBlock",
         kind: .node(kind: .memberBlock),
         documentation:
-          "The members of the struct declaration. Because struct extension declarations may declare additional members the contents of this member block isn't guaranteed to be a complete list of members for this type."
+          "The members of the struct declaration. Because struct extension declarations may declare additional members the contents of this member block isn't guaranteed to be a complete list of members for this type.",
+        optionality: .implicitlyUnwrapped
       ),
     ]
   ),

CodeGeneration/Sources/SyntaxSupport/GrammarGenerator.swift

@@ -34,7 +34,7 @@ struct GrammarGenerator {
   }
 
   private func grammar(for child: Child) -> String {
-    let optionality = child.isOptional ? "?" : ""
+    let optionality = child.optionality?.rawValue ?? ""
     switch child.kind {
     case .node(let kind):
       return "\(kind.doccLink)\(optionality)"

CodeGeneration/Sources/SyntaxSupport/Traits.swift

@@ -63,7 +63,7 @@ public let TRAITS: [Trait] = [
           "A `where` clause that places additional constraints on generic parameters like `where Element: Hashable`.",
         isOptional: true
       ),
-      Child(name: "memberBlock", kind: .node(kind: .memberBlock)),
+      Child(name: "memberBlock", kind: .node(kind: .memberBlock), optionality: .implicitlyUnwrapped),
     ]
   ),
   Trait(

CodeGeneration/Sources/Utils/SyntaxBuildableChild.swift

@@ -40,7 +40,7 @@ extension Child {
     }
     return SyntaxBuildableType(
       kind: buildableKind,
-      isOptional: isOptional
+      optionality: optionality
     )
   }
 
@@ -58,7 +58,7 @@ extension Child {
   }
 
   public var defaultValue: ExprSyntax? {
-    if isOptional || isUnexpectedNodes {
+    if optionality != nil || isUnexpectedNodes {
       if buildableType.isBaseType && kind.isNodeChoicesEmpty {
         return ExprSyntax("\(buildableType.buildable).none")
       } else {
@@ -131,7 +131,7 @@ extension Child {
     }
 
     var preconditionChoices: [ExprSyntax] = []
-    if buildableType.isOptional {
+    if buildableType.optionality != nil {
       preconditionChoices.append(
         ExprSyntax(
           SequenceExprSyntax {

CodeGeneration/Sources/Utils/SyntaxBuildableType.swift

@@ -20,10 +20,10 @@ import SyntaxSupport
 /// kind.
 public struct SyntaxBuildableType: Hashable {
   public let kind: SyntaxOrTokenNodeKind
-  public let isOptional: Bool
+  public let optionality: Optionality?
 
-  public init(kind: SyntaxOrTokenNodeKind, isOptional: Bool = false) {
-    self.isOptional = isOptional
+  public init(kind: SyntaxOrTokenNodeKind, optionality: Optionality? = nil) {
+    self.optionality = optionality
     self.kind = kind
   }
 
@@ -60,7 +60,7 @@ public struct SyntaxBuildableType: Hashable {
   /// with fixed test), return an expression that can be used as the
   /// default value for a function parameter. Otherwise, return `nil`.
   public var defaultValue: ExprSyntax? {
-    if isOptional {
+    if optionality != nil {
       return ExprSyntax(NilLiteralExprSyntax())
     } else if let token = token {
       if token.text != nil {
@@ -108,7 +108,7 @@ public struct SyntaxBuildableType: Hashable {
       if case .some(.some(let builderInitializableType)) = BUILDER_INITIALIZABLE_TYPES[kind] {
         return Self(
           kind: .node(kind: builderInitializableType),
-          isOptional: isOptional
+          optionality: optionality
         )
       } else {
         return self
@@ -132,8 +132,8 @@ public struct SyntaxBuildableType: Hashable {
   /// The corresponding `*Syntax` type defined in the `SwiftSyntax` module,
   /// which will eventually get built from `SwiftSyntaxBuilder`. If the type
   /// is optional, this terminates with a `?`.
-  public var syntax: TypeSyntax {
-    return optionalWrapped(type: syntaxBaseName)
+  public func syntax(canUseIUO: Bool = true) -> TypeSyntax {
+    return optionalWrapped(type: syntaxBaseName, canUseIUO: canUseIUO)
   }
 
   /// The type that is used for parameters in SwiftSyntaxBuilder that take this
@@ -167,27 +167,33 @@ public struct SyntaxBuildableType: Hashable {
     }
   }
 
-  /// Wraps a type in an optional depending on whether `isOptional` is true.
-  public func optionalWrapped(type: some TypeSyntaxProtocol) -> TypeSyntax {
-    if isOptional {
+  /// Wraps a type in an optional depending on whether `optionality` is true.
+  public func optionalWrapped(type: some TypeSyntaxProtocol, canUseIUO: Bool = true) -> TypeSyntax {
+    switch optionality {
+    case .some(.implicitlyUnwrapped):
+      if canUseIUO {
+        return TypeSyntax(ImplicitlyUnwrappedOptionalTypeSyntax(wrappedType: type))
+      }
+      fallthrough
+    case .some(.normal):
       return TypeSyntax(OptionalTypeSyntax(wrappedType: type))
-    } else {
+    case nil:
       return TypeSyntax(type)
     }
   }
 
-  /// Wraps a type in an optional chaining depending on whether `isOptional` is true.
+  /// Wraps a type in an optional chaining depending on whether `optionality` is true.
   public func optionalChained(expr: some ExprSyntaxProtocol) -> ExprSyntax {
-    if isOptional {
+    if optionality != nil {
       return ExprSyntax(OptionalChainingExprSyntax(expression: expr))
     } else {
       return ExprSyntax(expr)
     }
   }
 
-  /// Wraps a type in a force unwrap expression depending on whether `isOptional` is true.
+  /// Wraps a type in a force unwrap expression depending on whether `optionality` is true.
   public func forceUnwrappedIfNeeded(expr: some ExprSyntaxProtocol) -> ExprSyntax {
-    if isOptional {
+    if optionality != nil {
       return ExprSyntax(ForceUnwrapExprSyntax(expression: expr))
     } else {
       return ExprSyntax(expr)

CodeGeneration/Sources/generate-swift-syntax/LayoutNode+Extensions.swift

@@ -31,11 +31,11 @@ extension LayoutNode {
         paramType = child.syntaxNodeKind.syntaxType
       }
 
-      if child.isOptional {
+      if let optionality = child.optionality {
         if paramType.is(SomeOrAnyTypeSyntax.self) {
-          paramType = "(\(paramType))?"
+          paramType = "(\(paramType))\(raw: optionality.rawValue)"
         } else {
-          paramType = "\(paramType)?"
+          paramType = "\(paramType)\(raw: optionality.rawValue)"
         }
       }
 
@@ -125,7 +125,7 @@ extension LayoutNode {
         let builderInitializableType = child.buildableType.builderInitializableType
         if child.buildableType.builderInitializableType != child.buildableType {
           let param = Node.from(type: child.buildableType).layoutNode!.singleNonDefaultedChild
-          if child.isOptional {
+          if child.optionality != nil {
             produceExpr = ExprSyntax(
               "\(childName)Builder().map { \(child.buildableType.syntaxBaseName)(\(param.labelDeclName): $0) }"
             )
@@ -139,7 +139,7 @@ extension LayoutNode {
         }
         builderParameters.append(
           FunctionParameterSyntax(
-            "@\(builderInitializableType.resultBuilderType) \(childName)Builder: () throws-> \(builderInitializableType.syntax)"
+            "@\(builderInitializableType.resultBuilderType) \(childName)Builder: () throws -> \(builderInitializableType.syntax(canUseIUO: false))"
           )
         )
       } else {

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/RawSyntaxNodesFile.swift

@@ -161,7 +161,7 @@ func rawSyntaxNodesFile(nodesStartingWith: [Character]) -> SourceFileSyntax {
               let list = ExprListSyntax {
                 ExprSyntax("layout.initialize(repeating: nil)")
                 for (index, child) in node.children.enumerated() {
-                  let optionalMark = child.isOptional ? "?" : ""
+                  let optionalMark = (child.optionality != nil) ? "?" : ""
 
                   ExprSyntax(
                     "layout[\(raw: index)] = \(child.baseCallName)\(raw: optionalMark).raw"
@@ -188,7 +188,7 @@ func rawSyntaxNodesFile(nodesStartingWith: [Character]) -> SourceFileSyntax {
             try VariableDeclSyntax(
               "public var \(child.varDeclName): Raw\(child.buildableType.buildable)"
             ) {
-              let exclamationMark = child.isOptional ? "" : "!"
+              let exclamationMark = child.optionality != nil ? "" : "!"
 
               if child.syntaxNodeKind == .syntax {
                 ExprSyntax("layoutView.children[\(raw: index)]\(raw: exclamationMark)")
@@ -232,7 +232,7 @@ fileprivate extension Child {
     var paramType: TypeSyntax
     if !kind.isNodeChoicesEmpty {
       paramType = "\(syntaxChoicesType)"
-    } else if hasBaseType && !isOptional {
+    } else if hasBaseType && optionality == nil {
       // we restrict the use of generic type to non-optional parameter types, otherwise call sites would no longer be
       // able to just pass `nil` to this parameter without specializing `(some Raw<Kind>SyntaxNodeProtocol)?`
       //

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/RenamedChildrenCompatibilityFile.swift

@@ -22,7 +22,14 @@ let renamedChildrenCompatibilityFile = try! SourceFileSyntax(leadingTrivia: copy
         if let deprecatedVarName = child.deprecatedVarName {
           let childType: TypeSyntax =
             child.kind.isNodeChoicesEmpty ? child.syntaxNodeKind.syntaxType : child.syntaxChoicesType
-          let type = child.isOptional ? TypeSyntax("\(childType)?") : childType
+          let type = switch child.optionality {
+          case .none:
+            childType
+          case .normal:
+            TypeSyntax("\(childType)?")
+          case .implicitlyUnwrapped:
+            TypeSyntax("\(childType)!")
+          }
 
           DeclSyntax(
             """

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/SyntaxNodesFile.swift

@@ -135,7 +135,7 @@ func syntaxNode(nodesStartingWith: [Character]) -> SourceFileSyntax {
 
           let childType: TypeSyntax =
             child.kind.isNodeChoicesEmpty ? child.syntaxNodeKind.syntaxType : child.syntaxChoicesType
-          let type = child.isOptional ? TypeSyntax("\(childType)?") : TypeSyntax("\(childType)")
+          let type = TypeSyntax("\(childType)\(raw: child.optionality?.rawValue ?? "")")
 
           try! VariableDeclSyntax(
             """
@@ -145,7 +145,7 @@ func syntaxNode(nodesStartingWith: [Character]) -> SourceFileSyntax {
           ) {
             AccessorDeclSyntax(accessorSpecifier: .keyword(.get)) {
               let optionalityMarker: TokenSyntax =
-                child.isOptional ? .infixQuestionMarkToken() : .exclamationMarkToken()
+                child.optionality != nil ? .infixQuestionMarkToken() : .exclamationMarkToken()
               StmtSyntax("return Syntax(self).child(at: \(raw: index))\(optionalityMarker).cast(\(childType).self)")
             }
 

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/SyntaxTraitsFile.swift

@@ -28,7 +28,7 @@ let syntaxTraitsFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     ) {
       for child in trait.children {
-        let questionMark = child.isOptional ? "?" : ""
+        let questionMark = child.optionality?.rawValue ?? ""
 
         DeclSyntax(
           """

CodeGeneration/Tests/ValidateSyntaxNodes/ValidateSyntaxNodes.swift

@@ -81,7 +81,7 @@ fileprivate extension ChildKind {
 fileprivate extension Child {
   func hasSameType(as other: Child) -> Bool {
     return identifier.description == other.identifier.description && kind.hasSameType(as: other.kind)
-      && isOptional == other.isOptional
+      && optionality == other.optionality
   }
 
   func isFollowedByColonToken(in node: LayoutNode) -> Bool {
@@ -816,7 +816,7 @@ class ValidateSyntaxNodes: XCTestCase {
 
     for node in SYNTAX_NODES.compactMap(\.layoutNode) {
       for child in node.children {
-        if case .collection = child.kind, child.isOptional, !child.isUnexpectedNodes {
+        if case .collection = child.kind, child.optionality != nil, !child.isUnexpectedNodes {
           failures.append(
             ValidationFailure(
               node: node.kind,

Sources/SwiftLexicalLookup/Scopes/ScopeImplementations.swift

@@ -645,7 +645,8 @@ import SwiftSyntax
     var results: [LookupResult] = []
 
     if let primaryAssociatedTypeClause,
-      primaryAssociatedTypeClause.range.contains(lookUpPosition)
+       primaryAssociatedTypeClause.range.contains(lookUpPosition),
+       let memberBlock
     {
       results = memberBlock.lookupAssociatedTypeDeclarations(
         identifier,