Add a --experimental-prepare-for-indexing-no-skipping command line option to prepare a target without function body skipping (#7720)

ahoppen · web-flow · commit e695f5993e66 · 2024-07-02T06:58:27.000+02:00
The flags for lazy type checking and function body skipping are still
experimental and haven’t been thoroughly tested in the wild yet. In case
they are causing issues, add an option to prepare a target without these
flags, which users could enable as a workaround.

rdar://130333568
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 .build
+.index-build
 DerivedData
 /.previous-build
 xcuserdata
diff --git a/Sources/Build/BuildDescription/SwiftModuleBuildDescription.swift b/Sources/Build/BuildDescription/SwiftModuleBuildDescription.swift
@@ -338,7 +338,7 @@ public final class SwiftModuleBuildDescription {
             return
         }
 
-        guard 
+        guard
             self.buildParameters.triple.isDarwin() &&
             self.buildParameters.testingParameters.experimentalTestOutput
         else {
@@ -541,14 +541,19 @@ public final class SwiftModuleBuildDescription {
             args += ["-emit-module-interface-path", self.parseableModuleInterfaceOutputPath.pathString]
         }
 
-        if self.buildParameters.prepareForIndexing {
+        switch self.buildParameters.prepareForIndexing {
+        case .off:
+            break
+        case .on:
+            args += ["-Xfrontend", "-experimental-lazy-typecheck",]
             if !args.contains("-enable-testing") {
                 // enable-testing needs the non-exportable-decls
                 args += ["-Xfrontend", "-experimental-skip-non-exportable-decls"]
             }
+            fallthrough
+        case .noLazy:
             args += [
                 "-Xfrontend", "-experimental-skip-all-function-bodies",
-                "-Xfrontend", "-experimental-lazy-typecheck",
                 "-Xfrontend", "-experimental-allow-module-with-compiler-errors",
                 "-Xfrontend", "-empty-abi-descriptor"
             ]
@@ -594,7 +599,7 @@ public final class SwiftModuleBuildDescription {
 
         // Pass `-user-module-version` for versioned packages that aren't pre-releases.
         if
-          let version = package.manifest.version, 
+          let version = package.manifest.version,
           version.prereleaseIdentifiers.isEmpty &&
           version.buildMetadataIdentifiers.isEmpty &&
           toolsVersion >= .v6_0
@@ -607,7 +612,7 @@ public final class SwiftModuleBuildDescription {
             isPackageNameSupported: self.buildParameters.driverParameters.isPackageAccessModifierSupported
         )
         args += try self.macroArguments()
-        
+
         // rdar://117578677
         // Pass -fno-omit-frame-pointer to support backtraces
         // this can be removed once the backtracer uses DWARF instead of frame pointers
@@ -621,7 +626,7 @@ public final class SwiftModuleBuildDescription {
 
         return args
     }
-    
+
     /// Determines the arguments needed to run `swift-symbolgraph-extract` for
     /// this module.
     package func symbolGraphExtractArguments() throws -> [String] {
diff --git a/Sources/Build/BuildManifest/LLBuildManifestBuilder+Swift.swift b/Sources/Build/BuildManifest/LLBuildManifestBuilder+Swift.swift
@@ -41,7 +41,7 @@ extension LLBuildManifestBuilder {
         let inputs = try self.computeSwiftCompileCmdInputs(target)
 
         // Outputs.
-        let objectNodes = target.buildParameters.prepareForIndexing ? [] : try target.objects.map(Node.file)
+        let objectNodes = target.buildParameters.prepareForIndexing == .off ? try target.objects.map(Node.file) : []
         let moduleNode = Node.file(target.moduleOutputPath)
         let cmdOutputs = objectNodes + [moduleNode]
 
@@ -395,7 +395,7 @@ extension LLBuildManifestBuilder {
             isLibrary: isLibrary,
             wholeModuleOptimization: target.buildParameters.configuration == .release,
             outputFileMapPath: try target.writeOutputFileMap(), // FIXME: Eliminate side effect.
-            prepareForIndexing: target.buildParameters.prepareForIndexing
+            prepareForIndexing: target.buildParameters.prepareForIndexing != .off
         )
     }
 
@@ -433,7 +433,7 @@ extension LLBuildManifestBuilder {
             if target.underlying is ProvidedLibraryModule { return }
 
             // Depend on the binary for executable targets.
-            if target.type == .executable && !prepareForIndexing {
+            if target.type == .executable && prepareForIndexing == .off {
                 // FIXME: Optimize.
                 if let productDescription = try plan.productMap.values.first(where: {
                     try $0.product.type == .executable && $0.product.executableModule.id == target.id
@@ -447,7 +447,7 @@ extension LLBuildManifestBuilder {
             case .swift(let target)?:
                 inputs.append(file: target.moduleOutputPath)
             case .clang(let target)?:
-                if prepareForIndexing {
+                if prepareForIndexing != .off {
                     // In preparation, we're only building swiftmodules
                     // propagate the dependency to the header files in this target
                     for header in target.clangTarget.headers {
diff --git a/Sources/Build/BuildOperation.swift b/Sources/Build/BuildOperation.swift
@@ -1117,9 +1117,9 @@ extension BuildDescription {
             fileSystem: fileSystem,
             observabilityScope: config.observabilityScope
         )
-        let buildManifest = plan.destinationBuildParameters.prepareForIndexing
-            ? try llbuild.generatePrepareManifest(at: config.manifestPath)
-            : try llbuild.generateManifest(at: config.manifestPath)
+        let buildManifest = plan.destinationBuildParameters.prepareForIndexing == .off
+            ? try llbuild.generateManifest(at: config.manifestPath)
+            : try llbuild.generatePrepareManifest(at: config.manifestPath)
 
         let swiftCommands = llbuild.manifest.getCmdToolMap(kind: SwiftCompilerTool.self)
         let swiftFrontendCommands = llbuild.manifest.getCmdToolMap(kind: SwiftFrontendTool.self)
diff --git a/Sources/CoreCommands/Options.swift b/Sources/CoreCommands/Options.swift
@@ -439,9 +439,21 @@ public struct BuildOptions: ParsableArguments {
     @Flag(help: "Enable or disable indexing-while-building feature")
     public var indexStoreMode: StoreMode = .autoIndexStore
 
+    /// Instead of building the target, perform the minimal amount of work to prepare it for indexing.
+    ///
+    /// This builds Swift module files for all dependencies but skips generation of object files. It also continues
+    /// building modules in the presence of compilation errors.
     @Flag(name: .customLong("experimental-prepare-for-indexing"), help: .hidden)
     var prepareForIndexing: Bool = false
 
+    /// Don't pass `-experimental-lazy-typecheck` during preparation.
+    ///
+    /// This is intended as a workaround if lazy type checking is causing compiler crashes.
+    ///
+    /// Only applicable in conjunction with `--experimental-prepare-for-indexing`
+    @Flag(name: .customLong("experimental-prepare-for-indexing-no-lazy"), help: .hidden)
+    var prepareForIndexingNoLazy: Bool = false
+
     /// Whether to enable generation of `.swiftinterface`s alongside `.swiftmodule`s.
     @Flag(name: .customLong("enable-parseable-module-interfaces"))
     public var shouldEnableParseableModuleInterfaces: Bool = false
diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift
@@ -773,6 +773,13 @@ public final class SwiftCommandState {
             observabilityScope.emit(warning: Self.entitlementsMacOSWarning)
         }
 
+        let prepareForIndexingMode: BuildParameters.PrepareForIndexingMode =
+            switch (options.build.prepareForIndexing, options.build.prepareForIndexingNoLazy) {
+                case (false, _): .off
+                case (true, false): .on
+                case (true, true): .noLazy
+            }
+
         return try BuildParameters(
             destination: destination,
             dataPath: dataPath,
@@ -786,7 +793,7 @@ public final class SwiftCommandState {
             sanitizers: options.build.enabledSanitizers,
             indexStoreMode: options.build.indexStoreMode.buildParameter,
             isXcodeBuildSystemEnabled: options.build.buildSystem == .xcode,
-            prepareForIndexing: prepareForIndexing ?? options.build.prepareForIndexing,
+            prepareForIndexing: prepareForIndexingMode,
             debuggingParameters: .init(
                 debugInfoFormat: options.build.debugInfoFormat.buildParameter,
                 triple: triple,
diff --git a/Sources/SPMBuildCore/BuildParameters/BuildParameters+Debugging.swift b/Sources/SPMBuildCore/BuildParameters/BuildParameters+Debugging.swift
@@ -63,7 +63,7 @@ extension BuildParameters {
 
     /// The debugging strategy according to the current build parameters.
     public var debuggingStrategy: DebuggingStrategy? {
-        guard configuration == .debug, !prepareForIndexing else {
+        guard configuration == .debug, prepareForIndexing == .off else {
             return nil
         }
 
diff --git a/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift b/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift
@@ -16,6 +16,17 @@ import PackageModel
 import PackageGraph
 
 public struct BuildParameters: Encodable {
+    public enum PrepareForIndexingMode: Encodable {
+        /// Perform a normal build and don't prepare for indexing
+        case off
+        /// Prepare for indexing but don't pass `-experimental-lazy-typecheck`.
+        ///
+        /// This is intended as a workaround if lazy type checking is causing compiler crashes.
+        case noLazy
+        /// Do minimal build to prepare for indexing
+        case on
+    }
+
     /// Mode for the indexing-while-building feature.
     public enum IndexStoreMode: String, Encodable {
         /// Index store should be enabled.
@@ -115,7 +126,7 @@ public struct BuildParameters: Encodable {
     public var shouldSkipBuilding: Bool
 
     /// Do minimal build to prepare for indexing
-    public var prepareForIndexing: Bool
+    public var prepareForIndexing: PrepareForIndexingMode
 
     /// Build parameters related to debugging.
     public var debuggingParameters: Debugging
@@ -147,7 +158,7 @@ public struct BuildParameters: Encodable {
         indexStoreMode: IndexStoreMode = .auto,
         isXcodeBuildSystemEnabled: Bool = false,
         shouldSkipBuilding: Bool = false,
-        prepareForIndexing: Bool = false,
+        prepareForIndexing: PrepareForIndexingMode = .off,
         debuggingParameters: Debugging? = nil,
         driverParameters: Driver = .init(),
         linkingParameters: Linking = .init(),
diff --git a/Sources/_InternalTestSupport/MockBuildTestHelper.swift b/Sources/_InternalTestSupport/MockBuildTestHelper.swift
@@ -90,7 +90,7 @@ public func mockBuildParameters(
     linkerDeadStrip: Bool = true,
     linkTimeOptimizationMode: BuildParameters.LinkTimeOptimizationMode? = nil,
     omitFramePointers: Bool? = nil,
-    prepareForIndexing: Bool = false
+    prepareForIndexing: BuildParameters.PrepareForIndexingMode = .off
 ) -> BuildParameters {
     try! BuildParameters(
         destination: destination,
diff --git a/Tests/BuildTests/PrepareForIndexTests.swift b/Tests/BuildTests/PrepareForIndexTests.swift
@@ -27,8 +27,8 @@ class PrepareForIndexTests: XCTestCase {
         let (graph, fs, scope) = try macrosPackageGraph()
 
         let plan = try BuildPlan(
-            destinationBuildParameters: mockBuildParameters(destination: .target, prepareForIndexing: true),
-            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: false),
+            destinationBuildParameters: mockBuildParameters(destination: .target, prepareForIndexing: .on),
+            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: .off),
             graph: graph,
             fileSystem: fs,
             observabilityScope: scope
@@ -72,8 +72,8 @@ class PrepareForIndexTests: XCTestCase {
         let (graph, fs, scope) = try trivialPackageGraph()
 
         let plan = try BuildPlan(
-            destinationBuildParameters: mockBuildParameters(destination: .target, prepareForIndexing: true),
-            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: false),
+            destinationBuildParameters: mockBuildParameters(destination: .target, prepareForIndexing: .on),
+            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: .off),
             graph: graph,
             fileSystem: fs,
             observabilityScope: scope
@@ -116,8 +116,8 @@ class PrepareForIndexTests: XCTestCase {
 
         // Under debug, enable-testing is turned on by default. Make sure the flag is not added.
         let debugPlan = try BuildPlan(
-            destinationBuildParameters: mockBuildParameters(destination: .target, config: .debug, prepareForIndexing: true),
-            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: false),
+            destinationBuildParameters: mockBuildParameters(destination: .target, config: .debug, prepareForIndexing: .on),
+            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: .off),
             graph: graph,
             fileSystem: fs,
             observabilityScope: observability.topScope
@@ -137,8 +137,8 @@ class PrepareForIndexTests: XCTestCase {
 
         // Under release, enable-testing is turned off by default so we should see our flag
         let releasePlan = try BuildPlan(
-            destinationBuildParameters: mockBuildParameters(destination: .target, config: .release, prepareForIndexing: true),
-            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: false),
+            destinationBuildParameters: mockBuildParameters(destination: .target, config: .release, prepareForIndexing: .on),
+            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: .off),
             graph: graph,
             fileSystem: fs,
             observabilityScope: observability.topScope
@@ -156,4 +156,31 @@ class PrepareForIndexTests: XCTestCase {
                 && !swiftCommand.otherArguments.contains("-enable-testing")
         }).count, 1)
     }
+
+    func testPrepareNoLazy() throws {
+        let (graph, fs, scope) = try macrosPackageGraph()
+
+        let plan = try BuildPlan(
+            destinationBuildParameters: mockBuildParameters(destination: .target, prepareForIndexing: .noLazy),
+            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: .off),
+            graph: graph,
+            fileSystem: fs,
+            observabilityScope: scope
+        )
+
+        let builder = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: scope)
+        let manifest = try builder.generatePrepareManifest(at: "/manifest")
+
+        // Ensure swiftmodules built with correct arguments
+        let coreCommands = manifest.commands.values.filter {
+            $0.tool.outputs.contains(where: {
+                $0.name.hasSuffix("debug/Core.build/Core.swiftmodule")
+            })
+        }
+        XCTAssertEqual(coreCommands.count, 1)
+        let coreSwiftc = try XCTUnwrap(coreCommands.first?.tool as? SwiftCompilerTool)
+        XCTAssertFalse(coreSwiftc.otherArguments.contains("-experimental-lazy-typecheck"))
+        XCTAssertTrue(coreSwiftc.otherArguments.contains("-experimental-allow-module-with-compiler-errors"))
+    }
+
 }

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`.build`
	`2`	`+.index-build`
`2`	`3`	`DerivedData`
`3`	`4`	`/.previous-build`
`4`	`5`	`xcuserdata`
Original file line number	Diff line number	Diff line change
`@@ -63,7 +63,7 @@ extension BuildParameters {`
`63`	`63`
`64`	`64`	`/// The debugging strategy according to the current build parameters.`
`65`	`65`	`public var debuggingStrategy: DebuggingStrategy? {`
`66`		`- guard configuration == .debug, !prepareForIndexing else {`
	`66`	`+ guard configuration == .debug, prepareForIndexing == .off else {`
`67`	`67`	`return nil`
`68`	`68`	`}`
`69`	`69`