Plan swiftinterface job and verify interface job together

Always plan the job producing swiftinterface and verifying
swiftinterface together. This make sure the swiftinterface exists on
disk or in CAS when verification job is running.

rdar://147501517

Author: cachemeifyoucan

Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift

@@ -51,11 +51,12 @@ extension IncrementalCompilationState {
 
     public func compute(batchJobFormer: inout Driver) throws -> FirstWave {
       return try blockingConcurrentAccessOrMutation {
-        let (initiallySkippedCompileJobs, mandatoryJobsInOrder) =
+        let (initiallySkippedCompileJobs, mandatoryJobsInOrder, afterCompiles) =
         try computeInputsAndGroups(batchJobFormer: &batchJobFormer)
         return FirstWave(
           initiallySkippedCompileJobs: initiallySkippedCompileJobs,
-          mandatoryJobsInOrder: mandatoryJobsInOrder)
+          mandatoryJobsInOrder: mandatoryJobsInOrder,
+          jobsAfterCompiles: afterCompiles)
       }
     }
   }
@@ -74,34 +75,39 @@ extension IncrementalCompilationState.FirstWaveComputer {
   /// listed in fingerprintExternalDependencies.
   private func computeInputsAndGroups(batchJobFormer: inout Driver)
   throws -> (initiallySkippedCompileJobs: [TypedVirtualPath: Job],
-             mandatoryJobsInOrder: [Job])
+             mandatoryJobsInOrder: [Job],
+             jobsAfterCompiles: [Job])
   {
     let compileJobs =
       Dictionary(uniqueKeysWithValues:
                   jobsInPhases.compileJobs.map { ($0.primaryInputs[0], $0) })
     let buildRecord = self.moduleDependencyGraph.buildRecord
-    let jobCreatingPch = jobsInPhases.beforeCompiles.first(where: {$0.kind == .generatePCH})
-    guard !buildRecord.inputInfos.isEmpty else {
-      func everythingIsMandatory()
-        throws -> (initiallySkippedCompileJobs: [TypedVirtualPath: Job],
-                   mandatoryJobsInOrder: [Job])
-      {
-        let mandatoryCompileJobsInOrder = self.inputFiles.swiftSourceFiles.compactMap {
-          input -> Job? in
-          compileJobs[input.typedFile]
-        }
+    let jobCreatingPch = jobsInPhases.beforeCompiles.first { $0.kind == .generatePCH }
+
+    func everythingIsMandatory()
+      throws -> (initiallySkippedCompileJobs: [TypedVirtualPath: Job],
+                 mandatoryJobsInOrder: [Job],
+                 jobsAfterCompiles: [Job])
+    {
+      let mandatoryCompileJobsInOrder = self.inputFiles.swiftSourceFiles.compactMap {
+        input -> Job? in
+        compileJobs[input.typedFile]
+      }
 
-        let mandatoryJobsInOrder = try
-        jobsInPhases.beforeCompiles +
-        batchJobFormer.formBatchedJobs(
-          mandatoryCompileJobsInOrder,
-          showJobLifecycle: showJobLifecycle,
-          jobCreatingPch: jobCreatingPch)
+      let mandatoryJobsInOrder = try
+      jobsInPhases.beforeCompiles +
+      batchJobFormer.formBatchedJobs(
+        mandatoryCompileJobsInOrder,
+        showJobLifecycle: showJobLifecycle,
+        jobCreatingPch: jobCreatingPch)
+
+      moduleDependencyGraph.setPhase(to: .buildingAfterEachCompilation)
+      return (initiallySkippedCompileJobs: [:],
+              mandatoryJobsInOrder: mandatoryJobsInOrder,
+              jobsAfterCompiles: jobsInPhases.afterCompiles)
+    }
 
-        moduleDependencyGraph.setPhase(to: .buildingAfterEachCompilation)
-        return (initiallySkippedCompileJobs: [:],
-                mandatoryJobsInOrder: mandatoryJobsInOrder)
-      }
+    guard !buildRecord.inputInfos.isEmpty else {
       return try everythingIsMandatory()
     }
     moduleDependencyGraph.setPhase(to: .updatingAfterCompilation)
@@ -115,9 +121,7 @@ extension IncrementalCompilationState.FirstWaveComputer {
 
     let mandatoryCompileJobsInOrder = inputFiles.compactMap {
       input -> Job? in
-      initiallySkippedInputs.contains(input)
-        ? nil
-        : compileJobs[input]
+      return initiallySkippedInputs.contains(input) ? nil : compileJobs[input]
     }
 
     let batchedCompilationJobs = try batchJobFormer.formBatchedJobs(
@@ -129,9 +133,16 @@ extension IncrementalCompilationState.FirstWaveComputer {
     // we can skip running `beforeCompiles` jobs if we also ensure that none of the `afterCompiles` jobs
     // have any dependencies on them.
     let skipAllJobs = batchedCompilationJobs.isEmpty ? !nonVerifyAfterCompileJobsDependOnBeforeCompileJobs() : false
-    let mandatoryJobsInOrder = skipAllJobs ? [] : jobsInPhases.beforeCompiles + batchedCompilationJobs
+
+    // Schedule emitModule job together with verify module interface job.
+    let beforeCompileJobs = skipAllJobs ? [] : jobsInPhases.beforeCompiles
+    let afterCompileJobs = jobsInPhases.afterCompiles.compactMap { job in
+      skipAllJobs && job.kind == .verifyModuleInterface ? nil : job
+    }
+    let mandatoryJobsInOrder = beforeCompileJobs + batchedCompilationJobs
     return (initiallySkippedCompileJobs: initiallySkippedCompileJobs,
-            mandatoryJobsInOrder: mandatoryJobsInOrder)
+            mandatoryJobsInOrder: mandatoryJobsInOrder,
+            jobsAfterCompiles: afterCompileJobs)
   }
 
   /// Determine if any of the jobs in the `afterCompiles` group depend on outputs produced by jobs in

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift

@@ -69,6 +69,8 @@ extension IncrementalCompilationState {
     /// for the first wave to execute.
     /// The primaries could be other than .swift files, i.e. .sib
     let mandatoryJobsInOrder: [Job]
+
+    let jobsAfterCompiles: [Job]
   }
 }
 

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift

@@ -77,7 +77,7 @@ public final class IncrementalCompilationState {
       jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH}),
       &driver)
     self.mandatoryJobsInOrder = firstWave.mandatoryJobsInOrder
-    self.jobsAfterCompiles = jobsInPhases.afterCompiles
+    self.jobsAfterCompiles = firstWave.jobsAfterCompiles
   }
 
   /// Allow concurrent access to while preventing mutation of ``IncrementalCompilationState/protectedState``

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -815,6 +815,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
       }
       """))
 
+      let swiftInterfaceOutput = path.appending(component: "Test.swiftinterface")
       let cHeadersPath: AbsolutePath =
           try testInputsPath.appending(component: "ExplicitModuleBuilds")
                             .appending(component: "CHeaders")
@@ -825,11 +826,12 @@ final class ExplicitModuleBuildTests: XCTestCase {
       let invocationArguments = ["swiftc",
                                  "-incremental", "-c",
                                  "-emit-module",
-                                 "-enable-library-evolution", "-emit-module-interface", "-driver-show-incremental", "-v",
+                                 "-enable-library-evolution", "-emit-module-interface", "-driver-show-incremental",
                                  "-Xcc", "-Xclang", "-Xcc", "-fbuiltin-headers-in-system-modules",
                                  "-I", cHeadersPath.nativePathString(escaped: true),
                                  "-I", swiftModuleInterfacesPath.nativePathString(escaped: true),
                                  "-explicit-module-build",
+                                 "-emit-module-interface-path", swiftInterfaceOutput.nativePathString(escaped: true),
                                  "-output-file-map", outputFileMap.nativePathString(escaped: true),
                                  "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
                                  "-working-directory", path.nativePathString(escaped: true),
@@ -843,6 +845,15 @@ final class ExplicitModuleBuildTests: XCTestCase {
       let incrementalJobs = try incrementalDriver.planBuild()
       try incrementalDriver.run(jobs: incrementalJobs)
       XCTAssertFalse(incrementalDriver.diagnosticEngine.hasErrors)
+
+      // TODO: emitModule job should run again if interface is deleted.
+      // try localFileSystem.removeFileTree(swiftInterfaceOutput)
+
+      // This should be a null build but it is actually building the main module due to the previous build of all the modules.
+      var reDriver = try Driver(args: invocationArguments + ["-color-diagnostics"])
+      let reJobs = try reDriver.planBuild()
+      XCTAssertFalse(reJobs.contains { $0.kind == .emitModule })
+      XCTAssertFalse(reJobs.contains { $0.kind == .verifyModuleInterface })
     }
   }