Retry IO access to compilation outputs and try to recover when a file is unavailable

Gedochao · Gedochao · commit 4ff01712f6d2 · 2024-12-31T08:47:28.000+01:00
diff --git a/modules/build/src/main/scala/scala/build/Build.scala b/modules/build/src/main/scala/scala/build/Build.scala
@@ -52,16 +52,17 @@ object Build {
     output: os.Path,
     diagnostics: Option[Seq[(Either[String, os.Path], bsp4j.Diagnostic)]],
     generatedSources: Seq[GeneratedSource],
-    isPartial: Boolean
+    isPartial: Boolean,
+    logger: Logger
   ) extends Build {
     def success: Boolean                  = true
     def successfulOpt: Some[this.type]    = Some(this)
     def outputOpt: Some[os.Path]          = Some(output)
     def dependencyClassPath: Seq[os.Path] = sources.resourceDirs ++ artifacts.classPath
     def fullClassPath: Seq[os.Path]       = Seq(output) ++ dependencyClassPath
-    private lazy val mainClassesFoundInProject: Seq[String] = MainClass.find(output).sorted
+    private lazy val mainClassesFoundInProject: Seq[String] = MainClass.find(output, logger).sorted
     private lazy val mainClassesFoundOnExtraClasspath: Seq[String] =
-      options.classPathOptions.extraClassPath.flatMap(MainClass.find).sorted
+      options.classPathOptions.extraClassPath.flatMap(MainClass.find(_, logger)).sorted
     private lazy val mainClassesFoundInUserExtraDependencies: Seq[String] =
       artifacts.jarsForUserExtraDependencies.flatMap(MainClass.findInDependency).sorted
     def foundMainClasses(): Seq[String] = {
@@ -1166,27 +1167,28 @@ object Build {
 
     if (success)
       Successful(
-        inputs,
-        options,
-        scalaParams,
-        scope,
-        sources,
-        artifacts,
-        project,
-        classesDir0,
-        buildClient.diagnostics,
-        generatedSources,
-        partial
+        inputs = inputs,
+        options = options,
+        scalaParams = scalaParams,
+        scope = scope,
+        sources = sources,
+        artifacts = artifacts,
+        project = project,
+        output = classesDir0,
+        diagnostics = buildClient.diagnostics,
+        generatedSources = generatedSources,
+        isPartial = partial,
+        logger = logger
       )
     else
       Failed(
-        inputs,
-        options,
-        scope,
-        sources,
-        artifacts,
-        project,
-        buildClient.diagnostics
+        inputs = inputs,
+        options = options,
+        scope = scope,
+        sources = sources,
+        artifacts = artifacts,
+        project = project,
+        diagnostics = buildClient.diagnostics
       )
   }
 
diff --git a/modules/build/src/main/scala/scala/build/internal/MainClass.scala b/modules/build/src/main/scala/scala/build/internal/MainClass.scala
@@ -4,11 +4,13 @@ import org.objectweb.asm
 import org.objectweb.asm.ClassReader
 
 import java.io.{ByteArrayInputStream, InputStream}
+import java.nio.file.NoSuchFileException
 import java.util.jar.{Attributes, JarFile, JarInputStream, Manifest}
 import java.util.zip.ZipEntry
 
 import scala.build.input.Element
 import scala.build.internal.zip.WrappedZipInputStream
+import scala.build.{Logger, retry}
 
 object MainClass {
 
@@ -44,29 +46,53 @@ object MainClass {
       if (foundMainClass) nameOpt else None
   }
 
-  def findInClass(path: os.Path): Iterator[String] =
-    findInClass(os.read.inputStream(path))
-  def findInClass(is: InputStream): Iterator[String] =
+  def findInClass(path: os.Path, logger: Logger): Iterator[String] =
     try {
-      val reader  = new ClassReader(is)
-      val checker = new MainMethodChecker
-      reader.accept(checker, 0)
-      checker.mainClassOpt.iterator
+      val is = retry()(logger)(os.read.inputStream(path))
+      findInClass(is, logger)
+    }
+    catch {
+      case e: NoSuchFileException =>
+        e.getStackTrace.foreach(ste => logger.debug(ste.toString))
+        logger.log(s"Class file $path not found: $e, trying to recover...")
+        Iterator.empty
+    }
+  def findInClass(is: InputStream, logger: Logger): Iterator[String] =
+    try
+      retry()(logger) {
+        val reader  = new ClassReader(is)
+        val checker = new MainMethodChecker
+        reader.accept(checker, 0)
+        checker.mainClassOpt.iterator
+      }
+    catch {
+      case e: ArrayIndexOutOfBoundsException =>
+        e.getStackTrace.foreach(ste => logger.debug(ste.toString))
+        logger.log(s"Class input stream could not be created: $e, trying to recover...")
+        Iterator.empty
     }
     finally is.close()
 
-  def findInJar(path: os.Path): Iterator[String] = {
-    val content        = os.read.bytes(path)
-    val jarInputStream = WrappedZipInputStream.create(new ByteArrayInputStream(content))
-    jarInputStream.entries().flatMap(ent =>
-      if !ent.isDirectory && ent.getName.endsWith(".class") then {
-        val content     = jarInputStream.readAllBytes()
-        val inputStream = new ByteArrayInputStream(content)
-        findInClass(inputStream)
+  def findInJar(path: os.Path, logger: Logger): Iterator[String] =
+    try
+      retry()(logger) {
+        val content        = os.read.bytes(path)
+        val jarInputStream = WrappedZipInputStream.create(new ByteArrayInputStream(content))
+        jarInputStream.entries().flatMap(ent =>
+          if !ent.isDirectory && ent.getName.endsWith(".class") then {
+            val content     = jarInputStream.readAllBytes()
+            val inputStream = new ByteArrayInputStream(content)
+            findInClass(inputStream, logger)
+          }
+          else Iterator.empty
+        )
       }
-      else Iterator.empty
-    )
-  }
+    catch {
+      case e: NoSuchFileException =>
+        e.getStackTrace.foreach(ste => logger.debug(ste.toString))
+        logger.log(s"JAR file $path not found: $e, trying to recover...")
+        Iterator.empty
+    }
 
   def findInDependency(jar: os.Path): Option[String] =
     jar match {
@@ -79,19 +105,19 @@ object MainClass {
       case _ => None
     }
 
-  def find(output: os.Path): Seq[String] =
+  def find(output: os.Path, logger: Logger): Seq[String] =
     output match {
       case o if os.isFile(o) && o.last.endsWith(".class") =>
-        findInClass(o).toVector
+        findInClass(o, logger).toVector
       case o if os.isFile(o) && o.last.endsWith(".jar") =>
-        findInJar(o).toVector
+        findInJar(o, logger).toVector
       case o if os.isDir(o) =>
         os.walk(o)
           .iterator
           .filter(os.isFile)
           .flatMap {
             case classFilePath if classFilePath.last.endsWith(".class") =>
-              findInClass(classFilePath)
+              findInClass(classFilePath, logger)
             case _ => Iterator.empty
           }
           .toVector
diff --git a/modules/build/src/main/scala/scala/build/package.scala b/modules/build/src/main/scala/scala/build/package.scala
@@ -0,0 +1,31 @@
+package scala
+
+import scala.annotation.tailrec
+import scala.concurrent.duration.DurationConversions.*
+import scala.concurrent.duration.{DurationInt, FiniteDuration}
+
+package object build {
+  def retry[T](
+    maxAttempts: Int = 3,
+    waitDuration: FiniteDuration = 1.seconds
+  )(logger: Logger)(
+    run: => T
+  ): T = {
+    @tailrec
+    def helper(count: Int): T =
+      try run
+      catch {
+        case t: Throwable =>
+          if (count >= maxAttempts)
+            throw new Exception(t)
+          else {
+            t.getStackTrace.foreach(ste => logger.debug(ste.toString))
+            logger.log(s"Caught $t, trying again in $waitDuration…")
+            Thread.sleep(waitDuration.toMillis)
+            helper(count + 1)
+          }
+      }
+
+    helper(1)
+  }
+}
diff --git a/modules/build/src/main/scala/scala/build/postprocessing/AsmPositionUpdater.scala b/modules/build/src/main/scala/scala/build/postprocessing/AsmPositionUpdater.scala
@@ -1,8 +1,12 @@
 package scala.build.postprocessing
 
 import org.objectweb.asm
+import os.SubProcess.InputStream
 
-import scala.build.{Logger, Os}
+import java.io
+import java.nio.file.NoSuchFileException
+
+import scala.build.{Logger, Os, retry}
 
 object AsmPositionUpdater {
 
@@ -53,15 +57,33 @@ object AsmPositionUpdater {
       .filter(os.isFile(_))
       .filter(_.last.endsWith(".class"))
       .foreach { path =>
-        val is = os.read.inputStream(path)
+        val is: io.InputStream =
+          try retry()(logger)(os.read.inputStream(path))
+          catch {
+            case e: NoSuchFileException =>
+              e.getStackTrace.foreach(ste => logger.debug(ste.toString))
+              logger.message(
+                s"Error while processing ${path.relativeTo(Os.pwd)}: $e, trying to recover..."
+              )
+              io.InputStream.nullInputStream()
+          }
         val updateByteCodeOpt =
-          try {
-            val reader  = new asm.ClassReader(is)
-            val writer  = new asm.ClassWriter(reader, 0)
-            val checker = new LineNumberTableClassVisitor(mappings, writer)
-            reader.accept(checker, 0)
-            if (checker.mappedStuff) Some(writer.toByteArray)
-            else None
+          try
+            retry()(logger) {
+              val reader  = new asm.ClassReader(is)
+              val writer  = new asm.ClassWriter(reader, 0)
+              val checker = new LineNumberTableClassVisitor(mappings, writer)
+              reader.accept(checker, 0)
+              if (checker.mappedStuff) Some(writer.toByteArray)
+              else None
+            }
+          catch {
+            case e: ArrayIndexOutOfBoundsException =>
+              e.getStackTrace.foreach(ste => logger.debug(ste.toString))
+              logger.log(
+                s"Error while processing ${path.relativeTo(Os.pwd)}: $e, trying to recover..."
+              )
+              None
           }
           finally is.close()
         for (b <- updateByteCodeOpt) {
diff --git a/modules/build/src/main/scala/scala/build/postprocessing/TastyPostProcessor.scala b/modules/build/src/main/scala/scala/build/postprocessing/TastyPostProcessor.scala
@@ -1,9 +1,11 @@
 package scala.build.postprocessing
 
+import java.nio.file.NoSuchFileException
+
 import scala.build.internal.Constants
 import scala.build.options.BuildOptions
 import scala.build.tastylib.{TastyData, TastyVersions}
-import scala.build.{GeneratedSource, Logger}
+import scala.build.{GeneratedSource, Logger, retry}
 
 case object TastyPostProcessor extends PostProcessor {
 
@@ -51,28 +53,39 @@ case object TastyPostProcessor extends PostProcessor {
     updatedPaths: Map[String, String]
   )(f: os.Path): Unit = {
     logger.debug(s"Reading TASTy file $f")
-    val content = os.read.bytes(f)
-    TastyData.read(content) match {
-      case Left(ex) => logger.debug(s"Ignoring exception during TASty postprocessing: $ex")
-      case Right(data) =>
-        logger.debug(s"Parsed TASTy file $f")
-        var updatedOne = false
-        val updatedData = data.mapNames { n =>
-          updatedPaths.get(n) match {
-            case Some(newName) =>
-              updatedOne = true
-              newName
-            case None =>
-              n
-          }
-        }
-        if (updatedOne) {
-          logger.debug(
-            s"Overwriting ${if (f.startsWith(os.pwd)) f.relativeTo(os.pwd) else f}"
-          )
-          val updatedContent = TastyData.write(updatedData)
-          os.write.over(f, updatedContent)
+    try
+      retry()(logger) {
+        val content = os.read.bytes(f)
+        TastyData.read(content) match {
+          case Left(ex) => logger.debug(s"Ignoring exception during TASty postprocessing: $ex")
+          case Right(data) =>
+            logger.debug(s"Parsed TASTy file $f")
+            var updatedOne = false
+            val updatedData = data.mapNames { n =>
+              updatedPaths.get(n) match {
+                case Some(newName) =>
+                  updatedOne = true
+                  newName
+                case None =>
+                  n
+              }
+            }
+            if (updatedOne) {
+              logger.debug(
+                s"Overwriting ${if (f.startsWith(os.pwd)) f.relativeTo(os.pwd) else f}"
+              )
+              val updatedContent = TastyData.write(updatedData)
+              os.write.over(f, updatedContent)
+            }
         }
+      }
+    catch {
+      case e: NoSuchFileException =>
+        e.getStackTrace.foreach(ste => logger.debug(ste.toString))
+        logger.log(s"Tasty file $f not found: $e, trying to recover...")
+      case e: ArrayIndexOutOfBoundsException =>
+        e.getStackTrace.foreach(ste => logger.debug(ste.toString))
+        logger.log(s"Error while processing $f: $e, trying to recover...")
     }
   }
 }
diff --git a/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala
@@ -1,6 +1,7 @@
 package scala.cli.integration
 
 import com.eed3si9n.expecty.Expecty.expect
+import os.SubProcess
 
 import java.io.{ByteArrayOutputStream, File}
 import java.nio.charset.Charset
@@ -2304,4 +2305,34 @@ abstract class RunTestDefinitions
       expect(err.contains(main2))
     }
   }
+
+  for {
+    (input, code) <- Seq(
+      os.rel / "script.sc" -> """println(args.mkString(" "))""",
+      os.rel / "raw.scala" -> """object Main { def main(args: Array[String]) = println(args.mkString(" ")) }"""
+    )
+    testInputs = TestInputs(input -> code)
+  }
+    test(s"run several instances of $input in parallel") {
+      testInputs.fromRoot {
+        root =>
+          val processes: Seq[(SubProcess, Int)] =
+            (0 to 20).map { i =>
+              os.proc(
+                TestUtil.cli,
+                "run",
+                input.toString(),
+                extraOptions,
+                "--",
+                "iteration",
+                i.toString
+              )
+                .spawn(cwd = root)
+            }.zipWithIndex
+          processes.foreach { case (p, _) => p.waitFor() }
+          processes.foreach { case (p, _) => expect(p.exitCode == 0) }
+          processes.foreach { case (p, i) => expect(p.stdout.trim() == s"iteration $i") }
+      }
+
+    }
 }
