Test local cache in the CI

.github/workflows/ci.yaml

@@ -26,10 +26,10 @@ on:
     - cron: '0 3 * * *'  # Every day at 3 AM
   workflow_dispatch:
 
-# Cancels any in-progress runs within the same group identified by workflow name and GH reference (branch or tag) 
+# Cancels any in-progress runs within the same group identified by workflow name and GH reference (branch or tag)
 # For example it would:
 # - terminate previous PR CI execution after pushing more changes to the same PR branch
-# - terminate previous on-push CI run after merging new PR to main 
+# - terminate previous on-push CI run after merging new PR to main
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
@@ -53,12 +53,65 @@ env:
 # text on stderr and so can break tests which check the output of a program).
 
 jobs:
+  build-non-bootstrapped:
+    name: Build Non-Bootstrapped Compiler
+    runs-on: [self-hosted, Linux]
+    container:
+      image: lampepfl/dotty:2024-10-18
+      options: --cpu-shares 4096
+      volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
+        - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
+        - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
+        - ${{ github.workspace }}/../../cache/general:/root/.cache
+    steps:
+      - name: Set JDK 17 as default
+        run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
+      - name: Checkout cleanup script
+        uses: actions/checkout@v4
+      - name: Cleanup
+        run: .github/workflows/cleanup.sh
+      - name: Git Checkout
+        uses: actions/checkout@v4
+      - name: Add SBT proxy repositories
+        run: cp -vf .github/workflows/repositories /root/.sbt/ ; true
+      - name: Compile Non-Bootstrapped Dotty
+        run: ./project/scripts/sbt "; clean; compile"
+
+  build-bootstrapped:
+    name: Build Bootstrapped Compiler
+    needs: build-non-bootstrapped
+    runs-on: [self-hosted, Linux]
+    container:
+      image: lampepfl/dotty:2024-10-18
+      options: --cpu-shares 4096
+      volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
+        - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
+        - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
+        - ${{ github.workspace }}/../../cache/general:/root/.cache
+    steps:
+      - name: Set JDK 17 as default
+        run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
+      - name: Checkout cleanup script
+        uses: actions/checkout@v4
+      - name: Cleanup
+        run: .github/workflows/cleanup.sh
+      - name: Git Checkout
+        uses: actions/checkout@v4
+      - name: Add SBT proxy repositories
+        run: cp -vf .github/workflows/repositories /root/.sbt/ ; true
+      - name: Compile Bootstrapped Dotty
+        run: ./project/scripts/sbt "; clean; scala3-bootstrapped/compile"
+
   test_non_bootstrapped:
+    needs: build-non-bootstrapped
     runs-on: [self-hosted, Linux]
     container:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -106,11 +159,13 @@ jobs:
           ./project/scripts/cmdTests
 
   test:
+    needs: build-bootstrapped
     runs-on: [self-hosted, Linux]
     container:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -166,11 +221,13 @@ jobs:
         run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty; scala2-library-cc/compile; scala2-library-cc-tasty/compile; scala3-bootstrapped/testCompilation i3"
 
   test_scala2_library_tasty:
+    needs: build-bootstrapped
     runs-on: [self-hosted, Linux]
     container:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -293,11 +350,13 @@ jobs:
 
   mima:
     name: MiMa
+    needs: build-bootstrapped
     runs-on: [self-hosted, Linux]
     container:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -344,11 +403,13 @@ jobs:
           ./project/scripts/scala2-library-tasty-mima.sh
 
   community_build_a:
+    needs: build-bootstrapped
     runs-on: [self-hosted, Linux]
     container:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -401,11 +462,13 @@ jobs:
         run: cat community-build/dotty-community-build-deps || true
 
   community_build_b:
+    needs: build-bootstrapped
     runs-on: [self-hosted, Linux]
     container:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -458,11 +521,13 @@ jobs:
         run: cat community-build/dotty-community-build-deps || true
 
   community_build_c:
+    needs: build-bootstrapped
     runs-on: [self-hosted, Linux]
     container:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -515,11 +580,13 @@ jobs:
         run: cat community-build/dotty-community-build-deps || true
 
   test_sbt:
+    needs: build-bootstrapped
     runs-on: [self-hosted, Linux]
     container:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -565,6 +632,7 @@ jobs:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -625,6 +693,7 @@ jobs:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -696,6 +765,7 @@ jobs:
       image: lampepfl/dotty:2024-10-18
       options: --cpu-shares 4096
       volumes:
+        # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
         - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
         - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
         - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -855,7 +925,7 @@ jobs:
           scala3-${{ env.RELEASE_TAG }}*.zip \
           scala3-${{ env.RELEASE_TAG }}*.tar.gz \
           scala3-${{ env.RELEASE_TAG }}*.sha256 \
-          scala3-${{ env.RELEASE_TAG }}.msi 
+          scala3-${{ env.RELEASE_TAG }}.msi
 
       - name: Publish Release
         run: ./project/scripts/sbtPublish ";project scala3-bootstrapped ;publishSigned ;sonatypeBundleUpload"

.sbtopts

@@ -0,0 +1,2 @@
+# read build from develocity cache
+-Ddevelocity.scan.metaBuild

project/Build.scala

@@ -339,24 +339,28 @@ object Build {
           buildScan
             .withPublishing(Publishing.onlyIf(_.authenticated))
             .withBackgroundUpload(!isInsideCI)
-            .tag(if (isInsideCI) "CI" else "Local")
+            .withTag(if (isInsideCI) "CI" else "Local")
             .withLinks(buildScan.links ++ GithubEnv.develocityLinks)
             .withValues(buildScan.values ++ GithubEnv.develocityValues)
             .withObfuscation(buildScan.obfuscation.withIpAddresses(_.map(_ => "0.0.0.0")))
         )
         .withBuildCache(
           buildCache
-            .withLocal(buildCache.local.withEnabled(false))
-            .withRemote(buildCache.remote.withEnabled(false))
+            .withLocal(buildCache.local.withEnabled(true).withStoreEnabled(true))
+            .withRemote(buildCache.remote.withEnabled(true).withStoreEnabled(isInsideCI))
+            .withRequireClean(!isInsideCI) // always cache in CI
         )
-        .withTestRetryConfiguration(
-          config.testRetryConfiguration
+        .withTestRetry(
+          config.testRetry
             .withFlakyTestPolicy(FlakyTestPolicy.Fail)
             .withMaxRetries(if (isInsideCI) 1 else 0)
             .withMaxFailures(10)
             .withClassesFilter((className, _) => !noRetryTestClasses.contains(className))
         )
-    }
+    },
+    // Deactivate Develocity's test caching because it caches all tests or nothing.
+    // Also at the moment, it does not take compilation files as inputs.
+    Test / develocityBuildCacheClient := None,
   )
 
   // Settings shared globally (scoped in Global). Used in build.sbt
@@ -1015,10 +1019,6 @@ object Build {
           sjsSources
         } (Set(scalaJSIRSourcesJar)).toSeq
       }.taskValue,
-
-      // Develocity's Build Cache does not work with our compilation tests
-      // at the moment: it does not take compilation files as inputs.
-      Test / develocityBuildCacheClient := None,
   )
 
   def insertClasspathInArgs(args: List[String], cp: String): List[String] = {
@@ -1117,7 +1117,7 @@ object Build {
     libraryDependencies += "org.scala-lang" % "scala-library" % stdlibVersion,
     (Compile / scalacOptions) ++= Seq(
       // Needed so that the library sources are visible when `dotty.tools.dotc.core.Definitions#init` is called
-      "-sourcepath", (Compile / sourceDirectories).value.map(_.getAbsolutePath).distinct.mkString(File.pathSeparator),
+      "-sourcepath", (Compile / sourceDirectories).value.map(_.getCanonicalPath).distinct.mkString(File.pathSeparator),
       "-Yexplicit-nulls",
     ),
     (Compile / doc / scalacOptions) ++= ScaladocConfigs.DefaultGenerationSettings.value.settings,

project/build.sbt

@@ -2,3 +2,21 @@
 libraryDependencies += "org.eclipse.jgit" % "org.eclipse.jgit" % "4.11.0.201803080745-r"
 
 libraryDependencies += Dependencies.`jackson-databind`
+
+// Configuration to publish build compilation to local and remote cache
+ThisBuild / develocityConfiguration := {
+  val isInsideCI = insideCI.value
+  val config = develocityConfiguration.value
+  val buildScan = config.buildScan
+  val buildCache = config.buildCache
+  config
+    .withProjectId(ProjectId("scala3-build"))
+    .withServer(config.server.withUrl(Some(url("https://develocity.scala-lang.org"))))
+    .withBuildScan(buildScan.withPublishing(Publishing.onlyIf(_ => false)))
+    .withBuildCache(
+      buildCache
+        .withLocal(buildCache.local.withEnabled(true).withStoreEnabled(true))
+        .withRemote(buildCache.remote.withEnabled(true).withStoreEnabled(isInsideCI))
+        .withRequireClean(!isInsideCI) // always cache inside CI
+    )
+}

project/plugins.sbt

@@ -22,4 +22,7 @@ addSbtPlugin("ch.epfl.scala" % "sbt-tasty-mima" % "1.0.0")
 
 addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.0")
 
-addSbtPlugin("com.gradle" % "sbt-develocity" % "1.1.1")
+resolvers +=
+  "Develocity Artifactory" at "https://repo.grdev.net/artifactory/public/"
+
+addSbtPlugin("com.gradle" % "sbt-develocity" % "1.2-rc-2")

project/project/plugins.sbt

@@ -0,0 +1,3 @@
+resolvers += "Develocity Artifactory" at "https://repo.grdev.net/artifactory/public/"
+
+addSbtPlugin("com.gradle" % "sbt-develocity" % "1.2-rc-2")

project/scripts/sbt

@@ -10,4 +10,6 @@ CMD="${1:?Missing sbt command}"
 sbt -J-XX:ReservedCodeCacheSize=512m \
     -DSBT_PGP_USE_GPG=false \
     -no-colors \
+    -Ddevelocity.internal.cache.verbose=true \
+    -Ddevelocity.internal.cache.defaultLogLevel=info \
     "$CMD"