diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b9fd3b5b864b..df69323a79c9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,10 +26,10 @@ on: - cron: '0 3 * * *' # Every day at 3 AM workflow_dispatch: -# Cancels any in-progress runs within the same group identified by workflow name and GH reference (branch or tag) +# Cancels any in-progress runs within the same group identified by workflow name and GH reference (branch or tag) # For example it would: # - terminate previous PR CI execution after pushing more changes to the same PR branch -# - terminate previous on-push CI run after merging new PR to main +# - terminate previous on-push CI run after merging new PR to main concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -53,12 +53,65 @@ env: # text on stderr and so can break tests which check the output of a program). jobs: + build-non-bootstrapped: + name: Build Non-Bootstrapped Compiler + runs-on: [self-hosted, Linux] + container: + image: lampepfl/dotty:2024-10-18 + options: --cpu-shares 4096 + volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache + - ${{ github.workspace }}/../../cache/sbt:/root/.sbt + - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache + - ${{ github.workspace }}/../../cache/general:/root/.cache + steps: + - name: Set JDK 17 as default + run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH + - name: Checkout cleanup script + uses: actions/checkout@v4 + - name: Cleanup + run: .github/workflows/cleanup.sh + - name: Git Checkout + uses: actions/checkout@v4 + - name: Add SBT proxy repositories + run: cp -vf .github/workflows/repositories /root/.sbt/ ; true + - name: Compile Non-Bootstrapped Dotty + run: ./project/scripts/sbt "; clean; compile" + + build-bootstrapped: + name: Build Bootstrapped Compiler + needs: build-non-bootstrapped + runs-on: [self-hosted, Linux] + container: + image: lampepfl/dotty:2024-10-18 + options: --cpu-shares 4096 + volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache + - ${{ github.workspace }}/../../cache/sbt:/root/.sbt + - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache + - ${{ github.workspace }}/../../cache/general:/root/.cache + steps: + - name: Set JDK 17 as default + run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH + - name: Checkout cleanup script + uses: actions/checkout@v4 + - name: Cleanup + run: .github/workflows/cleanup.sh + - name: Git Checkout + uses: actions/checkout@v4 + - name: Add SBT proxy repositories + run: cp -vf .github/workflows/repositories /root/.sbt/ ; true + - name: Compile Bootstrapped Dotty + run: ./project/scripts/sbt "; clean; scala3-bootstrapped/compile" + test_non_bootstrapped: + needs: build-non-bootstrapped runs-on: [self-hosted, Linux] container: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -106,11 +159,13 @@ jobs: ./project/scripts/cmdTests test: + needs: build-bootstrapped runs-on: [self-hosted, Linux] container: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -166,11 +221,13 @@ jobs: run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty; scala2-library-cc/compile; scala2-library-cc-tasty/compile; scala3-bootstrapped/testCompilation i3" test_scala2_library_tasty: + needs: build-bootstrapped runs-on: [self-hosted, Linux] container: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -293,11 +350,13 @@ jobs: mima: name: MiMa + needs: build-bootstrapped runs-on: [self-hosted, Linux] container: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -344,11 +403,13 @@ jobs: ./project/scripts/scala2-library-tasty-mima.sh community_build_a: + needs: build-bootstrapped runs-on: [self-hosted, Linux] container: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -401,11 +462,13 @@ jobs: run: cat community-build/dotty-community-build-deps || true community_build_b: + needs: build-bootstrapped runs-on: [self-hosted, Linux] container: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -458,11 +521,13 @@ jobs: run: cat community-build/dotty-community-build-deps || true community_build_c: + needs: build-bootstrapped runs-on: [self-hosted, Linux] container: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -515,11 +580,13 @@ jobs: run: cat community-build/dotty-community-build-deps || true test_sbt: + needs: build-bootstrapped runs-on: [self-hosted, Linux] container: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -565,6 +632,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -625,6 +693,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -696,6 +765,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -855,7 +925,7 @@ jobs: scala3-${{ env.RELEASE_TAG }}*.zip \ scala3-${{ env.RELEASE_TAG }}*.tar.gz \ scala3-${{ env.RELEASE_TAG }}*.sha256 \ - scala3-${{ env.RELEASE_TAG }}.msi + scala3-${{ env.RELEASE_TAG }}.msi - name: Publish Release run: ./project/scripts/sbtPublish ";project scala3-bootstrapped ;publishSigned ;sonatypeBundleUpload" diff --git a/.sbtopts b/.sbtopts new file mode 100644 index 000000000000..8ffd2c3d0632 --- /dev/null +++ b/.sbtopts @@ -0,0 +1,2 @@ +# read build from develocity cache +-Ddevelocity.scan.metaBuild diff --git a/project/Build.scala b/project/Build.scala index 463abab3f6fd..924488effaba 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -339,24 +339,28 @@ object Build { buildScan .withPublishing(Publishing.onlyIf(_.authenticated)) .withBackgroundUpload(!isInsideCI) - .tag(if (isInsideCI) "CI" else "Local") + .withTag(if (isInsideCI) "CI" else "Local") .withLinks(buildScan.links ++ GithubEnv.develocityLinks) .withValues(buildScan.values ++ GithubEnv.develocityValues) .withObfuscation(buildScan.obfuscation.withIpAddresses(_.map(_ => "0.0.0.0"))) ) .withBuildCache( buildCache - .withLocal(buildCache.local.withEnabled(false)) - .withRemote(buildCache.remote.withEnabled(false)) + .withLocal(buildCache.local.withEnabled(true).withStoreEnabled(true)) + .withRemote(buildCache.remote.withEnabled(true).withStoreEnabled(isInsideCI)) + .withRequireClean(!isInsideCI) // always cache in CI ) - .withTestRetryConfiguration( - config.testRetryConfiguration + .withTestRetry( + config.testRetry .withFlakyTestPolicy(FlakyTestPolicy.Fail) .withMaxRetries(if (isInsideCI) 1 else 0) .withMaxFailures(10) .withClassesFilter((className, _) => !noRetryTestClasses.contains(className)) ) - } + }, + // Deactivate Develocity's test caching because it caches all tests or nothing. + // Also at the moment, it does not take compilation files as inputs. + Test / develocityBuildCacheClient := None, ) // Settings shared globally (scoped in Global). Used in build.sbt @@ -1015,10 +1019,6 @@ object Build { sjsSources } (Set(scalaJSIRSourcesJar)).toSeq }.taskValue, - - // Develocity's Build Cache does not work with our compilation tests - // at the moment: it does not take compilation files as inputs. - Test / develocityBuildCacheClient := None, ) def insertClasspathInArgs(args: List[String], cp: String): List[String] = { @@ -1117,7 +1117,7 @@ object Build { libraryDependencies += "org.scala-lang" % "scala-library" % stdlibVersion, (Compile / scalacOptions) ++= Seq( // Needed so that the library sources are visible when `dotty.tools.dotc.core.Definitions#init` is called - "-sourcepath", (Compile / sourceDirectories).value.map(_.getAbsolutePath).distinct.mkString(File.pathSeparator), + "-sourcepath", (Compile / sourceDirectories).value.map(_.getCanonicalPath).distinct.mkString(File.pathSeparator), "-Yexplicit-nulls", ), (Compile / doc / scalacOptions) ++= ScaladocConfigs.DefaultGenerationSettings.value.settings, diff --git a/project/build.sbt b/project/build.sbt index 188dfa5c6702..5b984b79367c 100644 --- a/project/build.sbt +++ b/project/build.sbt @@ -2,3 +2,21 @@ libraryDependencies += "org.eclipse.jgit" % "org.eclipse.jgit" % "4.11.0.201803080745-r" libraryDependencies += Dependencies.`jackson-databind` + +// Configuration to publish build compilation to local and remote cache +ThisBuild / develocityConfiguration := { + val isInsideCI = insideCI.value + val config = develocityConfiguration.value + val buildScan = config.buildScan + val buildCache = config.buildCache + config + .withProjectId(ProjectId("scala3-build")) + .withServer(config.server.withUrl(Some(url("https://develocity.scala-lang.org")))) + .withBuildScan(buildScan.withPublishing(Publishing.onlyIf(_ => false))) + .withBuildCache( + buildCache + .withLocal(buildCache.local.withEnabled(true).withStoreEnabled(true)) + .withRemote(buildCache.remote.withEnabled(true).withStoreEnabled(isInsideCI)) + .withRequireClean(!isInsideCI) // always cache inside CI + ) +} diff --git a/project/plugins.sbt b/project/plugins.sbt index b3e4af1f72e7..02fe1c39563d 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -22,4 +22,7 @@ addSbtPlugin("ch.epfl.scala" % "sbt-tasty-mima" % "1.0.0") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.0") -addSbtPlugin("com.gradle" % "sbt-develocity" % "1.1.1") +resolvers += + "Develocity Artifactory" at "https://repo.grdev.net/artifactory/public/" + +addSbtPlugin("com.gradle" % "sbt-develocity" % "1.2-rc-2") diff --git a/project/project/plugins.sbt b/project/project/plugins.sbt new file mode 100644 index 000000000000..becbd79d12e5 --- /dev/null +++ b/project/project/plugins.sbt @@ -0,0 +1,3 @@ +resolvers += "Develocity Artifactory" at "https://repo.grdev.net/artifactory/public/" + +addSbtPlugin("com.gradle" % "sbt-develocity" % "1.2-rc-2") diff --git a/project/scripts/sbt b/project/scripts/sbt index 8c6f5aa1c280..9eeb7567dff0 100755 --- a/project/scripts/sbt +++ b/project/scripts/sbt @@ -10,4 +10,6 @@ CMD="${1:?Missing sbt command}" sbt -J-XX:ReservedCodeCacheSize=512m \ -DSBT_PGP_USE_GPG=false \ -no-colors \ + -Ddevelocity.internal.cache.verbose=true \ + -Ddevelocity.internal.cache.defaultLogLevel=info \ "$CMD"