Bundle scala cli in scala command (#20351)

fixes #20098

Proposed changes to zip/targz archive:
- in the `/bin` directory store an extra launcher for Scala CLI (either
JAR, or native per platform).
- `/bin/scala[.bat]` is modified to invoke Scala CLI stored in `/bin`
- new `/maven2` directory, which stores all the Jars and POM files
necessary (in maven repo style) for scala-cli to invoke scala compiler
offline (using the `-r` launcher option).
- CHOICE: either replace jar files in `/lib` by aliases to the
corresponding jar in `/maven2`, OR delete `/lib` and update references
from scripts. (Looks like symlinks are not portable, so probably we
should encode the classpath in a file, or adjust slightly how we build
the toolchain)
- add platform specific suffixes to artefacts:
- e.g. `scala-3.5.0-x86_64-pc-linux.tar.gz` (for the artefact that
bundles the x64 linux launcher)

---------

Co-authored-by: Hamza REMMAL <[email protected]>

Author: bishabosha

Diff for: .github/workflows/ci.yaml

@@ -141,7 +141,8 @@ jobs:
 
       - name: Cmd Tests
         run: |
-          ./project/scripts/sbt ";dist/pack; scala3-bootstrapped/compile; scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/*; scala3-compiler-bootstrapped/scala3CompilerCoursierTest:test"
+          ./project/scripts/buildScalaBinary
+          ./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/* ;scala3-compiler-bootstrapped/scala3CompilerCoursierTest:test"
           ./project/scripts/cmdTests
           ./project/scripts/bootstrappedOnlyCmdTests
 
@@ -230,7 +231,7 @@ jobs:
         shell: cmd
 
       - name: build binary
-        run: sbt "dist/pack" & bash -version
+        run: sbt "dist-win-x86_64/pack" & bash -version
         shell: cmd
 
       - name: cygwin tests
@@ -269,8 +270,12 @@ jobs:
       - name: Git Checkout
         uses: actions/checkout@v4
 
+      - name: build binary
+        run: sbt "dist-win-x86_64/pack"
+        shell: cmd
+
       - name: Test
-        run: sbt ";dist/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test"
+        run: sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test"
         shell: cmd
 
       - name: Scala.js Test
@@ -596,7 +601,8 @@ jobs:
 
       - name: Test
         run: |
-          ./project/scripts/sbt ";dist/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/*"
+          ./project/scripts/buildScalaBinary
+          ./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/*"
           ./project/scripts/cmdTests
           ./project/scripts/bootstrappedOnlyCmdTests
 

Diff for: .github/workflows/launchers.yml

@@ -0,0 +1,96 @@
+name: Test CLI Launchers on all the platforms
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  linux-x86_64:
+    name: Deploy and Test on Linux x64 architecture
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'sbt'
+      - name: Build and test launcher command
+        run: ./project/scripts/native-integration/bashTests
+        env:
+          LAUNCHER_EXPECTED_PROJECT: "dist-linux-x86_64"
+
+  linux-aarch64:
+    name: Deploy and Test on Linux ARM64 architecture
+    runs-on: macos-latest
+    if: ${{ false }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'sbt'
+      # https://github.com/actions/runner-images/issues/9369
+      - name: Install sbt
+        run: brew install sbt
+      - name: Build and test launcher command
+        run: ./project/scripts/native-integration/bashTests
+        env:
+          LAUNCHER_EXPECTED_PROJECT: "dist-linux-aarch64"
+
+  mac-x86_64:
+    name: Deploy and Test on Mac x64 architecture
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'sbt'
+      # https://github.com/actions/runner-images/issues/9369
+      - name: Install sbt
+        run: brew install sbt
+      - name: Build and test launcher command
+        run: ./project/scripts/native-integration/bashTests
+        env:
+          LAUNCHER_EXPECTED_PROJECT: "dist-mac-x86_64"
+
+  mac-aarch64:
+    name: Deploy and Test on Mac ARM64 architecture
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'sbt'
+      # https://github.com/actions/runner-images/issues/9369
+      - name: Install sbt
+        run: brew install sbt
+      - name: Build and test launcher command
+        run: ./project/scripts/native-integration/bashTests
+        env:
+          LAUNCHER_EXPECTED_PROJECT: "dist-mac-aarch64"
+
+  win-x86_64:
+    name: Deploy and Test on Windows x64 architecture
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'sbt'
+      - name: Build the launcher command
+        run: sbt "dist-win-x86_64/pack"
+      - name: Run the launcher command tests
+        run: './project/scripts/native-integration/winTests.bat'
+        shell: cmd

Diff for: bin/common

@@ -9,15 +9,18 @@ target="$1"
 
 shift # Mutates $@ by deleting the first element ($1)
 
+# set the $DIST_PROJECT and $DIST_DIR variables
+source "$ROOT/bin/common-platform"
+
 # Marker file used to obtain the date of latest call to sbt-back
-version="$ROOT/dist/target/pack/VERSION"
+version="$ROOT/$DIST_DIR/target/pack/VERSION"
 
 # Create the target if absent or if file changed in ROOT/compiler
 new_files="$(find "$ROOT/compiler" \( -iname "*.scala" -o -iname "*.java" \) -newer "$version" 2> /dev/null)"
 
 if [ ! -f "$version" ] || [ ! -z "$new_files" ]; then
   echo "Building Dotty..."
-  (cd $ROOT && sbt "dist/pack")
+  (cd $ROOT && sbt "$DIST_PROJECT/pack")
 fi
 
-"$target" "$@"
+"$ROOT/$DIST_DIR/target/pack/bin/$target" "$@"

Diff for: bin/common-platform

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+unset cygwin mingw msys darwin
+
+# COLUMNS is used together with command line option '-pageWidth'.
+if command -v tput >/dev/null 2>&1; then
+  export COLUMNS="$(tput -Tdumb cols)"
+fi
+
+case "`uname`" in
+  CYGWIN*) cygwin=true
+          ;;
+  MINGW*) mingw=true
+          ;;
+  MSYS*) msys=true
+          ;;
+  Darwin*) darwin=true
+           ;;
+esac
+
+unset DIST_PROJECT DIST_DIR
+
+if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then
+  DIST_PROJECT="dist-win-x86_64"
+  DIST_DIR="dist/win-x86_64"
+else
+  # OS and arch logic taken from https://github.com/VirtusLab/scala-cli/blob/main/scala-cli.sh
+  unset arch ARCH_NORM
+  arch=$(uname -m)
+  if [[ "$arch" == "aarch64" ]] || [[ "$arch" == "x86_64" ]]; then
+    ARCH_NORM="$arch"
+  elif [[ "$arch" == "amd64" ]]; then
+    ARCH_NORM="x86_64"
+  elif [[ "$arch" == "arm64" ]]; then
+    ARCH_NORM="aarch64"
+  else
+    ARCH_NORM="unknown"
+  fi
+
+  if [ "$(expr substr $(uname -s) 1 5 2>/dev/null)" == "Linux" ]; then
+    if [[ "$ARCH_NORM" == "unknown" ]]; then
+      echo >&2 "unknown Linux CPU architecture, defaulting to JVM launcher"
+      DIST_PROJECT="dist"
+      DIST_DIR="dist"
+    else
+      DIST_PROJECT="dist-linux-$ARCH_NORM"
+      DIST_DIR="dist/linux-$ARCH_NORM"
+    fi
+  elif [ "$(uname)" == "Darwin" ]; then
+    if [[ "$ARCH_NORM" == "unknown" ]]; then
+      echo >&2 "unknown Darwin CPU architecture, defaulting to JVM launcher"
+      DIST_PROJECT="dist"
+      DIST_DIR="dist"
+    else
+      DIST_PROJECT="dist-mac-$ARCH_NORM"
+      DIST_DIR="dist/mac-$ARCH_NORM"
+    fi
+  else
+    echo >&2 "unknown OS, defaulting to JVM launcher"
+    DIST_PROJECT="dist"
+    DIST_DIR="dist"
+  fi
+fi

Diff for: bin/scala

@@ -2,4 +2,37 @@
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.."
 
-"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scala" "$@"
+scala_args() {
+
+  declare -a CLI_ARGS
+  declare -a SCRIPT_ARGS
+  declare DISABLE_BLOOP=1
+
+  while (( "$#" )); do
+    case "$1" in
+      "--")
+        shift
+        SCRIPT_ARGS+=("--")
+        SCRIPT_ARGS+=("$@")
+        break
+        ;;
+      "clean" | "version" | "--version" | "-version" | "help" | "--help" | "-help")
+        CLI_ARGS+=("$1")
+        DISABLE_BLOOP=0 # clean command should not add --offline --server=false
+        shift
+        ;;
+      *)
+        CLI_ARGS+=("$1")
+        shift
+        ;;
+    esac
+  done
+
+  if [ $DISABLE_BLOOP -eq 1 ]; then
+    CLI_ARGS+=("--offline" "--server=false")
+  fi
+
+  echo "--power ${CLI_ARGS[@]} ${SCRIPT_ARGS[@]}"
+}
+
+"$ROOT/bin/common" "scala" $(scala_args "$@")

Diff for: bin/scalac

@@ -2,4 +2,4 @@
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.."
 
-"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scalac" "$@"
+"$ROOT/bin/common" "scalac" "$@"

Diff for: bin/scaladoc

@@ -2,4 +2,4 @@
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.."
 
-"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scaladoc" "$@"
+"$ROOT/bin/common" "scaladoc" "$@"

Diff for: bin/test/TestScripts.scala

@@ -57,7 +57,7 @@ class TestScripts {
       s"bin/scalac script did not run properly. Output:$lineSep$dotcOutput"
     )
 
-    val (retDotr, dotrOutput) = executeScript("./bin/scala HelloWorld")
+    val (retDotr, dotrOutput) = executeScript("./bin/scala -M HelloWorld")
     assert(
       retDotr == 0 && dotrOutput == "hello world\n",
       s"Running hello world exited with status: $retDotr and output: $dotrOutput"

Diff for: build.sbt

@@ -28,6 +28,11 @@ val `scaladoc-js-main` = Build.`scaladoc-js-main`
 val `scaladoc-js-contributors` = Build.`scaladoc-js-contributors`
 val `scala3-bench-run` = Build.`scala3-bench-run`
 val dist = Build.dist
+val `dist-mac-x86_64` = Build.`dist-mac-x86_64`
+val `dist-mac-aarch64` = Build.`dist-mac-aarch64`
+val `dist-win-x86_64` = Build.`dist-win-x86_64`
+val `dist-linux-x86_64` = Build.`dist-linux-x86_64`
+val `dist-linux-aarch64` = Build.`dist-linux-aarch64`
 val `community-build` = Build.`community-build`
 val `sbt-community-build` = Build.`sbt-community-build`
 val `scala3-presentation-compiler` = Build.`scala3-presentation-compiler`

Diff for: compiler/src/dotty/tools/MainGenericRunner.scala

@@ -266,6 +266,22 @@ object MainGenericRunner {
           run(settings.withExecuteMode(ExecuteMode.Run))
         else
           run(settings.withExecuteMode(ExecuteMode.Repl))
+    end run
+
+    val ranByCoursierBootstrap =
+      sys.props.isDefinedAt("coursier.mainJar")
+      || sys.props.get("bootstrap.mainClass").contains("dotty.tools.MainGenericRunner")
+
+    val silenced = sys.props.get("scala.use_legacy_launcher") == Some("true")
+
+    if !silenced then
+      Console.err.println(s"[warning] MainGenericRunner class is deprecated since Scala 3.5.0, and Scala CLI features will not work.")
+      Console.err.println(s"[warning] Please be sure to update to the Scala CLI launcher to use the new features.")
+      if ranByCoursierBootstrap then
+        Console.err.println(s"[warning] It appears that your Coursier-based Scala installation is misconfigured.")
+        Console.err.println(s"[warning] To update to the new Scala CLI runner, please update (coursier, cs) commands first before re-installing scala.")
+      Console.err.println(s"[warning] Check the Scala 3.5.0 release notes to troubleshoot your installation.")
+
 
     run(settings) match
       case Some(ex: (StringDriverException | ScriptingException)) => errorFn(ex.getMessage)

Diff for: compiler/test-coursier/dotty/tools/coursier/CoursierScalaTests.scala

@@ -166,7 +166,7 @@ object CoursierScalaTests:
       case Nil => args
       case _ => "--" +: args
     val newJOpts = jOpts.map(s => s"--java-opt ${s.stripPrefix("-J")}").mkString(" ")
-    execCmd("./cs", (s"""launch "org.scala-lang:scala3-compiler_3:${sys.env("DOTTY_BOOTSTRAPPED_VERSION")}" $newJOpts --main-class "$entry" --property "scala.usejavacp=true"""" +: newOptions)*)._2
+    execCmd("./cs", (s"""launch "org.scala-lang:scala3-compiler_3:${sys.env("DOTTY_BOOTSTRAPPED_VERSION")}" $newJOpts --main-class "$entry" --property "scala.usejavacp=true" --property "scala.use_legacy_launcher=true"""" +: newOptions)*)._2
 
   /** Get coursier script */
   @BeforeClass def setup(): Unit =

Diff for: compiler/test-resources/scripting/argfileClasspath.sc

@@ -1,8 +1,8 @@
-#!bin/scala -classpath 'dist/target/pack/lib/*'
-
+#!/usr/bin/env bin/scala
+// This file is a Scala CLI script.
 import java.nio.file.Paths
 
-def main(args: Array[String]): Unit =
+// def main(args: Array[String]): Unit = // MIGRATION: Scala CLI expects `*.sc` files to be straight-line code
   val cwd = Paths.get(".").toAbsolutePath.normalize.toString.norm
   printf("cwd: %s\n", cwd)
   printf("classpath: %s\n", sys.props("java.class.path").norm)

Diff for: compiler/test-resources/scripting/classpathReport.sc renamed to compiler/test-resources/scripting/classpathReport_scalacli.sc

@@ -1,2 +1,4 @@
+// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class
+
 def main(args: Array[String]): Unit =
   println("Hello " + util.Properties.propOrNull("key"))

Diff for: compiler/test-resources/scripting/cpArgumentsFile.txt

@@ -0,0 +1,3 @@
+// This file is a Scala CLI script.
+
+println("Hello " + util.Properties.propOrNull("key"))

Diff for: compiler/test-resources/scripting/envtest.sc

@@ -1,4 +1,4 @@
-#!/usr/bin/env scala
+#!/usr/bin/env fake-program-to-test-hashbang-removal
 # comment
 STUFF=nada
 !#

Diff for: compiler/test-resources/scripting/envtest_scalacli.sc

@@ -1,8 +1,8 @@
-#!/usr/bin/env scala
+#!/usr/bin/env fake-program-to-test-hashbang-removal
 # comment
 STUFF=nada
 !#
-
+// everything above this point should be ignored by the compiler
 def main(args: Array[String]): Unit =
   System.err.printf("mainClassFromStack: %s\n",mainFromStack)
   assert(mainFromStack.contains("hashBang"),s"fromStack[$mainFromStack]")

Diff for: compiler/test-resources/scripting/hashBang.sc

@@ -1,4 +1,4 @@
-#!/usr/bin/env scala
+// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class
 
   def main(args: Array[String]): Unit =
     val name = Option(sys.props("script.name")) match {

Diff for: compiler/test-resources/scripting/hashBang.scala

@@ -1,4 +1,4 @@
-#!dist/target/pack/bin/scala
+// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class
 
   def main(args: Array[String]): Unit =
     args.zipWithIndex.foreach { case (arg,i) => printf("arg %d: [%s]\n",i,arg) }