Make better-fors a preview feature in 3.7 (#22776)

Makes the SIP-62 better fors a preview feature in Scala 3.7. Shall be
stabilised no earlier then Scala 3.8

Author: WojciechMazur

Diff for: compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -1953,9 +1953,9 @@ object desugar {
     /** Create tree for for-comprehension `<for (enums) do body>` or
      *   `<for (enums) yield body>` where mapName and flatMapName are chosen
      *  corresponding to whether this is a for-do or a for-yield.
-     *  If sourceVersion >= 3.7 are enabled, the creation performs the following rewrite rules:
+     *  If betterFors are enabled, the creation performs the following rewrite rules:
      *
-     *  1. if sourceVersion >= 3.7:
+     *  1. if betterFors is enabled:
      *
      *    for () do E  ==>  E
      *      or
@@ -1986,13 +1986,13 @@ object desugar {
      *      ==>
      *    for (P <- G.withFilter (P => E); ...) ...
      *
-     *  6. For any N, if sourceVersion >= 3.7:
+     *  6. For any N, if betterFors is enabled:
      *
      *    for (P <- G; P_1 = E_1; ... P_N = E_N; P1 <- G1; ...) ...
      *      ==>
      *    G.flatMap (P => for (P_1 = E_1; ... P_N = E_N; ...))
      *
-     *  7. For any N, if sourceVersion >= 3.7:
+     *  7. For any N, if betterFors is enabled:
      *
      *    for (P <- G; P_1 = E_1; ... P_N = E_N) ...
      *      ==>
@@ -2013,7 +2013,7 @@ object desugar {
      *    If any of the P_i are variable patterns, the corresponding `x_i @ P_i` is not generated
      *    and the variable constituting P_i is used instead of x_i
      *
-     *  9. For any N, if sourceVersion >= 3.7:
+     *  9. For any N, if betterFors is enabled:
      *
      *    for (P_1 = E_1; ... P_N = E_N; ...)
      *      ==>
@@ -2157,15 +2157,15 @@ object desugar {
           case _ => false
 
       def markTrailingMap(aply: Apply, gen: GenFrom, selectName: TermName): Unit =
-        if sourceVersion.isAtLeast(`3.7`)
+        if sourceVersion.enablesBetterFors
           && selectName == mapName
           && gen.checkMode != GenCheckMode.Filtered // results of withFilter have the wrong type
           && (deepEquals(gen.pat, body) || deepEquals(body, Tuple(Nil)))
         then
           aply.putAttachment(TrailingForMap, ())
 
       enums match {
-        case Nil if sourceVersion.isAtLeast(`3.7`) => body
+        case Nil if sourceVersion.enablesBetterFors => body
         case (gen: GenFrom) :: Nil =>
           val aply = Apply(rhsSelect(gen, mapName), makeLambda(gen, body))
           markTrailingMap(aply, gen, mapName)
@@ -2174,7 +2174,7 @@ object desugar {
           val cont = makeFor(mapName, flatMapName, rest, body)
           Apply(rhsSelect(gen, flatMapName), makeLambda(gen, cont))
         case (gen: GenFrom) :: rest
-        if sourceVersion.isAtLeast(`3.7`)
+        if sourceVersion.enablesBetterFors
           && rest.dropWhile(_.isInstanceOf[GenAlias]).headOption.forall(e => e.isInstanceOf[GenFrom]) // possible aliases followed by a generator or end of for
           && !rest.takeWhile(_.isInstanceOf[GenAlias]).exists(a => isNestedGivenPattern(a.asInstanceOf[GenAlias].pat)) =>
           val cont = makeFor(mapName, flatMapName, rest, body)
@@ -2202,9 +2202,9 @@ object desugar {
           makeFor(mapName, flatMapName, vfrom1 :: rest1, body)
         case (gen: GenFrom) :: test :: rest =>
           val filtered = Apply(rhsSelect(gen, nme.withFilter), makeLambda(gen, test))
-          val genFrom = GenFrom(gen.pat, filtered, if sourceVersion.isAtLeast(`3.7`) then GenCheckMode.Filtered else GenCheckMode.Ignore)
+          val genFrom = GenFrom(gen.pat, filtered, if sourceVersion.enablesBetterFors then GenCheckMode.Filtered else GenCheckMode.Ignore)
           makeFor(mapName, flatMapName, genFrom :: rest, body)
-        case GenAlias(_, _) :: _ if sourceVersion.isAtLeast(`3.7`) =>
+        case GenAlias(_, _) :: _ if sourceVersion.enablesBetterFors =>
           val (valeqs, rest) = enums.span(_.isInstanceOf[GenAlias])
           val pats = valeqs.map { case GenAlias(pat, _) => pat }
           val rhss = valeqs.map { case GenAlias(_, rhs) => rhs }

Diff for: compiler/src/dotty/tools/dotc/config/SourceVersion.scala

@@ -3,6 +3,8 @@ package dotc
 package config
 
 import core.Decorators.*
+import core.Contexts.*
+import Feature.isPreviewEnabled
 import util.Property
 
 enum SourceVersion:
@@ -35,6 +37,7 @@ enum SourceVersion:
   def enablesClauseInterleaving = isAtLeast(`3.6`)
   def enablesNewGivens = isAtLeast(`3.6`)
   def enablesNamedTuples = isAtLeast(`3.7`)
+  def enablesBetterFors(using Context) = isAtLeast(`3.7`) && isPreviewEnabled
 
 object SourceVersion extends Property.Key[SourceVersion]:
   def defaultSourceVersion = `3.7`

Diff for: compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -2956,7 +2956,7 @@ object Parsers {
     /** Enumerators ::= Generator {semi Enumerator | Guard}
      */
     def enumerators(): List[Tree] =
-      if sourceVersion.isAtLeast(`3.7`) then
+      if sourceVersion.enablesBetterFors then
         aliasesUntilGenerator() ++ enumeratorsRest()
       else
         generator() :: enumeratorsRest()

Diff for: compiler/test/dotty/tools/debug/DebugTests.scala

@@ -18,7 +18,8 @@ class DebugTests:
     implicit val testGroup: TestGroup = TestGroup("debug")
     CompilationTest.aggregateTests(
       compileFile("tests/debug-custom-args/eval-explicit-nulls.scala", TestConfiguration.explicitNullsOptions),
-      compileFilesInDir("tests/debug", TestConfiguration.defaultOptions)
+      compileFilesInDir("tests/debug", TestConfiguration.defaultOptions),
+      compileFilesInDir("tests/debug-preview", TestConfiguration.defaultOptions.and("-preview"))
     ).checkDebug()
 
 object DebugTests extends ParallelTesting:

Diff for: docs/_docs/reference/changed-features/better-fors.md renamed to docs/_docs/reference/preview/better-fors.md

@@ -1,10 +1,10 @@
 ---
 layout: doc-page
 title: "Better fors"
-nightlyOf: https://docs.scala-lang.org/scala3/reference/changed-features/better-fors.html
+nightlyOf: https://docs.scala-lang.org/scala3/reference/preview/better-fors.html
 ---
 
-Starting in Scala `3.7`, the usability of `for`-comprehensions is improved.
+Starting in Scala `3.7` under `-preview` mode, the usability of `for`-comprehensions is improved.
 
 The biggest user facing change is the new ability to start `for`-comprehensions with aliases. This means that the following previously invalid code is now valid:
 

Diff for: docs/_docs/reference/preview/overview.md

@@ -0,0 +1,24 @@
+---
+layout: doc-page
+title: "Preview"
+nightlyOf: https://docs.scala-lang.org/scala3/reference/preview/overview.html
+---
+
+## Preview language features
+
+New Scala language features or standard library APIs are initially introduced as experimental, but once they become fully implemented and accepted by the [SIP](https://docs.scala-lang.org/sips/) these can become a preview features.
+
+Preview language features and APIs are guaranteed to be standardized in some next Scala minor release, but allow the compiler team to introduce small, possibly binary incompatible, changes based on the community feedback.
+These can be used by early adopters who can accept the possibility of binary compatibility breakage. For instance, preview features could be used in some internal tool or application. On the other hand, preview features are discouraged in publicly available libraries.
+
+More information about preview featues can be found in [preview defintions guide](../other-new-features/preview-defs.md)
+
+### `-preview` compiler flag
+
+This flag enables the use of all preview language feature in the project.
+
+
+## List of available preview features
+
+* [`better-fors`](./better-fors.md): Enables new for-comprehension behaviour under SIP-62 under `-source:3.7` or later
+

Diff for: docs/sidebar.yml

@@ -116,7 +116,6 @@ subsection:
           - page: reference/changed-features/lazy-vals-init.md
           - page: reference/changed-features/main-functions.md
           - page: reference/changed-features/interpolation-escapes.md
-          - page: reference/changed-features/better-fors.md
       - title: Dropped Features
         index: reference/dropped-features/dropped-features.md
         subsection:
@@ -140,6 +139,11 @@ subsection:
           - page: reference/dropped-features/nonlocal-returns.md
           - page: reference/dropped-features/this-qualifier.md
           - page: reference/dropped-features/wildcard-init.md
+      - title: Preview Features
+        directory: preview
+        index: reference/preview/overview.md
+        subsection:
+          - page: reference/preview/better-fors.md
       - title: Experimental Features
         directory: experimental
         index: reference/experimental/overview.md

Diff for: library/src/scala/runtime/stdLibPatches/language.scala

@@ -140,7 +140,7 @@ object language:
      * @see [[https://github.com/scala/improvement-proposals/pull/79]]
      */
     @compileTimeOnly("`betterFors` can only be used at compile time in import statements")
-    @deprecated("The `experimental.betterFors` language import is no longer needed since the feature is now standard", since = "3.7")
+    @deprecated("The `experimental.betterFors` language import no longer has any effect, the feature is being stablised and can be enabled using `-preview` flag", since = "3.7")
     object betterFors
 
     /** Experimental support for package object values

Diff for: presentation-compiler/test/dotty/tools/pc/tests/tokens/SemanticTokensSuite.scala

@@ -1,10 +1,14 @@
 package dotty.tools.pc.tests.tokens
 
 import dotty.tools.pc.base.BaseSemanticTokensSuite
+import java.nio.file.Path
 
 import org.junit.Test
 
 class SemanticTokensSuite extends BaseSemanticTokensSuite:
+  // -preview required for `for-comprehension` test
+  override protected def scalacOptions(classpath: Seq[Path]): Seq[String] =
+    super.scalacOptions(classpath) ++ Seq("-preview")
 
   @Test def `class, object, var, val(readonly), method, type, parameter, String(single-line)` =
     check(

Diff for: tests/debug-preview/eval-in-for-comprehension.check

@@ -0,0 +1,27 @@
+break Test$ 5 // in main
+eval list(0)
+result 1
+// TODO can we remove debug line in adapted methods?
+break Test$ 5 // in main$$anonfun$adapted$1
+break Test$ 6 // in main$$anonfun$1
+eval list(0)
+result 1
+eval x
+result 1
+break Test$ 7 // in main$$anonfun$1$$anonfun$1
+eval x + y
+result 2
+
+break Test$ 11 // in main$$anonfun$2
+eval x
+result 1
+
+break Test$ 13 // in main
+eval list(0)
+result 1
+break Test$ 13 // in main$$anonfun$4
+
+break Test$ 14 // in main
+eval list(0)
+result 1
+break Test$ 14 // in main$$anonfun$5

Diff for: tests/debug-preview/eval-in-for-comprehension.scala

@@ -0,0 +1,14 @@
+object Test:
+  def main(args: Array[String]): Unit =
+    val list = List(1)
+    for
+      x <- list
+      y <- list
+      z = x + y
+    yield x
+    for
+      x <- list
+      if x == 1
+    yield x
+    for x <- list yield x
+    for x <- list do println(x)

Diff for: tests/debug/eval-in-for-comprehension.check

@@ -8,9 +8,16 @@ eval list(0)
 result 1
 eval x
 result 1
+break Test$ 6 // in main$$anonfun$1$$anonfun$adapted$1
 break Test$ 7 // in main$$anonfun$1$$anonfun$1
 eval x + y
 result 2
+// TODO this line position does not make any sense
+break Test$ 6 // in main$$anonfun$1$$anonfun$1
+break Test$ 7 // in main$$anonfun$1$$anonfun$1
+break Test$ 6 // in main$$anonfun$1$$anonfun$2
+break Test$ 6 // in main$$anonfun$1$$anonfun$2
+break Test$ 7 // in main$$anonfun$1$$anonfun$2
 
 break Test$ 11 // in main$$anonfun$2
 eval x

Diff for: tests/pos/better-fors-given.scala

@@ -1,3 +1,5 @@
+//> using options -preview
+
 @main def Test: Unit =
   for
     x <- Option(23 -> "abc")

Diff for: tests/pos/better-fors-i21804.scala

@@ -1,4 +1,5 @@
-import scala.language.experimental.betterFors
+//> using options -preview
+// import scala.language.experimental.betterFors
 
 case class Container[A](val value: A) {
   def map[B](f: A => B): Container[B] = Container(f(value))

Diff for: tests/run/better-fors-map-elim.scala

@@ -1,4 +1,5 @@
-import scala.language.experimental.betterFors
+//> using options -preview
+// import scala.language.experimental.betterFors
 
 class myOptionModule(doOnMap: => Unit) {
   sealed trait MyOption[+A] {

Diff for: tests/run/better-fors.scala

@@ -1,3 +1,6 @@
+//> using options -preview
+// import scala.language.experimental.betterFors
+
 def for1 =
   for {
     a = 1

Diff for: tests/run/fors.scala

@@ -1,3 +1,4 @@
+//> using options -preview
 //############################################################################
 // for-comprehensions (old and new syntax)
 //############################################################################

Diff for: tests/semanticdb/expect/ForComprehension.expect.scala

@@ -3,43 +3,43 @@ package example
 class ForComprehension/*<-example::ForComprehension#*/ {
   for {
     a/*<-local0*/ <- List/*->scala::package.List.*/(1)
-    b/*<-local1*/ <- List/*->scala::package.List.*/(1)
+    b/*<-local1*//*->local1*/ <- List/*->scala::package.List.*/(1)
     if b/*->local1*/ >/*->scala::Int#`>`(+3).*/ 1
-    c/*<-local2*/ = a/*->local0*/ +/*->scala::Int#`+`(+4).*/ b/*->local1*/
+    c/*<-local2*//*->local2*/ = a/*->local0*/ +/*->scala::Int#`+`(+4).*/ b/*->local1*/
   } yield (a/*->local0*/, b/*->local1*/, c/*->local2*/)
   for {
-    a/*<-local3*/ <- List/*->scala::package.List.*/(1)
-    b/*<-local4*/ <- List/*->scala::package.List.*/(a/*->local3*/)
+    a/*<-local4*/ <- List/*->scala::package.List.*/(1)
+    b/*<-local5*/ <- List/*->scala::package.List.*/(a/*->local4*/)
     if (
-      a/*->local3*/,
-      b/*->local4*/
+      a/*->local4*/,
+      b/*->local5*/
     ) ==/*->scala::Any#`==`().*/ (1, 2)
     (
-      c/*<-local6*/,
-      d/*<-local7*/
-    ) <- List/*->scala::package.List.*/((a/*->local3*/, b/*->local4*/))
+      c/*<-local7*/,
+      d/*<-local8*/
+    ) <- List/*->scala::package.List.*/((a/*->local4*/, b/*->local5*/))
     if (
-      a/*->local3*/,
-      b/*->local4*/,
-      c/*->local6*/,
-      d/*->local7*/
+      a/*->local4*/,
+      b/*->local5*/,
+      c/*->local7*/,
+      d/*->local8*/
     ) ==/*->scala::Any#`==`().*/ (1, 2, 3, 4)
-    e/*<-local8*//*->local8*/ = (
-      a/*->local3*/,
-      b/*->local4*/,
-      c/*->local6*/,
-      d/*->local7*/
+    e/*<-local9*//*->local9*/ = (
+      a/*->local4*/,
+      b/*->local5*/,
+      c/*->local7*/,
+      d/*->local8*/
     )
-    if e/*->local8*/ ==/*->scala::Any#`==`().*/ (1, 2, 3, 4)
-    f/*<-local9*/ <- List/*->scala::package.List.*/(e/*->local8*/)
+    if e/*->local9*/ ==/*->scala::Any#`==`().*/ (1, 2, 3, 4)
+    f/*<-local10*/ <- List/*->scala::package.List.*/(e/*->local9*/)
   } yield {
     (
-      a/*->local3*/,
-      b/*->local4*/,
-      c/*->local6*/,
-      d/*->local7*/,
-      e/*->local8*/,
-      f/*->local9*/
+      a/*->local4*/,
+      b/*->local5*/,
+      c/*->local7*/,
+      d/*->local8*/,
+      e/*->local9*/,
+      f/*->local10*/
     )
   }
 }