Implement SIP-42 Support for binary integer literals (#20246)

Closes #19695

Cherry-picked without changes from reverted PR #19405

Author: sjrd

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -884,7 +884,7 @@ object Scanners {
             nextChar()
             ch match {
               case 'x' | 'X' => base = 16 ; nextChar()
-              //case 'b' | 'B' => base = 2  ; nextChar()
+              case 'b' | 'B' => base = 2  ; nextChar()
               case _         => base = 10 ; putChar('0')
             }
             if (base != 10 && !isNumberSeparator(ch) && digit2int(ch, base) < 0)

docs/_docs/reference/other-new-features/binary-literals.md

@@ -0,0 +1,19 @@
+---
+layout: doc-page
+title: "Binary Integer Literals"
+nightlyOf: https://docs.scala-lang.org/scala3/reference/changed-features/binary-integer-literals.html
+---
+
+A new syntax for integer literals has been added, it is now possible to do the following:
+```scala
+val bitmask = 0b0010_0000 // equivalent to 32, 0x20
+```
+
+Binary integer literals behave similarly to hex integer literals (`0x...`), for example:
+* Both `0b...` and `0B...` are allowed
+* `0b`/`0B` on its own is disallowed, possible alternatives: `0`, `0b0`, `0B0`
+* Only `0` and `1` are allowed after the b (`b`/`B`)
+* Underscores `_` are allowed anywhere between digits, and are ignored: `0b__1 == 0b1`
+
+
+Note: This change has been backported to Scala 2.13.13, it is therefore not technically a changed feature

docs/_spec/01-lexical-syntax.md

@@ -332,9 +332,10 @@ Literal  ::=  [‘-’] integerLiteral
 ### Integer Literals
 
 ```ebnf
-integerLiteral   ::=  (decimalNumeral | hexNumeral) [‘L’ | ‘l’]
+integerLiteral   ::=  (decimalNumeral | hexNumeral | binaryNumeral) [‘L’ | ‘l’]
 decimalNumeral   ::=  ‘0’ | digit [{digit | ‘_’} digit]
 hexNumeral       ::=  ‘0’ (‘x’ | ‘X’) hexDigit [{hexDigit | ‘_’} hexDigit]
+binaryNumeral    ::=  ‘0’ (‘b’ | ‘B’) binaryDigit [{binaryDigit | ‘_’} binaryDigit]
 ```
 
 Values of type `Int` are all integer numbers between $-2\^{31}$ and $2\^{31}-1$, inclusive.
@@ -357,7 +358,7 @@ The numeric ranges given by these types are:
 The digits of a numeric literal may be separated by arbitrarily many underscores for purposes of legibility.
 
 > ```scala
-> 0           21_000      0x7F        -42L        0xFFFF_FFFF
+> 0           21_000      0x7F        -42L        0xFFFF_FFFF        0b0100_0010
 > ```
 
 ### Floating Point Literals

docs/sidebar.yml

@@ -81,6 +81,7 @@ subsection:
           - page: reference/other-new-features/safe-initialization.md
           - page: reference/other-new-features/type-test.md
           - page: reference/other-new-features/experimental-defs.md
+          - page: reference/other-new-features/binary-literals.md
       - title: Other Changed Features
         directory: changed-features
         index: reference/changed-features/changed-features.md

project/resources/referenceReplacements/sidebar.yml

@@ -77,6 +77,7 @@ subsection:
       - page: reference/other-new-features/safe-initialization.md
       - page: reference/other-new-features/type-test.md
       - page: reference/other-new-features/experimental-defs.md
+      - page: reference/other-new-features/binary-literals.md
   - title: Other Changed Features
     directory: changed-features
     index: reference/changed-features/changed-features.md

tests/neg/binaryLiterals.scala

@@ -0,0 +1,8 @@
+
+object Test:
+  val x = 0b1__0000_0000_0000_0000__0000_0000_0000_0000 // error: number too large
+  val X = 0B1__0000_0000_0000_0000__0000_0000_0000_0000 // error: number too large
+  val y = 0b1__0000_0000_0000_0000__0000_0000_0000_0000__0000_0000_0000_0000__0000_0000_0000_0000L // error: number too large
+  val Y = 0B1__0000_0000_0000_0000__0000_0000_0000_0000__0000_0000_0000_0000__0000_0000_0000_0000L // error: number too large
+  0b // error: invalid literal number
+  0b2 // error: invalid literal number

tests/run/binaryLiterals.scala

@@ -0,0 +1,72 @@
+@main
+def Test =
+  val kenobi = 0b1
+
+  assert(kenobi == 1)
+
+  assert(0B0000 == 0)
+  assert(0B0001 == 1)
+  assert(0B0010 == 2)
+  assert(0B0100 == 4)
+  assert(0B1000 == 8)
+
+  assert(0b0000 == 0)
+  assert(0b0001 == 1)
+  assert(0b0010 == 2)
+  assert(0b0100 == 4)
+  assert(0b1000 == 8)
+
+  assert(0b0001_0000 == 16)
+  assert(0b0010_0000 == 32)
+  assert(0b0100_0000 == 64)
+  assert(0b1000_0000 == 128)
+
+  assert(0b0001_0000_0000 == 256)
+  assert(0b0010_0000_0000 == 512)
+  assert(0b0100_0000_0000 == 1024)
+  assert(0b1000_0000_0000 == 2048)
+
+  assert(0b0001_0000_0000_0000 == 4096)
+  assert(0b0010_0000_0000_0000 == 8192)
+  assert(0b0100_0000_0000_0000 == 16384)
+  assert(0b1000_0000_0000_0000 == 32768)
+
+  assert(0b0001__0000_0000_0000_0000 == 65536)
+  assert(0b0010__0000_0000_0000_0000 == 131072)
+  assert(0b0100__0000_0000_0000_0000 == 262144)
+  assert(0b1000__0000_0000_0000_0000 == 524288)
+
+  assert(0b0001_0000__0000_0000_0000_0000 == 1048576)
+  assert(0b0010_0000__0000_0000_0000_0000 == 2097152)
+  assert(0b0100_0000__0000_0000_0000_0000 == 4194304)
+  assert(0b1000_0000__0000_0000_0000_0000 == 8388608)
+
+  assert(0b0001_0000_0000__0000_0000_0000_0000 == 16777216)
+  assert(0b0010_0000_0000__0000_0000_0000_0000 == 33554432)
+  assert(0b0100_0000_0000__0000_0000_0000_0000 == 67108864)
+  assert(0b1000_0000_0000__0000_0000_0000_0000 == 134217728)
+
+  assert(0b0001_0000_0000_0000__0000_0000_0000_0000 == 268435456)
+  assert(0b0010_0000_0000_0000__0000_0000_0000_0000 == 536870912)
+  assert(0b0100_0000_0000_0000__0000_0000_0000_0000 == 1073741824)
+  assert(0b1000_0000_0000_0000__0000_0000_0000_0000L == 2147483648L)
+
+  assert(0b1000_0000_0000_0000__0000_0000_0000_0000 == -2147483648) // Signed !
+  assert(0b1111_1111_1111_1111__1111_1111_1111_1111 == -1)
+
+  // Randomly generated using https://numbergenerator.org/random-32-bit-binary-number#!numbers=10&length=32&addfilters=
+  // Converted to signed decimal using https://onlinetoolz.net/unsigned-signed#base=2&bits=32
+  assert(0b0110_1000_1100_0101_0010_1100_0100_0011 ==  1757752387)
+  assert(0b1111_0101_0100_1011_0101_1000_0011_0110 == -179611594)
+  assert(0b0000_0011_0000_1010_1010_0011_0000_0000 ==  51028736)
+  assert(0b0101_0010_1111_1001_0100_0101_1101_1011 ==  1392068059)
+  assert(0b1001_0000_1111_1001_1011_1101_1100_1111 == -1862681137)
+
+  assert(0B0000_0111_1110_1100_0111_1100_1000_0010 ==  132938882)
+  assert(0B0000_1011_0111_1011_0001_1010_1010_1000 ==  192617128)
+  assert(0B1100_1100_1000_1010_1111_0111_0100_1101 == -863307955)
+  assert(0B1000_0000_0001_0010_0001_1001_0101_1110 == -2146297506)
+  assert(0B1110_0000_0110_1100_0111_0110_1100_1111 == -529762609)
+
+  assert(0b0010_1001_0101_1001__1010_0100_1000_1010__1001_1000_0011_0111__1100_1011_0111_0101L ==  2979593543648529269L)
+  assert(0b1101_1110_0100_1000__0010_1101_1010_0010__0111_1000_1111_1001__1010_1001_0101_1000L == -2429641823128802984L)