Run test262 tests in CI

Run test262 tests for Linux, macOS and Cosmopolitan

Add a Makefile `test2-bootstrap` helper to clone and patch test262 tests at
a particular commit. Running with the latest commit noticed a few tests were
failing and added them to the errors list.

Noticed a few flaky staging math tests (acosh and cbrt) and made a PR to
test262 to fix those.

The CI environment, especially for macOS, was a bit more unpredictable and was
failing in atomics tests due to lower timeouts in the `atomicsHelper` so double
the timeouts. The time didn't worsen too much. It still takes about 2-3 min.

Author: nickva

.github/workflows/ci.yml

@@ -36,6 +36,10 @@ jobs:
       - name: Run microbench
         run: |
           make microbench
+      - name: Run test262
+        run: |
+          make test2-bootstrap
+          make test2
 
   linux-lto:
     name: Linux LTO
@@ -138,6 +142,10 @@ jobs:
       - name: Run built-in tests
         run: |
           make test
+      - name: Run test262
+        run: |
+          make test2-bootstrap
+          make test2
 
   macos-asan:
     runs-on: macos-latest
@@ -202,6 +210,10 @@ jobs:
       - name: Run built-in tests
         run: |
           make CONFIG_COSMO=y test
+      - name: Run test262
+        run: |
+          make test2-bootstrap
+          make CONFIG_COSMO=y test2
 
   mingw-windows:
     name: MinGW Windows target
@@ -258,7 +270,6 @@ jobs:
         run: |
           make microbench
 
-
   qemu-alpine:
     runs-on: ubuntu-latest
 

Makefile

@@ -464,6 +464,12 @@ stats: qjs$(EXE)
 microbench: qjs$(EXE)
 	$(WINE) ./qjs$(EXE) --std tests/microbench.js
 
+TEST262_COMMIT?=c6251c64b10800c53cc535adbedd92674dc87806
+
+test2-bootstrap:
+	git clone --single-branch --shallow-since=2025-05-12 https://github.com/tc39/test262.git
+	(cd test262 && git checkout -q $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..)
+
 ifeq ($(wildcard test262o/tests.txt),)
 test2o test2o-update:
 	@echo test262o tests not installed

test262.conf

@@ -230,6 +230,7 @@ Uint32Array
 Uint8Array
 uint8array-base64=skip
 Uint8ClampedArray
+upsert=skip
 WeakMap
 WeakRef
 WeakSet

test262_errors.txt

@@ -1,3 +1,7 @@
+test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: TypeError: ArrayBuffer is detached
+test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: strict mode: TypeError: ArrayBuffer is detached
+test262/test/harness/verifyProperty-value-error.js:33: Error: The error thrown did not define the specified message
+test262/test/harness/verifyProperty-value-error.js:33: strict mode: Error: The error thrown did not define the specified message
 test262/test/language/module-code/top-level-await/module-graphs-does-not-hang.js:10: TypeError: $DONE() not called
 test262/test/staging/sm/Date/UTC-convert-all-arguments.js:75: Test262Error: index 1: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true
 test262/test/staging/sm/Date/constructor-convert-all-arguments.js:75: Test262Error: index undefined: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true
@@ -12,7 +16,6 @@ test262/test/staging/sm/Function/function-toString-builtin.js:14: Test262Error:
 }' Expected SameValue(«null», «null») to be false
 test262/test/staging/sm/Function/implicit-this-in-parameter-expression.js:13: Test262Error: Expected SameValue(«[object Object]», «undefined») to be true
 test262/test/staging/sm/Function/invalid-parameter-list.js:35: Error: Assertion failed: expected exception SyntaxError, no exception thrown
-test262/test/staging/sm/Math/cbrt-approx.js:26: Error: got 1.39561242508609, expected a number near 1.3956124250860895 (relative error: 2)
 test262/test/staging/sm/RegExp/constructor-ordering-2.js:15: Test262Error: Expected SameValue(«false», «true») to be true
 test262/test/staging/sm/RegExp/regress-613820-1.js:13: Test262Error: Expected SameValue(«"aaa"», «"aa"») to be true
 test262/test/staging/sm/RegExp/regress-613820-2.js:13: Test262Error: Expected SameValue(«"f"», «undefined») to be true

tests/test262.patch

@@ -14,9 +14,9 @@ index 9828b15..4a5919d 100644
 +//  small: 200,
 +//  long: 1000,
 +//  huge: 10000,
-+  yield: 20,
-+  small: 20,
-+  long: 100,
++  yield: 40,
++  small: 40,
++  long: 200,
 +  huge: 1000,
  };