Bytecode DSL interpreter migration

Author: DSouzaM

Diff for: graalpython/com.oracle.graal.python.benchmarks/python/harness.py

@@ -501,6 +501,9 @@ def run_benchmark(args):
     else:
         print("### no extra module search paths specified")
 
+    if GRAALPYTHON:
+        print(f"### using bytecode DSL interpreter: {__graalpython__.is_bytecode_dsl_interpreter}")
+
     BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup, warmup_runs=warmup_runs, startup=startup, live_results=live_results).run()
 
 

Diff for: graalpython/com.oracle.graal.python.frozen/freeze_modules.py

@@ -153,7 +153,7 @@ def relpath_for_posix_display(path, base):
 #######################################
 # specs
 
-def parse_frozen_specs():
+def parse_frozen_specs(suffix):
     seen = {}
     for section, specs in FROZEN:
         parsed = _parse_specs(specs, section, seen)
@@ -162,7 +162,7 @@ def parse_frozen_specs():
             try:
                 source = seen[frozenid]
             except KeyError:
-                source = FrozenSource.from_id(frozenid, pyfile)
+                source = FrozenSource.from_id(frozenid, suffix, pyfile)
                 seen[frozenid] = source
             else:
                 assert not pyfile or pyfile == source.pyfile, item
@@ -270,11 +270,11 @@ def iter_subs():
 class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile deepfreezefile')):
 
     @classmethod
-    def from_id(cls, frozenid, pyfile=None):
+    def from_id(cls, frozenid, suffix, pyfile=None):
         if not pyfile:
             pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'
             #assert os.path.exists(pyfile), (frozenid, pyfile)
-        frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR)
+        frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR, suffix)
         return cls(frozenid, pyfile, frozenfile, STDLIB_DIR)
 
     @classmethod
@@ -310,7 +310,7 @@ def isbootstrap(self):
         return self.id in BOOTSTRAP
 
 
-def resolve_frozen_file(frozenid, destdir):
+def resolve_frozen_file(frozenid, destdir, suffix):
     """Return the filename corresponding to the given frozen ID.
 
     For stdlib modules the ID will always be the full name
@@ -323,7 +323,7 @@ def resolve_frozen_file(frozenid, destdir):
             raise ValueError(f'unsupported frozenid {frozenid!r}')
     # We use a consistent naming convention for all frozen modules.
     frozen_symbol = FrozenSource.resolve_symbol(frozenid)
-    frozenfile = f"Frozen{frozen_symbol}.bin"
+    frozenfile = f"Frozen{frozen_symbol}.{suffix}"
 
     if not destdir:
         return frozenfile
@@ -633,11 +633,17 @@ def main():
     STDLIB_DIR = os.path.abspath(parsed_args.python_lib)
     FROZEN_MODULES_DIR = os.path.abspath(parsed_args.binary_dir)
 
+    if __graalpython__.is_bytecode_dsl_interpreter:
+        suffix = "bin_dsl"
+        assert os.path.isdir(parsed_args.binary_dir), "Frozen modules for the DSL should be built after the manual bytecode interpreter."
+    else:
+        suffix = "bin"
+        shutil.rmtree(parsed_args.binary_dir, ignore_errors=True)
+        os.makedirs(parsed_args.binary_dir)
+
     # create module specs
-    modules = list(parse_frozen_specs())
+    modules = list(parse_frozen_specs(suffix))
 
-    shutil.rmtree(parsed_args.binary_dir, ignore_errors=True)
-    os.makedirs(parsed_args.binary_dir)
     # write frozen module binary files containing the byte code and class files
     # used for importing the binary files
     for src in _iter_sources(modules):

Diff for: graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/scope/Scope.java

@@ -229,11 +229,15 @@ public boolean isNested() {
     }
 
     public HashMap<String, Integer> getSymbolsByType(EnumSet<DefUse> expectedFlags, int start) {
+        return getSymbolsByType(expectedFlags, EnumSet.noneOf(DefUse.class), start);
+    }
+
+    public HashMap<String, Integer> getSymbolsByType(EnumSet<DefUse> expectedFlags, EnumSet<DefUse> unexpectedFlags, int start) {
         int i = start;
         HashMap<String, Integer> mapping = new HashMap<>();
         for (String key : getSortedSymbols()) {
             EnumSet<DefUse> keyFlags = getUseOfName(key);
-            if (!Collections.disjoint(expectedFlags, keyFlags)) {
+            if (!Collections.disjoint(expectedFlags, keyFlags) && Collections.disjoint(unexpectedFlags, keyFlags)) {
                 mapping.put(key, i++);
             }
         }

Diff for: graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/scope/ScopeEnvironment.java

@@ -98,6 +98,7 @@ public class ScopeEnvironment {
     final HashMap<SSTNode, Scope> blocks = new HashMap<>();
     final ErrorCallback errorCallback;
     final EnumSet<FutureFeature> futureFeatures;
+    final HashMap<Scope, Scope> parents = new HashMap<>();
 
     public static ScopeEnvironment analyze(ModTy moduleNode, ErrorCallback errorCallback, EnumSet<FutureFeature> futureFeatures) {
         return new ScopeEnvironment(moduleNode, errorCallback, futureFeatures);
@@ -128,6 +129,14 @@ public Scope lookupScope(SSTNode node) {
         return blocks.get(node);
     }
 
+    public Scope lookupParent(Scope scope) {
+        return parents.get(scope);
+    }
+
+    public Scope getTopScope() {
+        return topScope;
+    }
+
     private void analyzeBlock(Scope scope, HashSet<String> bound, HashSet<String> free, HashSet<String> global) {
         HashSet<String> local = new HashSet<>();
         HashMap<String, DefUse> scopes = new HashMap<>();
@@ -328,6 +337,7 @@ private void enterBlock(String name, Scope.ScopeType type, SSTNode ast) {
             if (type == Scope.ScopeType.Annotation) {
                 return;
             }
+            env.parents.put(scope, prev);
             if (prev != null) {
                 prev.children.add(scope);
             }

Diff for: graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/sst/PatternTy.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -171,6 +171,7 @@ public static final class MatchOr extends PatternTy {
 
         public MatchOr(PatternTy[] patterns, SourceRange sourceRange) {
             super(sourceRange);
+            assert patterns.length > 1;
             this.patterns = patterns;
         }
 

Diff for: graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -420,6 +420,11 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
             }
         }
 
+        if (!ImageInfo.inImageCode() && Boolean.getBoolean("python.EnableBytecodeDSLInterpreter")) {
+            // forward the property on JVM
+            addRelaunchArg("--vm.Dpython.EnableBytecodeDSLInterpreter=true");
+        }
+
         // According to CPython if no arguments are given, they contain an empty string.
         if (programArgs.isEmpty()) {
             programArgs.add("");

Diff for: graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/generator/GeneratorTests.java

@@ -72,7 +72,7 @@ public void desugared() {
     }
 
     @Test
-    public void testYieldFrom() {
+    public void testYieldFromSimple() {
         String source = "def gen1():\n" +
                         "    yield 1\n" +
                         "    yield 2\n" +
@@ -83,4 +83,119 @@ public void testYieldFrom() {
                         "print(list(gen2()))\n";
         assertPrints("[1, 2]\n", source);
     }
+
+    @Test
+    public void testYieldFromIterable() {
+        // yield from should extract an iterator from a non-generator argument
+        String source = "class Foo:\n" +
+                        "    def __init__(self, wrapped):\n" +
+                        "        self.wrapped = wrapped\n" +
+                        "    def __iter__(self):\n" +
+                        "        return iter(self.wrapped)\n" +
+                        "def gen():\n" +
+                        "    foo = Foo([1,2,3])\n" +
+                        "    yield from foo\n" +
+                        "\n" +
+                        "print(list(gen()))\n";
+        assertPrints("[1, 2, 3]\n", source);
+    }
+
+    @Test
+    public void testYieldFromReturn() {
+        String source = "def gen1():\n" +
+                        "    yield 1\n" +
+                        "    yield 2\n" +
+                        "    return 3\n" +
+                        "\n" +
+                        "def gen2():\n" +
+                        "    final = yield from gen1()\n" +
+                        "    yield final\n" +
+                        "\n" +
+                        "print(list(gen2()))\n";
+        assertPrints("[1, 2, 3]\n", source);
+    }
+
+    @Test
+    public void testYieldFromSend() {
+        String source = "def gen1(x):\n" +
+                        "    yield (yield (yield x))\n" +
+                        "\n" +
+                        "def gen2():\n" +
+                        "    yield from gen1(2)\n" +
+                        "    yield 8\n" +
+                        "\n" +
+                        "gen = gen2()\n" +
+                        "print(gen.send(None))\n" +
+                        "print(gen.send(4))\n" +
+                        "print(gen.send(6))\n" +
+                        "print(gen.send(42))\n";
+        assertPrints("2\n4\n6\n8\n", source);
+    }
+
+    @Test
+    public void testYieldFromThrowCaught() {
+        String source = "def gen1():\n" +
+                        "    try:\n" +
+                        "        x = 1\n" +
+                        "        while True:\n" +
+                        "            x = yield x\n" +
+                        "    except ValueError:\n" +
+                        "        yield 42\n" +
+                        "\n" +
+                        "def gen2():\n" +
+                        "    yield from gen1()\n" +
+                        "\n" +
+                        "gen = gen2()\n" +
+                        "print(gen.send(None))\n" +
+                        "print(gen.send(2))\n" +
+                        "print(gen.send(3))\n" +
+                        "print(gen.throw(ValueError))\n";
+        assertPrints("1\n2\n3\n42\n", source);
+    }
+
+    @Test
+    public void testYieldFromThrowUncaught() {
+        String source = "def gen1():\n" +
+                        "    x = 1\n" +
+                        "    while True:\n" +
+                        "        x = yield x\n" +
+                        "\n" +
+                        "def gen2():\n" +
+                        "    yield from gen1()\n" +
+                        "\n" +
+                        "gen = gen2()\n" +
+                        "print(gen.send(None))\n" +
+                        "print(gen.send(2))\n" +
+                        "print(gen.send(3))\n" +
+                        "try:\n" +
+                        "    gen.throw(ValueError)\n" +
+                        "    print('error')\n" +
+                        "except ValueError:\n" +
+                        "    print('success')\n";
+        assertPrints("1\n2\n3\nsuccess\n", source);
+    }
+
+    @Test
+    public void testYieldFromClose() {
+        String source = "def gen1():\n" +
+                        "    x = 1\n" +
+                        "    try:\n" +
+                        "        while True:\n" +
+                        "            x = yield x\n" +
+                        "    except GeneratorExit:\n" +
+                        "        print('gen1 exit')\n" +
+                        "\n" +
+                        "def gen2():\n" +
+                        "    try:\n" +
+                        "        yield from gen1()\n" +
+                        "    except GeneratorExit:\n" +
+                        "        print('gen2 exit')\n" +
+                        "\n" +
+                        "gen = gen2()\n" +
+                        "print(gen.send(None))\n" +
+                        "print(gen.send(2))\n" +
+                        "print(gen.send(3))\n" +
+                        "gen.close()\n";
+        assertPrints("1\n2\n3\ngen1 exit\ngen2 exit\n", source);
+    }
 }

Diff for: graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/grammar/AsyncTests.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.test.integration.grammar;
+
+import static com.oracle.graal.python.test.integration.PythonTests.assertPrints;
+
+import org.junit.Assume;
+import org.junit.Test;
+
+public class AsyncTests {
+    @Test
+    public void nativeCoroutine() {
+        Assume.assumeFalse("TODO: not implemented PGenerator.getYieldFrom(PGenerator.java:204)", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        String source = "import asyncio\n" +
+                        "async def foo():\n" +
+                        "  return 42\n" +
+                        "async def main():\n" +
+                        "  print(await foo())\n" +
+                        "asyncio.run(main())";
+        assertPrints("42\n", source);
+    }
+
+    @Test
+    public void asyncWith() {
+        Assume.assumeFalse("TODO: not implemented PGenerator.getYieldFrom(PGenerator.java:204)", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        String source = "import asyncio\n" +
+                        "class AsyncContextManager:\n" +
+                        "  async def __aenter__(self):\n" +
+                        "    await asyncio.sleep(0.01)\n" +
+                        "    print(\"entered\")\n" +
+                        "  async def __aexit__(self, exc_type, exc_value, traceback):\n" +
+                        "    await asyncio.sleep(0.01)\n" +
+                        "    if exc_type:\n" +
+                        "      print(\"exited exceptionally\")\n" +
+                        "    else:\n" +
+                        "      print(\"exited normally\")\n" +
+                        "    return True\n" +
+                        "async def main(shouldRaise):\n" +
+                        "  async with AsyncContextManager():\n" +
+                        "    print(\"inside\")\n" +
+                        "    if shouldRaise:\n" +
+                        "      raise ValueError\n" +
+                        "asyncio.run(main(%s))";
+        assertPrints("entered\ninside\nexited normally\n", String.format(source, "False"));
+        assertPrints("entered\ninside\nexited exceptionally\n", String.format(source, "True"));
+    }
+
+    @Test
+    public void asyncWithExceptional() {
+        Assume.assumeFalse("TODO: not implemented PGenerator.getYieldFrom(PGenerator.java:204)", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
+        String source = "import asyncio\n" +
+                        "class AsyncContextManager:\n" +
+                        "  async def __aenter__(self):\n" +
+                        "    await asyncio.sleep(0.01)\n" +
+                        "    print(\"entered\")\n" +
+                        "  async def __aexit__(self, exc_type, exc_value, traceback):\n" +
+                        "    await asyncio.sleep(0.01)\n" +
+                        "    print(\"exited\")\n" +
+                        "    return False\n" + // don't handle exception
+                        "async def main(shouldRaise):\n" +
+                        "  async with AsyncContextManager():\n" +
+                        "    print(\"inside\")\n" +
+                        "    if shouldRaise:\n" +
+                        "      raise ValueError\n" +
+                        "try:\n" +
+                        "  asyncio.run(main(%s))\n" +
+                        "except ValueError:\n" +
+                        "  print(\"rethrew\")\n";
+        assertPrints("entered\ninside\nexited\n", String.format(source, "False"));
+        assertPrints("entered\ninside\nexited\nrethrew\n", String.format(source, "True"));
+    }
+}