Skip to content

Commit cd90234

Browse files
DSouzaMsteve-s
DSouzaM
authored and
steve-s
committed
Bytecode DSL interpreter migration
1 parent 5ee3714 commit cd90234

File tree

124 files changed

+12598
-1218
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

124 files changed

+12598
-1218
lines changed

‎graalpython/com.oracle.graal.python.benchmarks/python/harness.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -501,6 +501,9 @@ def run_benchmark(args):
501501
else:
502502
print("### no extra module search paths specified")
503503

504+
if GRAALPYTHON:
505+
print(f"### using bytecode DSL interpreter: {__graalpython__.is_bytecode_dsl_interpreter}")
506+
504507
BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup, warmup_runs=warmup_runs, startup=startup, live_results=live_results).run()
505508

506509

‎graalpython/com.oracle.graal.python.frozen/freeze_modules.py

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ def relpath_for_posix_display(path, base):
153153
#######################################
154154
# specs
155155

156-
def parse_frozen_specs():
156+
def parse_frozen_specs(suffix):
157157
seen = {}
158158
for section, specs in FROZEN:
159159
parsed = _parse_specs(specs, section, seen)
@@ -162,7 +162,7 @@ def parse_frozen_specs():
162162
try:
163163
source = seen[frozenid]
164164
except KeyError:
165-
source = FrozenSource.from_id(frozenid, pyfile)
165+
source = FrozenSource.from_id(frozenid, suffix, pyfile)
166166
seen[frozenid] = source
167167
else:
168168
assert not pyfile or pyfile == source.pyfile, item
@@ -270,11 +270,11 @@ def iter_subs():
270270
class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile deepfreezefile')):
271271

272272
@classmethod
273-
def from_id(cls, frozenid, pyfile=None):
273+
def from_id(cls, frozenid, suffix, pyfile=None):
274274
if not pyfile:
275275
pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'
276276
#assert os.path.exists(pyfile), (frozenid, pyfile)
277-
frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR)
277+
frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR, suffix)
278278
return cls(frozenid, pyfile, frozenfile, STDLIB_DIR)
279279

280280
@classmethod
@@ -310,7 +310,7 @@ def isbootstrap(self):
310310
return self.id in BOOTSTRAP
311311

312312

313-
def resolve_frozen_file(frozenid, destdir):
313+
def resolve_frozen_file(frozenid, destdir, suffix):
314314
"""Return the filename corresponding to the given frozen ID.
315315
316316
For stdlib modules the ID will always be the full name
@@ -323,7 +323,7 @@ def resolve_frozen_file(frozenid, destdir):
323323
raise ValueError(f'unsupported frozenid {frozenid!r}')
324324
# We use a consistent naming convention for all frozen modules.
325325
frozen_symbol = FrozenSource.resolve_symbol(frozenid)
326-
frozenfile = f"Frozen{frozen_symbol}.bin"
326+
frozenfile = f"Frozen{frozen_symbol}.{suffix}"
327327

328328
if not destdir:
329329
return frozenfile
@@ -633,11 +633,17 @@ def main():
633633
STDLIB_DIR = os.path.abspath(parsed_args.python_lib)
634634
FROZEN_MODULES_DIR = os.path.abspath(parsed_args.binary_dir)
635635

636+
if __graalpython__.is_bytecode_dsl_interpreter:
637+
suffix = "bin_dsl"
638+
assert os.path.isdir(parsed_args.binary_dir), "Frozen modules for the DSL should be built after the manual bytecode interpreter."
639+
else:
640+
suffix = "bin"
641+
shutil.rmtree(parsed_args.binary_dir, ignore_errors=True)
642+
os.makedirs(parsed_args.binary_dir)
643+
636644
# create module specs
637-
modules = list(parse_frozen_specs())
645+
modules = list(parse_frozen_specs(suffix))
638646

639-
shutil.rmtree(parsed_args.binary_dir, ignore_errors=True)
640-
os.makedirs(parsed_args.binary_dir)
641647
# write frozen module binary files containing the byte code and class files
642648
# used for importing the binary files
643649
for src in _iter_sources(modules):

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/scope/Scope.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,11 +229,15 @@ public boolean isNested() {
229229
}
230230

231231
public HashMap<String, Integer> getSymbolsByType(EnumSet<DefUse> expectedFlags, int start) {
232+
return getSymbolsByType(expectedFlags, EnumSet.noneOf(DefUse.class), start);
233+
}
234+
235+
public HashMap<String, Integer> getSymbolsByType(EnumSet<DefUse> expectedFlags, EnumSet<DefUse> unexpectedFlags, int start) {
232236
int i = start;
233237
HashMap<String, Integer> mapping = new HashMap<>();
234238
for (String key : getSortedSymbols()) {
235239
EnumSet<DefUse> keyFlags = getUseOfName(key);
236-
if (!Collections.disjoint(expectedFlags, keyFlags)) {
240+
if (!Collections.disjoint(expectedFlags, keyFlags) && Collections.disjoint(unexpectedFlags, keyFlags)) {
237241
mapping.put(key, i++);
238242
}
239243
}

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/scope/ScopeEnvironment.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ public class ScopeEnvironment {
9898
final HashMap<SSTNode, Scope> blocks = new HashMap<>();
9999
final ErrorCallback errorCallback;
100100
final EnumSet<FutureFeature> futureFeatures;
101+
final HashMap<Scope, Scope> parents = new HashMap<>();
101102

102103
public static ScopeEnvironment analyze(ModTy moduleNode, ErrorCallback errorCallback, EnumSet<FutureFeature> futureFeatures) {
103104
return new ScopeEnvironment(moduleNode, errorCallback, futureFeatures);
@@ -128,6 +129,14 @@ public Scope lookupScope(SSTNode node) {
128129
return blocks.get(node);
129130
}
130131

132+
public Scope lookupParent(Scope scope) {
133+
return parents.get(scope);
134+
}
135+
136+
public Scope getTopScope() {
137+
return topScope;
138+
}
139+
131140
private void analyzeBlock(Scope scope, HashSet<String> bound, HashSet<String> free, HashSet<String> global) {
132141
HashSet<String> local = new HashSet<>();
133142
HashMap<String, DefUse> scopes = new HashMap<>();
@@ -328,6 +337,7 @@ private void enterBlock(String name, Scope.ScopeType type, SSTNode ast) {
328337
if (type == Scope.ScopeType.Annotation) {
329338
return;
330339
}
340+
env.parents.put(scope, prev);
331341
if (prev != null) {
332342
prev.children.add(scope);
333343
}

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/sst/PatternTy.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -171,6 +171,7 @@ public static final class MatchOr extends PatternTy {
171171

172172
public MatchOr(PatternTy[] patterns, SourceRange sourceRange) {
173173
super(sourceRange);
174+
assert patterns.length > 1;
174175
this.patterns = patterns;
175176
}
176177

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,11 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
420420
}
421421
}
422422

423+
if (!ImageInfo.inImageCode() && Boolean.getBoolean("python.EnableBytecodeDSLInterpreter")) {
424+
// forward the property on JVM
425+
addRelaunchArg("--vm.Dpython.EnableBytecodeDSLInterpreter=true");
426+
}
427+
423428
// According to CPython if no arguments are given, they contain an empty string.
424429
if (programArgs.isEmpty()) {
425430
programArgs.add("");

‎graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/generator/GeneratorTests.java

Lines changed: 116 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public void desugared() {
7272
}
7373

7474
@Test
75-
public void testYieldFrom() {
75+
public void testYieldFromSimple() {
7676
String source = "def gen1():\n" +
7777
" yield 1\n" +
7878
" yield 2\n" +
@@ -83,4 +83,119 @@ public void testYieldFrom() {
8383
"print(list(gen2()))\n";
8484
assertPrints("[1, 2]\n", source);
8585
}
86+
87+
@Test
88+
public void testYieldFromIterable() {
89+
// yield from should extract an iterator from a non-generator argument
90+
String source = "class Foo:\n" +
91+
" def __init__(self, wrapped):\n" +
92+
" self.wrapped = wrapped\n" +
93+
" def __iter__(self):\n" +
94+
" return iter(self.wrapped)\n" +
95+
"def gen():\n" +
96+
" foo = Foo([1,2,3])\n" +
97+
" yield from foo\n" +
98+
"\n" +
99+
"print(list(gen()))\n";
100+
assertPrints("[1, 2, 3]\n", source);
101+
}
102+
103+
@Test
104+
public void testYieldFromReturn() {
105+
String source = "def gen1():\n" +
106+
" yield 1\n" +
107+
" yield 2\n" +
108+
" return 3\n" +
109+
"\n" +
110+
"def gen2():\n" +
111+
" final = yield from gen1()\n" +
112+
" yield final\n" +
113+
"\n" +
114+
"print(list(gen2()))\n";
115+
assertPrints("[1, 2, 3]\n", source);
116+
}
117+
118+
@Test
119+
public void testYieldFromSend() {
120+
String source = "def gen1(x):\n" +
121+
" yield (yield (yield x))\n" +
122+
"\n" +
123+
"def gen2():\n" +
124+
" yield from gen1(2)\n" +
125+
" yield 8\n" +
126+
"\n" +
127+
"gen = gen2()\n" +
128+
"print(gen.send(None))\n" +
129+
"print(gen.send(4))\n" +
130+
"print(gen.send(6))\n" +
131+
"print(gen.send(42))\n";
132+
assertPrints("2\n4\n6\n8\n", source);
133+
}
134+
135+
@Test
136+
public void testYieldFromThrowCaught() {
137+
String source = "def gen1():\n" +
138+
" try:\n" +
139+
" x = 1\n" +
140+
" while True:\n" +
141+
" x = yield x\n" +
142+
" except ValueError:\n" +
143+
" yield 42\n" +
144+
"\n" +
145+
"def gen2():\n" +
146+
" yield from gen1()\n" +
147+
"\n" +
148+
"gen = gen2()\n" +
149+
"print(gen.send(None))\n" +
150+
"print(gen.send(2))\n" +
151+
"print(gen.send(3))\n" +
152+
"print(gen.throw(ValueError))\n";
153+
assertPrints("1\n2\n3\n42\n", source);
154+
}
155+
156+
@Test
157+
public void testYieldFromThrowUncaught() {
158+
String source = "def gen1():\n" +
159+
" x = 1\n" +
160+
" while True:\n" +
161+
" x = yield x\n" +
162+
"\n" +
163+
"def gen2():\n" +
164+
" yield from gen1()\n" +
165+
"\n" +
166+
"gen = gen2()\n" +
167+
"print(gen.send(None))\n" +
168+
"print(gen.send(2))\n" +
169+
"print(gen.send(3))\n" +
170+
"try:\n" +
171+
" gen.throw(ValueError)\n" +
172+
" print('error')\n" +
173+
"except ValueError:\n" +
174+
" print('success')\n";
175+
assertPrints("1\n2\n3\nsuccess\n", source);
176+
}
177+
178+
@Test
179+
public void testYieldFromClose() {
180+
String source = "def gen1():\n" +
181+
" x = 1\n" +
182+
" try:\n" +
183+
" while True:\n" +
184+
" x = yield x\n" +
185+
" except GeneratorExit:\n" +
186+
" print('gen1 exit')\n" +
187+
"\n" +
188+
"def gen2():\n" +
189+
" try:\n" +
190+
" yield from gen1()\n" +
191+
" except GeneratorExit:\n" +
192+
" print('gen2 exit')\n" +
193+
"\n" +
194+
"gen = gen2()\n" +
195+
"print(gen.send(None))\n" +
196+
"print(gen.send(2))\n" +
197+
"print(gen.send(3))\n" +
198+
"gen.close()\n";
199+
assertPrints("1\n2\n3\ngen1 exit\ngen2 exit\n", source);
200+
}
86201
}

‎graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/grammar/AsyncTests.java

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
/*
2+
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.test.integration.grammar;
42+
43+
import static com.oracle.graal.python.test.integration.PythonTests.assertPrints;
44+
45+
import org.junit.Assume;
46+
import org.junit.Test;
47+
48+
public class AsyncTests {
49+
@Test
50+
public void nativeCoroutine() {
51+
Assume.assumeFalse("TODO: not implemented PGenerator.getYieldFrom(PGenerator.java:204)", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
52+
String source = "import asyncio\n" +
53+
"async def foo():\n" +
54+
" return 42\n" +
55+
"async def main():\n" +
56+
" print(await foo())\n" +
57+
"asyncio.run(main())";
58+
assertPrints("42\n", source);
59+
}
60+
61+
@Test
62+
public void asyncWith() {
63+
Assume.assumeFalse("TODO: not implemented PGenerator.getYieldFrom(PGenerator.java:204)", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
64+
String source = "import asyncio\n" +
65+
"class AsyncContextManager:\n" +
66+
" async def __aenter__(self):\n" +
67+
" await asyncio.sleep(0.01)\n" +
68+
" print(\"entered\")\n" +
69+
" async def __aexit__(self, exc_type, exc_value, traceback):\n" +
70+
" await asyncio.sleep(0.01)\n" +
71+
" if exc_type:\n" +
72+
" print(\"exited exceptionally\")\n" +
73+
" else:\n" +
74+
" print(\"exited normally\")\n" +
75+
" return True\n" +
76+
"async def main(shouldRaise):\n" +
77+
" async with AsyncContextManager():\n" +
78+
" print(\"inside\")\n" +
79+
" if shouldRaise:\n" +
80+
" raise ValueError\n" +
81+
"asyncio.run(main(%s))";
82+
assertPrints("entered\ninside\nexited normally\n", String.format(source, "False"));
83+
assertPrints("entered\ninside\nexited exceptionally\n", String.format(source, "True"));
84+
}
85+
86+
@Test
87+
public void asyncWithExceptional() {
88+
Assume.assumeFalse("TODO: not implemented PGenerator.getYieldFrom(PGenerator.java:204)", Boolean.getBoolean("python.EnableBytecodeDSLInterpreter"));
89+
String source = "import asyncio\n" +
90+
"class AsyncContextManager:\n" +
91+
" async def __aenter__(self):\n" +
92+
" await asyncio.sleep(0.01)\n" +
93+
" print(\"entered\")\n" +
94+
" async def __aexit__(self, exc_type, exc_value, traceback):\n" +
95+
" await asyncio.sleep(0.01)\n" +
96+
" print(\"exited\")\n" +
97+
" return False\n" + // don't handle exception
98+
"async def main(shouldRaise):\n" +
99+
" async with AsyncContextManager():\n" +
100+
" print(\"inside\")\n" +
101+
" if shouldRaise:\n" +
102+
" raise ValueError\n" +
103+
"try:\n" +
104+
" asyncio.run(main(%s))\n" +
105+
"except ValueError:\n" +
106+
" print(\"rethrew\")\n";
107+
assertPrints("entered\ninside\nexited\n", String.format(source, "False"));
108+
assertPrints("entered\ninside\nexited\nrethrew\n", String.format(source, "True"));
109+
}
110+
}

0 commit comments

Comments
 (0)