Skip to content

Commit 3fe4bfc

Browse files
DSouzaMDSouzaM
committed
implement co_positions, PyCode_Addr2Line, f_lasti, tb_lasti; add assertions to detect pyc encoding mismatch
1 parent 76460f2 commit 3fe4bfc

File tree

10 files changed

+79
-58
lines changed

10 files changed

+79
-58
lines changed

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1329,6 +1329,11 @@ private CodeUnit readCodeUnit() {
13291329
}
13301330

13311331
private BytecodeCodeUnit readBytecodeCodeUnit() {
1332+
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
1333+
throw new AssertionError(
1334+
"Attempted to deserialize a code object from the manual bytecode interpreter, but the DSL interpreter is enabled. Consider clearing or setting a different pycache folder.");
1335+
}
1336+
13321337
int fileVersion = readByte();
13331338
if (fileVersion != Compiler.BYTECODE_VERSION) {
13341339
throw new MarshalError(ValueError, ErrorMessages.BYTECODE_VERSION_MISMATCH, Compiler.BYTECODE_VERSION, fileVersion);
@@ -1369,6 +1374,11 @@ private BytecodeCodeUnit readBytecodeCodeUnit() {
13691374
}
13701375

13711376
private BytecodeDSLCodeUnit readBytecodeDSLCodeUnit() {
1377+
if (!PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
1378+
throw new AssertionError(
1379+
"Attempted to deserialize a code object from the Bytecode DSL interpreter, but the manual interpreter is enabled. Consider clearing or setting a different pycache folder.");
1380+
}
1381+
13721382
byte[] serialized = readBytes();
13731383
TruffleString name = readString();
13741384
TruffleString qualname = readString();

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextCodeBuiltins.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ static int addr2line(PCode code, int lasti) {
117117
if (lasti < 0) {
118118
return code.co_firstlineno();
119119
}
120-
return code.bciToLine(code.lastiToBci(lasti));
120+
return code.lastiToLine(lasti);
121121
}
122122
}
123123

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeBuiltins.java

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
import com.oracle.graal.python.util.PythonUtils;
7474
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
7575
import com.oracle.truffle.api.bytecode.introspection.BytecodeIntrospection;
76+
import com.oracle.truffle.api.bytecode.introspection.Instruction;
7677
import com.oracle.truffle.api.bytecode.introspection.SourceInformation;
7778
import com.oracle.truffle.api.dsl.Bind;
7879
import com.oracle.truffle.api.dsl.Cached;
@@ -83,6 +84,7 @@
8384
import com.oracle.truffle.api.frame.VirtualFrame;
8485
import com.oracle.truffle.api.library.CachedLibrary;
8586
import com.oracle.truffle.api.nodes.Node;
87+
import com.oracle.truffle.api.source.SourceSection;
8688
import com.oracle.truffle.api.strings.TruffleString;
8789

8890
@CoreFunctions(extendClasses = PythonBuiltinClassType.PCode)
@@ -350,22 +352,32 @@ Object positions(PCode self) {
350352
PTuple tuple;
351353
CodeUnit co = self.getCodeUnit();
352354
if (co != null) {
355+
List<PTuple> lines = new ArrayList<>();
353356
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
354-
// TODO implement
355-
throw new UnsupportedOperationException("not implemented");
357+
PBytecodeDSLRootNode rootNode = (PBytecodeDSLRootNode) self.getRootNode();
358+
for (Instruction instruction : rootNode.getIntrospectionData().getInstructions()) {
359+
SourceSection section = rootNode.getSourceSectionForLocation(instruction.getLocation());
360+
lines.add(factory.createTuple(new int[]{
361+
section.getStartLine(),
362+
section.getEndLine(),
363+
// 1-based inclusive to 0-based inclusive
364+
section.getStartColumn() - 1,
365+
// 1-based inclusive to 0-based exclusive (-1 + 1 = 0)
366+
section.getEndColumn()
367+
}));
368+
}
356369
} else {
357370
BytecodeCodeUnit bytecodeCo = (BytecodeCodeUnit) co;
358371
SourceMap map = bytecodeCo.getSourceMap();
359-
List<PTuple> lines = new ArrayList<>();
360372
if (map != null && map.startLineMap.length > 0) {
361373
byte[] bytecode = bytecodeCo.code;
362374
for (int i = 0; i < bytecode.length;) {
363375
lines.add(factory.createTuple(new int[]{map.startLineMap[i], map.endLineMap[i], map.startColumnMap[i], map.endColumnMap[i]}));
364376
i += OpCodes.fromOpCode(bytecode[i]).length();
365377
}
366378
}
367-
tuple = factory.createTuple(lines.toArray());
368379
}
380+
tuple = factory.createTuple(lines.toArray());
369381
} else {
370382
tuple = factory.createEmptyTuple();
371383
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,8 @@
8484
import com.oracle.truffle.api.CompilerDirectives;
8585
import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
8686
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
87+
import com.oracle.truffle.api.bytecode.BytecodeLocation;
88+
import com.oracle.truffle.api.bytecode.BytecodeNode;
8789
import com.oracle.truffle.api.bytecode.ContinuationRootNode;
8890
import com.oracle.truffle.api.RootCallTarget;
8991
import com.oracle.truffle.api.dsl.Cached;
@@ -470,25 +472,21 @@ public int getFirstLineNo() {
470472
}
471473

472474
@TruffleBoundary
473-
public int bciToLine(int bci) {
475+
public int lastiToLine(int lasti) {
474476
RootNode funcRootNode = rootNodeForExtraction(getRootNode());
475-
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER && funcRootNode instanceof PBytecodeDSLRootNode bytecodeDSLRootNode) {
476-
throw new UnsupportedOperationException("not implemented");
477+
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
478+
if (funcRootNode instanceof PBytecodeDSLRootNode bytecodeDSLRootNode) {
479+
BytecodeNode bytecodeNode = bytecodeDSLRootNode.getBytecodeNode();
480+
// Emulate CPython's fixed 2-word instructions.
481+
BytecodeLocation location = BytecodeLocation.fromInstructionIndex((lasti + 1) / 2, bytecodeNode);
482+
return location.findSourceLocation().getStartLine();
483+
}
477484
} else if (funcRootNode instanceof PBytecodeRootNode bytecodeRootNode) {
478-
return bytecodeRootNode.bciToLine(bci);
485+
return bytecodeRootNode.bciToLine(bytecodeRootNode.lastiToBci(lasti));
479486
}
480487
return -1;
481488
}
482489

483-
@TruffleBoundary
484-
public int lastiToBci(int lasti) {
485-
RootNode funcRootNode = rootNodeForExtraction(getRootNode());
486-
if (funcRootNode instanceof PBytecodeRootNode bytecodeRootNode) {
487-
return bytecodeRootNode.lastiToBci(lasti);
488-
}
489-
return lasti;
490-
}
491-
492490
public TruffleString getName() {
493491
if (name == null) {
494492
name = extractName(getRootNode());

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/frame/PFrame.java

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,10 @@ public Node getLocation() {
290290
return location;
291291
}
292292

293+
public BytecodeNode getBytecodeNode() {
294+
return (location instanceof BytecodeNode bytecodeNode) ? bytecodeNode : null;
295+
}
296+
293297
public int getBci() {
294298
return bci;
295299
}
@@ -299,14 +303,16 @@ public void setBci(int bci) {
299303
}
300304

301305
public int getLasti() {
302-
return bciToLasti(bci);
306+
return bciToLasti(bci, location);
303307
}
304308

305309
@TruffleBoundary
306-
public int bciToLasti(int bci) {
310+
public static int bciToLasti(int bci, Node location) {
307311
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
308-
// TODO implement
309-
throw new UnsupportedOperationException("not implemented");
312+
if (location instanceof BytecodeNode bytecodeNode) {
313+
// Emulate CPython's fixed 2-word instructions.
314+
return bytecodeNode.findInstructionIndex(bci) * 2;
315+
}
310316
} else {
311317
if (location instanceof PBytecodeRootNode bytecodeRootNode) {
312318
return bytecodeRootNode.bciToLasti(bci);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/traceback/PTraceback.java

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,10 @@
4646
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
4747
import com.oracle.graal.python.builtins.objects.frame.PFrame;
4848
import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject;
49+
import com.oracle.graal.python.runtime.PythonOptions;
4950
import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
51+
import com.oracle.truffle.api.bytecode.BytecodeNode;
52+
import com.oracle.truffle.api.nodes.Node;
5053
import com.oracle.truffle.api.strings.TruffleString;
5154

5255
public final class PTraceback extends PythonBuiltinObject {
@@ -56,6 +59,7 @@ public final class PTraceback extends PythonBuiltinObject {
5659
private PFrame.Reference frameInfo;
5760
private int lineno = UNKNOWN_LINE_NUMBER;
5861
private int bci = -1;
62+
private BytecodeNode bytecodeNode = null;
5963
private int lasti = -1;
6064
private PTraceback next;
6165
private LazyTraceback lazyTraceback;
@@ -107,13 +111,20 @@ public int getLineno() {
107111

108112
public int getLasti(PFrame pFrame) {
109113
if (lasti == -1 && bci >= 0) {
110-
lasti = pFrame.bciToLasti(bci);
114+
Node location;
115+
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
116+
location = bytecodeNode;
117+
} else {
118+
location = pFrame.getLocation();
119+
}
120+
lasti = PFrame.bciToLasti(bci, location);
111121
}
112122
return lasti;
113123
}
114124

115-
public void setBci(int bci) {
125+
public void setLocation(int bci, BytecodeNode bytecodeNode) {
116126
this.bci = bci;
127+
this.bytecodeNode = bytecodeNode; // nullable
117128
this.lasti = -1;
118129
}
119130

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/traceback/TracebackBuiltins.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,13 +137,13 @@ static void doMaterialize(Node inliningTarget, PTraceback tb,
137137
if (LazyTraceback.elementWantedForTraceback(element)) {
138138
PFrame pFrame = materializeFrame(element, materializeFrameNode);
139139
next = factory.createTraceback(pFrame, pFrame.getLine(), next);
140-
next.setBci(pFrame.getBci());
140+
next.setLocation(pFrame.getBci(), pFrame.getBytecodeNode());
141141
pyIndex++;
142142
}
143143
}
144144
}
145145
if (lazyTraceback.catchingFrameWantedForTraceback()) {
146-
tb.setBci(pException.getCatchBci());
146+
tb.setLocation(pException.getCatchBci(), pException.getBytecodeNode());
147147
tb.setLineno(pException.getCatchLine());
148148
tb.setNext(next);
149149
} else {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@ public class RootNodeCompiler implements BaseBytecodeDSLVisitor<BytecodeDSLCompi
8080
private final Map<String, BytecodeLocal> freeLocals;
8181
private final Scope scope;
8282
private final CompilationScope scopeType;
83-
private final GeneratorOrCoroutineKind generatorOrCoroutineKind;
8483

8584
private final boolean isInteractive;
8685
private final EnumSet<FutureFeature> futureFeatures;
@@ -103,7 +102,6 @@ public RootNodeCompiler(BytecodeDSLCompilerContext ctx, SSTNode rootNode, EnumSe
103102
this.freeLocals = new HashMap<>();
104103
this.scope = ctx.scopeEnvironment.lookupScope(rootNode);
105104
this.scopeType = getScopeType(scope, rootNode);
106-
this.generatorOrCoroutineKind = GeneratorOrCoroutineKind.getKind(scope);
107105

108106
this.isInteractive = rootNode instanceof ModTy.Interactive;
109107
this.futureFeatures = futureFeatures;
@@ -548,34 +546,6 @@ private static TruffleString getDocstring(StmtTy[] body) {
548546
return null;
549547
}
550548

551-
/**
552-
* To compile a generator/coroutine function f, we build a separate root node f' that executes
553-
* the body of f. We compile f itself to code that creates and returns a generator/coroutine
554-
* that uses f'.
555-
*
556-
* All of the locals are allocated and stored in f'. Since the generator/coroutine is created
557-
* using f's argument array, arguments are transparently forwarded to f' (there's no need for
558-
* closures).
559-
*/
560-
private enum GeneratorOrCoroutineKind {
561-
NONE,
562-
GENERATOR,
563-
COROUTINE,
564-
ASYNC_GENERATOR;
565-
566-
static GeneratorOrCoroutineKind getKind(Scope scope) {
567-
if (scope.isGenerator() && scope.isCoroutine()) {
568-
return ASYNC_GENERATOR;
569-
} else if (scope.isGenerator()) {
570-
return GENERATOR;
571-
} else if (scope.isCoroutine()) {
572-
return COROUTINE;
573-
} else {
574-
return NONE;
575-
}
576-
}
577-
}
578-
579549
@Override
580550
public BytecodeDSLCompilerResult visit(StmtTy.FunctionDef node) {
581551
return compileRootNode(node.name, ArgumentInfo.fromArguments(node.args), node.getSourceRange(),

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -392,8 +392,17 @@ protected Source getSource() {
392392

393393
@TruffleBoundary
394394
public int bciToLine(int bci, BytecodeNode bytecodeNode) {
395-
SourceSection sourceSection = null;
395+
return getSourceSectionForLocation(bci, bytecodeNode).getStartLine();
396+
}
397+
398+
@TruffleBoundary
399+
public SourceSection getSourceSectionForLocation(BytecodeLocation location) {
400+
return getSourceSectionForLocation(location.getBytecodeIndex(), location.getBytecodeNode());
401+
}
396402

403+
@TruffleBoundary
404+
public SourceSection getSourceSectionForLocation(int bci, BytecodeNode bytecodeNode) {
405+
SourceSection sourceSection = null;
397406
if (bytecodeNode != null) {
398407
BytecodeLocation bytecodeLocation = bytecodeNode.getBytecodeLocation(bci);
399408
if (bytecodeLocation != null) {
@@ -409,7 +418,8 @@ public int bciToLine(int bci, BytecodeNode bytecodeNode) {
409418
*/
410419
sourceSection = getSourceSection();
411420
}
412-
return sourceSection.getStartLine();
421+
422+
return sourceSection;
413423
}
414424

415425
@Override

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/exception/PException.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,10 @@ public int getCatchBci() {
199199
return catchBci;
200200
}
201201

202+
public BytecodeNode getBytecodeNode() {
203+
return bytecodeNode;
204+
}
205+
202206
public int getCatchLine() {
203207
if (rootNode == null) {
204208
return -1;

0 commit comments

Comments
 (0)