[GR-42711] Make dir(foreign_object) return both foreign methods and Python methods

eregon · eregon · commit 99c839b02c98 · 2025-02-19T16:13:32.000Z
PullRequest: graalpython/3664
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,9 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 25.0.0
+* `dir(foreign_object)` now returns both foreign methods and Python methods (it used to return only foreign methods).
+
 ## Version 24.2.0
 * Updated developer metadata of Maven artifacts.
 * Added gradle plugin for polyglot embedding of Python packages into Java.
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py
@@ -771,6 +771,14 @@ def test_java_array(self):
 
         assert il == [1, 2, 3] # unchanged
 
+    def test_dir(self):
+        from java.util import ArrayList
+
+        l = ArrayList()
+        self.assertIn('addAll', dir(l)) # a Java method
+        self.assertIn('extend', dir(l)) # a Python method
+        self.assertEqual(sorted(dir(l)), dir(l))
+
     def test_java_list(self):
         from java.util import ArrayList
 
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java
@@ -312,7 +312,7 @@ private static boolean mimeTypesComplete(ArrayList<String> mimeJavaStrings) {
 
     public static final TruffleString[] T_DEFAULT_PYTHON_EXTENSIONS = new TruffleString[]{T_PY_EXTENSION, tsLiteral(".pyc")};
 
-    private static final TruffleLogger LOGGER = TruffleLogger.getLogger(ID, PythonLanguage.class);
+    public static final TruffleLogger LOGGER = TruffleLogger.getLogger(ID, PythonLanguage.class);
 
     private static final LanguageReference<PythonLanguage> REFERENCE = LanguageReference.create(PythonLanguage.class);
 
@@ -671,6 +671,9 @@ public RootCallTarget parse(PythonContext context, Source source, InputType type
                     EnumSet<FutureFeature> futureFeatures) {
         RaisePythonExceptionErrorCallback errorCb = new RaisePythonExceptionErrorCallback(source, PythonOptions.isPExceptionWithJavaStacktrace(this));
         try {
+            if (context.getEnv().getOptions().get(PythonOptions.ParserLogFiles)) {
+                LOGGER.log(Level.FINE, () -> "parse '" + source.getName() + "'");
+            }
             Parser parser = Compiler.createParser(source.getCharacters().toString(), errorCb, type, interactiveTerminal);
             ModTy mod = (ModTy) parser.parse();
             assert mod != null;
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java
@@ -1318,6 +1318,8 @@ private void loadFile(TruffleString s, TruffleString prefix, PythonModule mod) {
             LOGGER.log(Level.FINE, () -> "import '" + s + "' # <frozen>");
             return;
         }
+
+        LOGGER.log(Level.FINE, () -> "import '" + s + "'");
         Supplier<CallTarget> getCode = () -> {
             Source source = getInternalSource(s, prefix);
             return getLanguage().parse(getContext(), source, InputType.FILE, false, 0, false, null, EnumSet.noneOf(FutureFeature.class));
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java
@@ -109,6 +109,7 @@
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.logging.Level;
 
 import com.oracle.graal.python.PythonFileDetector;
 import com.oracle.graal.python.PythonLanguage;
@@ -1079,6 +1080,9 @@ Object compile(TruffleString expression, TruffleString filename, TruffleString m
                 if (featureVersion < 7) {
                     compilerFlags.add(AbstractParser.Flags.ASYNC_HACKS);
                 }
+                if (context.getEnv().getOptions().get(PythonOptions.ParserLogFiles)) {
+                    PythonLanguage.LOGGER.log(Level.FINE, () -> "parse '" + source.getName() + "'");
+                }
                 Parser parser = Compiler.createParser(code.toJavaStringUncached(), errorCb, type, compilerFlags, featureVersion);
                 ModTy mod = (ModTy) parser.parse();
                 errorCb.triggerDeprecationWarnings();
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java
@@ -46,15 +46,21 @@
 import com.oracle.graal.python.builtins.PythonBuiltins;
 import com.oracle.graal.python.builtins.objects.PNone;
 import com.oracle.graal.python.builtins.objects.PythonAbstractObject;
+import com.oracle.graal.python.builtins.objects.list.ListBuiltins;
+import com.oracle.graal.python.builtins.objects.list.PList;
 import com.oracle.graal.python.builtins.objects.object.ObjectBuiltins;
 import com.oracle.graal.python.builtins.objects.object.ObjectNodes;
+import com.oracle.graal.python.builtins.objects.set.PSet;
+import com.oracle.graal.python.builtins.objects.set.SetNodes;
 import com.oracle.graal.python.builtins.objects.type.TpSlots;
+import com.oracle.graal.python.builtins.objects.type.TypeBuiltins;
 import com.oracle.graal.python.builtins.objects.type.slots.TpSlotGetAttr.GetAttrBuiltinNode;
 import com.oracle.graal.python.builtins.objects.type.slots.TpSlotSetAttr.SetAttrBuiltinNode;
 import com.oracle.graal.python.nodes.ErrorMessages;
 import com.oracle.graal.python.nodes.PGuards;
 import com.oracle.graal.python.nodes.PRaiseNode;
 import com.oracle.graal.python.nodes.attributes.LookupAttributeInMRONode;
+import com.oracle.graal.python.nodes.builtins.ListNodes;
 import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
@@ -68,7 +74,6 @@
 import com.oracle.graal.python.runtime.PythonOptions;
 import com.oracle.graal.python.runtime.exception.PException;
 import com.oracle.graal.python.runtime.exception.PythonErrorType;
-import com.oracle.graal.python.runtime.object.PFactory;
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Bind;
@@ -83,6 +88,7 @@
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.interop.ArityException;
 import com.oracle.truffle.api.interop.InteropLibrary;
+import com.oracle.truffle.api.interop.InvalidArrayIndexException;
 import com.oracle.truffle.api.interop.UnknownIdentifierException;
 import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.interop.UnsupportedTypeException;
@@ -268,29 +274,54 @@ static void doDelete(Object object, Object key, @SuppressWarnings("unused") PNon
         }
     }
 
-    // TODO dir(foreign) should list both foreign object members and attributes from class
     @Builtin(name = J___DIR__, minNumOfPositionalArgs = 1)
     @GenerateNodeFactory
     abstract static class DirNode extends PythonUnaryBuiltinNode {
         @Specialization
-        protected Object doIt(Object object,
+        protected Object doIt(VirtualFrame frame, Object object,
                         @Bind("this") Node inliningTarget,
                         @CachedLibrary(limit = "3") InteropLibrary lib,
+                        @CachedLibrary(limit = "3") InteropLibrary arrayInterop,
+                        @CachedLibrary(limit = "3") InteropLibrary stringInterop,
+                        @Cached TruffleString.SwitchEncodingNode switchEncodingNode,
                         @Cached GilNode gil,
-                        @Cached InlinedConditionProfile profile) {
+                        @Cached InlinedConditionProfile profile,
+                        @Cached GetClassNode getClassNode,
+                        @Cached TypeBuiltins.DirNode typeDirNode,
+                        @Cached SetNodes.AddNode addNode,
+                        @Cached(inline = false) ListBuiltins.ListSortNode sortNode,
+                        @Cached(inline = false) ListNodes.ConstructListNode constructListNode) {
+            // Inspired by ObjectBuiltins.DirNode
+            var pythonClass = getClassNode.execute(inliningTarget, object);
+            PSet attributes = typeDirNode.execute(frame, pythonClass);
+
             if (profile.profile(inliningTarget, lib.hasMembers(object))) {
+                final Object members;
                 gil.release(true);
                 try {
-                    return lib.getMembers(object);
+                    members = lib.getMembers(object);
                 } catch (UnsupportedMessageException e) {
-                    CompilerDirectives.transferToInterpreterAndInvalidate();
-                    throw new IllegalStateException("foreign object claims to have members, but does not return them");
+                    throw CompilerDirectives.shouldNotReachHere("foreign object claims to have members, but does not return them");
                 } finally {
                     gil.acquire();
                 }
-            } else {
-                return PFactory.createList(PythonLanguage.get(inliningTarget));
+
+                try {
+                    long size = arrayInterop.getArraySize(members);
+                    for (int i = 0; i < size; i++) {
+                        TruffleString memberString = stringInterop.asTruffleString(arrayInterop.readArrayElement(members, i));
+                        memberString = switchEncodingNode.execute(memberString, TS_ENCODING);
+                        addNode.execute(frame, attributes, memberString);
+                    }
+                } catch (UnsupportedMessageException | InvalidArrayIndexException e) {
+                    throw CompilerDirectives.shouldNotReachHere(e);
+                }
             }
+
+            // set to sorted list, like in PyObjectDir
+            PList list = constructListNode.execute(frame, attributes);
+            sortNode.execute(frame, list);
+            return list;
         }
     }
 
diff --git a/graalpython/lib-graalpython/_sre.py b/graalpython/lib-graalpython/_sre.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -44,6 +44,9 @@
 
 from sys import maxsize
 
+# This does not load _polyglot.py (as desired for faster startup), due to AbstractImportNode.importModule(),
+# when the context is not initialized, only looking up builtin modules (and not running their postInitialize()).
+import polyglot
 
 def _check_pos(pos):
     if pos > maxsize:
@@ -265,7 +268,7 @@ def __init__(self, pattern, flags):
             self.groupindex = {}
             self.__indexgroup = {}
         else:
-            group_names = dir(groups)
+            group_names = polyglot.__keys__(groups)
             self.groupindex = _mappingproxy({name: getattr(groups, name) for name in group_names})
             self.__indexgroup = {getattr(groups, name): name for name in group_names}
 

Original file line number	Diff line number	Diff line change
`@@ -1318,6 +1318,8 @@ private void loadFile(TruffleString s, TruffleString prefix, PythonModule mod) {`
`1318`	`1318`	`LOGGER.log(Level.FINE, () -> "import '" + s + "' # <frozen>");`
`1319`	`1319`	`return;`
`1320`	`1320`	`}`
	`1321`	`+`
	`1322`	`+ LOGGER.log(Level.FINE, () -> "import '" + s + "'");`
`1321`	`1323`	`Supplier<CallTarget> getCode = () -> {`
`1322`	`1324`	`Source source = getInternalSource(s, prefix);`
`1323`	`1325`	`return getLanguage().parse(getContext(), source, InputType.FILE, false, 0, false, null, EnumSet.noneOf(FutureFeature.class));`