[GR-33829] Add support for the PE encoded graph cache on libgraal.

PullRequest: graal/11882

Author: mukel

compiler/src/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/PEGraphDecoderTest.java

@@ -188,7 +188,7 @@ private StructuredGraph test(String methodName, EconomicMap<ResolvedJavaMethod,
             registerPlugins(graphBuilderConfig.getPlugins().getInvocationPlugins());
             targetGraph = new StructuredGraph.Builder(debug.getOptions(), debug, AllowAssumptions.YES).method(testMethod).build();
             CachingPEGraphDecoder decoder = new CachingPEGraphDecoder(getTarget().arch, targetGraph, getProviders(), graphBuilderConfig, OptimisticOptimizations.NONE, AllowAssumptions.YES,
-                            null, null, new InlineInvokePlugin[]{new InlineAll()}, null, null, null, null, null, graphCache, false);
+                            null, null, new InlineInvokePlugin[]{new InlineAll()}, null, null, null, null, null, graphCache, () -> null, false);
 
             decoder.decode(testMethod, false, false);
             debug.dump(DebugContext.BASIC_LEVEL, targetGraph, "Target Graph");

compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java

@@ -27,6 +27,7 @@
 import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE;
 
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
 
 import org.graalvm.collections.EconomicMap;
 import org.graalvm.compiler.bytecode.BytecodeProvider;
@@ -57,7 +58,9 @@
 import org.graalvm.compiler.phases.util.Providers;
 
 import jdk.vm.ci.code.Architecture;
+import jdk.vm.ci.meta.Assumptions;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
+import jdk.vm.ci.meta.ResolvedJavaType;
 
 /**
  * A graph decoder that provides all necessary encoded graphs on-the-fly (by parsing the methods and
@@ -71,14 +74,17 @@ public class CachingPEGraphDecoder extends PEGraphDecoder {
     protected final GraphBuilderConfiguration graphBuilderConfig;
     protected final OptimisticOptimizations optimisticOpts;
     private final AllowAssumptions allowAssumptions;
-    private final EconomicMap<ResolvedJavaMethod, EncodedGraph> graphCache;
+    private final EconomicMap<ResolvedJavaMethod, EncodedGraph> persistentGraphCache;
+    private final EconomicMap<ResolvedJavaMethod, EncodedGraph> localGraphCache;
+    private final Supplier<AutoCloseable> createPersistentCachedGraphScope;
     private final BasePhase<? super CoreProviders> postParsingPhase;
 
     public CachingPEGraphDecoder(Architecture architecture, StructuredGraph graph, Providers providers, GraphBuilderConfiguration graphBuilderConfig, OptimisticOptimizations optimisticOpts,
                     AllowAssumptions allowAssumptions, LoopExplosionPlugin loopExplosionPlugin, InvocationPlugins invocationPlugins, InlineInvokePlugin[] inlineInvokePlugins,
                     ParameterPlugin parameterPlugin,
                     NodePlugin[] nodePlugins, ResolvedJavaMethod peRootForInlining, SourceLanguagePositionProvider sourceLanguagePositionProvider,
-                    BasePhase<? super CoreProviders> postParsingPhase, EconomicMap<ResolvedJavaMethod, EncodedGraph> graphCache, boolean needsExplicitException) {
+                    BasePhase<? super CoreProviders> postParsingPhase, EconomicMap<ResolvedJavaMethod, EncodedGraph> persistentGraphCache, Supplier<AutoCloseable> createPersistentCachedGraphScope,
+                    boolean needsExplicitException) {
         super(architecture, graph, providers, loopExplosionPlugin,
                         invocationPlugins, inlineInvokePlugins, parameterPlugin, nodePlugins, peRootForInlining, sourceLanguagePositionProvider,
                         new ConcurrentHashMap<>(), new ConcurrentHashMap<>(), needsExplicitException);
@@ -87,8 +93,10 @@ public CachingPEGraphDecoder(Architecture architecture, StructuredGraph graph, P
         this.graphBuilderConfig = graphBuilderConfig;
         this.optimisticOpts = optimisticOpts;
         this.allowAssumptions = allowAssumptions;
-        this.graphCache = graphCache;
         this.postParsingPhase = postParsingPhase;
+        this.persistentGraphCache = persistentGraphCache;
+        this.createPersistentCachedGraphScope = createPersistentCachedGraphScope;
+        this.localGraphCache = EconomicMap.create();
     }
 
     protected GraphBuilderPhase.Instance createGraphBuilderPhaseInstance(IntrinsicContext initialIntrinsicContext) {
@@ -120,7 +128,7 @@ private EncodedGraph createGraph(ResolvedJavaMethod method, BytecodeProvider int
         }
 
         EncodedGraph encodedGraph = GraphEncoder.encodeSingleGraph(graphToEncode, architecture);
-        graphCache.put(method, encodedGraph);
+        persistentGraphCache.put(method, encodedGraph);
         return encodedGraph;
     }
 
@@ -152,13 +160,114 @@ private StructuredGraph buildGraph(ResolvedJavaMethod method, BytecodeProvider i
         return graphToEncode;
     }
 
+    private static boolean verifyAssumptions(EncodedGraph graph) {
+        Assumptions assumptions = graph.getAssumptions();
+        if (assumptions == null || assumptions.isEmpty()) {
+            return true; // verified
+        }
+        for (Assumptions.Assumption assumption : assumptions) {
+            if (assumption instanceof Assumptions.LeafType) {
+                Assumptions.LeafType leafType = (Assumptions.LeafType) assumption;
+                /*
+                 * LeafType cannot be fully verified because the assumption doesn't imply that the
+                 * type is (also) concrete. We check a common case (leaf + concrete type).
+                 */
+                Assumptions.AssumptionResult<ResolvedJavaType> assumptionResult = leafType.context.findLeafConcreteSubtype();
+                if (assumptionResult != null) {
+                    ResolvedJavaType candidate = assumptionResult.getResult();
+                    if (!leafType.context.equals(candidate)) {
+                        return false;
+                    }
+                }
+            } else if (assumption instanceof Assumptions.ConcreteSubtype) {
+                Assumptions.ConcreteSubtype concreteSubtype = (Assumptions.ConcreteSubtype) assumption;
+                /*
+                 * ConcreteSubtype cannot be fully verified because the assumption doesn't imply
+                 * that the concrete subtype is (also) a leaf. We check a common case (leaf +
+                 * concrete type).
+                 */
+                Assumptions.AssumptionResult<ResolvedJavaType> assumptionResult = concreteSubtype.context.findLeafConcreteSubtype();
+                if (assumptionResult != null) {
+                    ResolvedJavaType candidate = assumptionResult.getResult();
+                    /*
+                     * No equality check here because the ConcreteSubtype assumption allows
+                     * non-concrete subtypes for interfaces.
+                     */
+                    if (!concreteSubtype.subtype.isAssignableFrom(candidate)) {
+                        return false;
+                    }
+                }
+            } else if (assumption instanceof Assumptions.ConcreteMethod) {
+                Assumptions.ConcreteMethod concreteMethod = (Assumptions.ConcreteMethod) assumption;
+                /*
+                 * ConcreteMethod is the only assumption that can be verified since it matches
+                 * findUniqueConcreteMethod semantics. If the assumption cannot be retrieved
+                 * (findUniqueConcreteMethod returns null) then it was invalidated.
+                 */
+                Assumptions.AssumptionResult<ResolvedJavaMethod> assumptionResult = concreteMethod.context.findUniqueConcreteMethod(concreteMethod.method);
+                if (assumptionResult == null || !concreteMethod.impl.equals(assumptionResult.getResult())) {
+                    return false;
+                }
+            } else if (assumption instanceof Assumptions.NoFinalizableSubclass || assumption instanceof Assumptions.CallSiteTargetValue ||
+                            "org.graalvm.compiler.truffle.compiler.nodes.TruffleAssumption".equals(assumption.getClass().getName())) {
+                /*
+                 * These assumptions cannot be (even partially) verified. The cached graph will be
+                 * invalidated on code installation.
+                 */
+            } else {
+                throw GraalError.shouldNotReachHere("unexpected assumption " + assumption);
+            }
+        }
+        return true;
+    }
+
+    @SuppressWarnings({"unused", "try"})
+    private EncodedGraph lookupOrCreatePersistentEncodedGraph(ResolvedJavaMethod method, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution,
+                    boolean trackNodeSourcePosition) {
+        EncodedGraph result = persistentGraphCache.get(method);
+        if (result == null && method.hasBytecodes()) {
+            try (AutoCloseable scope = createPersistentCachedGraphScope.get()) {
+                // Encoded graphs that can be cached across compilations are wrapped by "scopes"
+                // provided by createCachedGraphScope.
+                result = createGraph(method, intrinsicBytecodeProvider, isSubstitution);
+            } catch (Throwable ex) {
+                throw debug.handle(ex);
+            }
+        }
+        return result;
+    }
+
     @Override
     protected EncodedGraph lookupEncodedGraph(ResolvedJavaMethod method, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution,
                     boolean trackNodeSourcePosition) {
-        EncodedGraph result = graphCache.get(method);
-        if (result == null && method.hasBytecodes()) {
-            result = createGraph(method, intrinsicBytecodeProvider, isSubstitution);
+        // Graph's assumptions are fresh or validated recently.
+        EncodedGraph result = localGraphCache.get(method);
+        if (result != null) {
+            return result;
         }
+
+        result = persistentGraphCache.get(method);
+        if (result == null) {
+            // Embedded assumptions in a fresh encoded graph should be up-to-date, so no need to
+            // validate them.
+            result = lookupOrCreatePersistentEncodedGraph(method, intrinsicBytecodeProvider, isSubstitution, trackNodeSourcePosition);
+            if (result != null) {
+                localGraphCache.put(method, result);
+            }
+        } else if (!verifyAssumptions(result)) {
+            // Cached graph has invalid assumptions, drop from persistent cache and re-parse.
+            persistentGraphCache.removeKey(method);
+            // Embedded assumptions in a fresh encoded graph should be up-to-date, so no need to
+            // validate them.
+            result = lookupOrCreatePersistentEncodedGraph(method, intrinsicBytecodeProvider, isSubstitution, trackNodeSourcePosition);
+            if (result != null) {
+                localGraphCache.put(method, result);
+            }
+        } else {
+            // Assumptions validated, avoid further checks.
+            localGraphCache.put(method, result);
+        }
+
         return result;
     }
 }

compiler/src/org.graalvm.compiler.truffle.common.hotspot.libgraal/src/org/graalvm/compiler/truffle/common/hotspot/libgraal/TruffleToLibGraal.java

@@ -82,6 +82,7 @@ enum Id {
         OpenDebugContext,
         OpenDebugContextScope,
         PendingTransferToInterpreterOffset,
+        PurgePartialEvaluationCaches,
         Shutdown;
     }
 }

compiler/src/org.graalvm.compiler.truffle.common.hotspot/src/org/graalvm/compiler/truffle/common/hotspot/HotSpotTruffleCompiler.java

@@ -48,7 +48,5 @@ public interface HotSpotTruffleCompiler extends TruffleCompiler {
     /**
      * Releases caches used for PE/compilation.
      */
-    default void purgeCaches() {
-        // nop
-    }
+    void purgePartialEvaluationCaches();
 }

compiler/src/org.graalvm.compiler.truffle.compiler.hotspot.libgraal/src/org/graalvm/compiler/truffle/compiler/hotspot/libgraal/TruffleToLibGraalEntryPoints.java

@@ -59,6 +59,7 @@
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleToLibGraal.Id.OpenDebugContext;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleToLibGraal.Id.OpenDebugContextScope;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleToLibGraal.Id.PendingTransferToInterpreterOffset;
+import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleToLibGraal.Id.PurgePartialEvaluationCaches;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleToLibGraal.Id.Shutdown;
 import static org.graalvm.jniutils.JNIUtil.GetArrayLength;
 import static org.graalvm.jniutils.JNIUtil.GetByteArrayElements;
@@ -723,6 +724,20 @@ public static void dumpChannelClose(JNIEnv env, JClass hsClass, @CEntryPoint.Iso
         }
     }
 
+    @TruffleToLibGraal(PurgePartialEvaluationCaches)
+    @CEntryPoint(name = "Java_org_graalvm_compiler_truffle_runtime_hotspot_libgraal_TruffleToLibGraalCalls_purgePartialEvaluationCaches")
+    @SuppressWarnings({"unused", "try"})
+    public static void purgePartialEvaluationCaches(JNIEnv env, JClass hsClass, @CEntryPoint.IsolateThreadContext long isolateThreadId, long compilerHandle) {
+        try (JNIMethodScope s = LibGraalUtil.openScope(TruffleToLibGraalEntryPoints.class, PurgePartialEvaluationCaches, env)) {
+            HotSpotTruffleCompilerImpl compiler = LibGraalObjectHandles.resolve(compilerHandle, HotSpotTruffleCompilerImpl.class);
+            if (compiler != null) {
+                compiler.purgePartialEvaluationCaches();
+            }
+        } catch (Throwable t) {
+            JNIExceptionWrapper.throwInHotSpot(env, t);
+        }
+    }
+
     @TruffleToLibGraal(GetExecutionID)
     @CEntryPoint(name = "Java_org_graalvm_compiler_truffle_runtime_hotspot_libgraal_TruffleToLibGraalCalls_getExecutionID")
     @SuppressWarnings({"unused", "try"})

compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotPartialEvaluator.java

@@ -24,19 +24,17 @@
  */
 package org.graalvm.compiler.truffle.compiler.hotspot;
 
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
 
 import org.graalvm.collections.EconomicMap;
+import org.graalvm.compiler.hotspot.HotSpotGraalServices;
 import org.graalvm.compiler.nodes.EncodedGraph;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration;
 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins;
 import org.graalvm.compiler.truffle.compiler.PartialEvaluator;
 import org.graalvm.compiler.truffle.compiler.TruffleCompilerConfiguration;
-import org.graalvm.compiler.truffle.options.PolyglotCompilerOptions;
 import org.graalvm.options.OptionValues;
 
 import jdk.vm.ci.meta.ResolvedJavaMethod;
@@ -45,16 +43,14 @@ public final class HotSpotPartialEvaluator extends PartialEvaluator {
 
     private final AtomicReference<EconomicMap<ResolvedJavaMethod, EncodedGraph>> graphCacheRef;
 
-    public boolean isEncodedGraphCacheEnabled() {
-        return encodedGraphCacheCapacity != 0;
-    }
-
-    private int encodedGraphCacheCapacity;
     private int jvmciReservedReference0Offset = -1;
 
+    private boolean disableEncodedGraphCachePurges;
+
     public HotSpotPartialEvaluator(TruffleCompilerConfiguration config, GraphBuilderConfiguration configForRoot) {
         super(config, configForRoot, new HotSpotKnownTruffleTypes(config.lastTier().providers().getMetaAccess()));
         this.graphCacheRef = new AtomicReference<>();
+        this.disableEncodedGraphCachePurges = false;
     }
 
     void setJvmciReservedReference0Offset(int jvmciReservedReference0Offset) {
@@ -68,7 +64,6 @@ public int getJvmciReservedReference0Offset() {
     @Override
     protected void initialize(OptionValues options) {
         super.initialize(options);
-        encodedGraphCacheCapacity = options.get(PolyglotCompilerOptions.EncodedGraphCacheCapacity);
     }
 
     @Override
@@ -78,42 +73,50 @@ protected void registerGraphBuilderInvocationPlugins(InvocationPlugins invocatio
                         (HotSpotKnownTruffleTypes) getKnownTruffleTypes());
     }
 
-    @SuppressWarnings("serial")
-    private Map<ResolvedJavaMethod, EncodedGraph> createEncodedGraphMap() {
-        if (encodedGraphCacheCapacity < 0) {
-            // Unbounded cache.
-            return new ConcurrentHashMap<>();
-        }
-
-        // Access-based LRU bounded cache. The overhead of the synchronized map is negligible
-        // compared to the cost of re-parsing the graphs.
-        return Collections.synchronizedMap(
-                        new LinkedHashMap<ResolvedJavaMethod, EncodedGraph>(16, 0.75f, true) {
-                            @Override
-                            protected boolean removeEldestEntry(Map.Entry<ResolvedJavaMethod, EncodedGraph> eldest) {
-                                // encodedGraphCacheCapacity < 0 => unbounded capacity
-                                return (encodedGraphCacheCapacity >= 0) && size() > encodedGraphCacheCapacity;
-                            }
-                        });
-    }
-
     @Override
-    public EconomicMap<ResolvedJavaMethod, EncodedGraph> getOrCreateEncodedGraphCache() {
-        if (encodedGraphCacheCapacity == 0) {
+    public EconomicMap<ResolvedJavaMethod, EncodedGraph> getOrCreateEncodedGraphCache(boolean persistentEncodedGraphCache) {
+        if (!persistentEncodedGraphCache) {
             // The encoded graph cache is disabled across different compilations. The returned map
             // can still be used and propagated within the same compilation unit.
-            return super.getOrCreateEncodedGraphCache();
+            return super.getOrCreateEncodedGraphCache(persistentEncodedGraphCache);
         }
         EconomicMap<ResolvedJavaMethod, EncodedGraph> cache;
         do {
             cache = graphCacheRef.get();
         } while (cache == null &&
-                        !graphCacheRef.compareAndSet(null, cache = EconomicMap.wrapMap(createEncodedGraphMap())));
+                        !graphCacheRef.compareAndSet(null, cache = EconomicMap.wrapMap(new ConcurrentHashMap<>())));
         assert cache != null;
         return cache;
     }
 
+    /**
+     * Called in unit-tests via reflection.
+     */
     public void purgeEncodedGraphCache() {
-        graphCacheRef.set(null);
+        // Disabling purges only for tests.
+        if (!disableEncodedGraphCachePurges) {
+            graphCacheRef.set(null);
+        }
+    }
+
+    /**
+     * Used only in unit-tests, to avoid transient failures caused by multiple compiler threads
+     * racing to purge the cache. Called reflectively from EncodedGraphCacheTest.
+     */
+    public boolean disableEncodedGraphCachePurges(boolean value) {
+        boolean oldValue = disableEncodedGraphCachePurges;
+        disableEncodedGraphCachePurges = value;
+        return oldValue;
+    }
+
+    @Override
+    protected Supplier<AutoCloseable> getCreateCachedGraphScope(boolean persistentEncodedGraphCache) {
+        if (persistentEncodedGraphCache) {
+            // The interpreter graphs may be cached across compilations, keep JavaConstants
+            // references to the application heap alive in the libgraal global scope.
+            return HotSpotGraalServices::enterGlobalCompilationContext;
+        } else {
+            return super.getCreateCachedGraphScope(persistentEncodedGraphCache);
+        }
     }
 }