8353298: AOT cache creation asserts with _array_klasses in an unregistered InstanceKlass

Reviewed-by: ccheung, kvn

Author: iklam

src/hotspot/share/oops/klass.cpp

@@ -830,8 +830,14 @@ void Klass::remove_java_mirror() {
     oop orig_mirror = src_k->java_mirror();
     if (orig_mirror == nullptr) {
       assert(CDSConfig::is_dumping_final_static_archive(), "sanity");
-      assert(is_instance_klass(), "sanity");
-      assert(InstanceKlass::cast(this)->is_shared_unregistered_class(), "sanity");
+      if (is_instance_klass()) {
+        assert(InstanceKlass::cast(this)->is_shared_unregistered_class(), "sanity");
+      } else {
+        precond(is_objArray_klass());
+        Klass *k = ObjArrayKlass::cast(this)->bottom_klass();
+        precond(k->is_instance_klass());
+        assert(InstanceKlass::cast(k)->is_shared_unregistered_class(), "sanity");
+      }
     } else {
       oop scratch_mirror = HeapShared::scratch_java_mirror(orig_mirror);
       if (scratch_mirror != nullptr) {

test/hotspot/jtreg/runtime/cds/appcds/aotCache/AOTCacheSupportForCustomLoaders.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @summary Test AOT cache support for array classes in custom class loaders.
+ * @bug 8353298
+ * @requires vm.cds.supports.aot.class.linking
+ * @comment work around JDK-8345635
+ * @requires !vm.jvmci.enabled
+ * @library /test/lib
+ * @build AOTCacheSupportForCustomLoaders
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar app.jar AppWithCustomLoaders AppWithCustomLoaders$MyLoader
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar cust.jar AppWithCustomLoaders$MyLoadeeA AppWithCustomLoaders$MyLoadeeB
+ * @run driver AOTCacheSupportForCustomLoaders AOT
+ */
+
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.io.File;
+import jdk.test.lib.cds.SimpleCDSAppTester;
+import jdk.test.lib.process.OutputAnalyzer;
+
+public class AOTCacheSupportForCustomLoaders {
+    public static void main(String... args) throws Exception {
+        SimpleCDSAppTester.of("AOTCacheSupportForCustomLoaders")
+            .classpath("app.jar")
+            .addVmArgs("-Xlog:cds+class=debug", "-Xlog:cds")
+            .appCommandLine("AppWithCustomLoaders")
+            .setAssemblyChecker((OutputAnalyzer out) -> {
+                    out.shouldMatch("cds,class.*unreg AppWithCustomLoaders[$]MyLoadeeA")
+                       .shouldMatch("cds,class.*array \\[LAppWithCustomLoaders[$]MyLoadeeA;");
+                })
+            .setProductionChecker((OutputAnalyzer out) -> {
+                    out.shouldContain("Using AOT-linked classes: true");
+                })
+            .runAOTWorkflow();
+    }
+}
+
+class AppWithCustomLoaders {
+    public static void main(String args[]) throws Exception {
+        File custJar = new File("cust.jar");
+        URL[] urls = new URL[] {custJar.toURI().toURL()};
+        MyLoader loader = new MyLoader(urls, AppWithCustomLoaders.class.getClassLoader());
+
+        // Test 1: array class of MyLoadeeA (JDK-8353298)
+        Class klass = loader.loadClass("AppWithCustomLoaders$MyLoadeeA");
+        klass.newInstance();
+
+        // TODO: more test cases JDK-8354557
+    }
+
+    public static class MyLoader extends URLClassLoader {
+        public MyLoader(URL[] urls, ClassLoader parent) {
+            super(urls, parent);
+        }
+    }
+
+    public static class MyLoadeeA {
+        static Object[] array1;
+
+        public MyLoadeeA() {
+            if (array1 == null) {
+                test();
+                Object o = array1[0];
+                System.out.println("array1[0] is of class: " + o.getClass());
+                if (!(o instanceof MyLoadeeA)) {
+                    throw new RuntimeException("array1[0] should be an instanceof MyLoadeeA");
+                }
+            }
+        }
+
+        static void test() {
+            array1 = new MyLoadeeA[10];
+            for (int i = 0; i < 10; i++) {
+                if ((i % 2) == 0) {
+                    array1[i] = new MyLoadeeB();
+                } else {
+                    array1[i] = new MyLoadeeA();
+                }
+            }
+        }
+    }
+
+    public static class MyLoadeeB extends MyLoadeeA {}
+}