8380409: JVM crashes when -XX:AOTMode=create uses app.aotconf generated with JVMTI agent

iklam · iklam · commit 30401d86ef91 · 2026-04-16T05:17:46.000Z
Backport-of: 1e3fb44
diff --git a/src/hotspot/share/cds/aotArtifactFinder.cpp b/src/hotspot/share/cds/aotArtifactFinder.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
 #include "cds/lambdaProxyClassDictionary.hpp"
 #include "cds/regeneratedClasses.hpp"
 #include "classfile/systemDictionaryShared.hpp"
+#include "classfile/vmClasses.hpp"
 #include "logging/log.hpp"
 #include "memory/metaspaceClosure.hpp"
 #include "oops/instanceKlass.hpp"
@@ -169,6 +170,7 @@ void AOTArtifactFinder::find_artifacts() {
   end_scanning_for_oops();
 
   TrainingData::cleanup_training_data();
+  check_critical_classes();
 }
 
 void AOTArtifactFinder::start_scanning_for_oops() {
@@ -233,6 +235,7 @@ void AOTArtifactFinder::add_cached_instance_class(InstanceKlass* ik) {
   bool created;
   _seen_classes->put_if_absent(ik, &created);
   if (created) {
+    check_critical_class(ik);
     append_to_all_cached_classes(ik);
 
     // All super types must be added.
@@ -310,3 +313,25 @@ void AOTArtifactFinder::all_cached_classes_do(MetaspaceClosure* it) {
     it->push(_all_cached_classes->adr_at(i));
   }
 }
+
+void AOTArtifactFinder::check_critical_classes() {
+  if (CDSConfig::is_dumping_static_archive()) {
+    // vmClasses are store in the AOT cache (or AOT config file, or static archive).
+    // If any of the vmClasses is excluded, (usually due to incompatible JVMTI agent),
+    // the resulting cache/config/archive is unusable.
+    for (auto id : EnumRange<vmClassID>{}) {
+      check_critical_class(vmClasses::klass_at(id));
+    }
+  }
+}
+
+void AOTArtifactFinder::check_critical_class(InstanceKlass* ik) {
+  if (SystemDictionaryShared::is_excluded_class(ik)) {
+    ResourceMark rm;
+    const char* msg = err_msg("Critical class %s has been excluded. %s cannot be written.",
+                              ik->external_name(),
+                              CDSConfig::type_of_archive_being_written());
+    AOTMetaspace::unrecoverable_writing_error(msg);
+  }
+}
+
diff --git a/src/hotspot/share/cds/aotArtifactFinder.hpp b/src/hotspot/share/cds/aotArtifactFinder.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -81,12 +81,14 @@ class AOTArtifactFinder : AllStatic {
   static void add_cached_type_array_class(TypeArrayKlass* tak);
   static void add_cached_instance_class(InstanceKlass* ik);
   static void append_to_all_cached_classes(Klass* k);
+  static void check_critical_class(InstanceKlass* ik);
 public:
   static void initialize();
   static void find_artifacts();
   static void add_cached_class(Klass* k);
   static void add_aot_inited_class(InstanceKlass* ik);
   static void all_cached_classes_do(MetaspaceClosure* it);
+  static void check_critical_classes();
   static void dispose();
 };
 
diff --git a/src/hotspot/share/cds/heapShared.cpp b/src/hotspot/share/cds/heapShared.cpp
@@ -502,6 +502,7 @@ bool HeapShared::archive_object(oop obj, oop referrer, KlassSubGraphInfo* subgra
     return false;
   }
 
+  AOTArtifactFinder::add_cached_class(obj->klass());
   AOTOopChecker::check(obj); // Make sure contents of this oop are safe.
   count_allocation(obj->size());
 
diff --git a/test/hotspot/jtreg/runtime/cds/appcds/aotCache/RedefineCriticalClasses.java b/test/hotspot/jtreg/runtime/cds/appcds/aotCache/RedefineCriticalClasses.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+
+/*
+ * @test
+ * @summary AOT training run should fail if critical classes have been redefined by JVMTI
+ * @bug 8380409
+ * @requires vm.cds.supports.aot.class.linking
+ * @library /test/lib
+ * @run driver RedefineClassHelper
+ * @build RedefineCriticalClasses
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar app.jar RedefineCriticalClassesApp
+ * @run driver RedefineCriticalClasses
+ */
+
+import java.io.InputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import jdk.test.lib.cds.CDSTestUtils;
+import jdk.test.lib.helpers.ClassFileInstaller;
+import jdk.test.lib.process.ProcessTools;
+
+public class RedefineCriticalClasses {
+    public static void main(String... args) throws Exception {
+        ArrayList<String> processArgs = new ArrayList<>();
+
+        // redefineagent.jar is created by "@run driver RedefineClassHelper"
+        processArgs.add("-javaagent:redefineagent.jar");
+
+        processArgs.add("-XX:AOTMode=record");
+        processArgs.add("-XX:AOTConfiguration=app.aotconfig");
+        processArgs.add("-Xlog:aot,cds");
+        processArgs.add("-cp");
+        processArgs.add("app.jar");
+        processArgs.add("RedefineCriticalClassesApp");
+
+        ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder(processArgs);
+        CDSTestUtils.executeAndLog(pb, "train")
+            .shouldContain("Redefined: class java.lang.Class")
+            .shouldContain("Skipping java/lang/Class: Has been redefined")
+            .shouldContain("Critical class java.lang.Class has been excluded. AOT configuration file cannot be written")
+            .shouldHaveExitValue(1);
+    }
+}
+
+class RedefineCriticalClassesApp {
+    public static void main(String[] args) throws Exception {
+        // Use RedefineClassHelper (loaded from redefineagent.jar into the boot class loader)
+        // to redefine java/lang/Class, using the exact same bytecodes as from the JDK.
+        // The JVM will mark it as having been redefined by JVMTI and will exclude it from the
+        // AOT configuration file.
+
+        URL url = new URL("jrt:/java.base/java/lang/Class.class");
+        try (InputStream in = url.openConnection().getInputStream()) {
+            byte[] b = in.readAllBytes();
+            System.out.println("Length = " + b.length);
+            RedefineClassHelper.redefineClass(Class.class, b);
+            System.out.println("Redefined: " + Class.class);
+        }
+    }
+}
diff --git a/test/hotspot/jtreg/runtime/cds/appcds/aotCache/TransformCriticalClasses.java b/test/hotspot/jtreg/runtime/cds/appcds/aotCache/TransformCriticalClasses.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+
+/*
+ * @test
+ * @summary AOT training run should fail if critical classes have been transformed by JVMTI
+ *          with ClassFileLoadHook
+ * @bug 8380409
+ * @requires vm.cds.supports.aot.class.linking
+ * @library /test/lib
+ * @build TransformCriticalClasses
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar app.jar TransformCriticalClassesApp
+ * @run main/othervm/native TransformCriticalClasses
+ */
+
+import java.util.ArrayList;
+import jdk.test.lib.cds.CDSTestUtils;
+import jdk.test.lib.helpers.ClassFileInstaller;
+import jdk.test.lib.process.ProcessTools;
+
+public class TransformCriticalClasses {
+    public static void main(String... args) throws Exception {
+        ArrayList<String> processArgs = new ArrayList<>();
+
+        // Tell the native agent SimpleClassFileLoadHook to do an dummy transformation
+        // of java/lang/Class. This class will be defined using the exact same bytecodes
+        // as from the JDK, but the JVM will mark it as having been transformed by JVMTI
+        // and will exclude it from the AOT configuration file.
+        processArgs.add("-agentlib:SimpleClassFileLoadHook=-early,java/lang/Class,xxxxxx,xxxxxx");
+
+        processArgs.add("-XX:AOTMode=record");
+        processArgs.add("-XX:AOTConfiguration=app.aotconfig");
+        processArgs.add("-Xlog:aot,cds");
+        processArgs.add("-cp");
+        processArgs.add("app.jar");
+        processArgs.add("TransformCriticalClassesApp");
+
+        ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder(processArgs);
+        CDSTestUtils.executeAndLog(pb, "train")
+            .shouldContain("Skipping java/lang/Class: From ClassFileLoadHook")
+            .shouldContain("Critical class java.lang.Class has been excluded. AOT configuration file cannot be written")
+            .shouldHaveExitValue(1);
+    }
+}
+
+class TransformCriticalClassesApp {
+    public static void main(String[] args) {
+        System.out.println("HelloWorld");
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -502,6 +502,7 @@ bool HeapShared::archive_object(oop obj, oop referrer, KlassSubGraphInfo* subgra`
`502`	`502`	`return false;`
`503`	`503`	`}`
`504`	`504`
	`505`	`+ AOTArtifactFinder::add_cached_class(obj->klass());`
`505`	`506`	`AOTOopChecker::check(obj); // Make sure contents of this oop are safe.`
`506`	`507`	`count_allocation(obj->size());`
`507`	`508`