8064701: Some CDS optimizations should be disabled if bootclasspath is modified by JVMTI

Summary: Added API to track bootclasspath modification
Reviewed-by: jiangli, dholmes, minqi

src/share/vm/classfile/classLoaderExt.hpp

@@ -63,6 +63,9 @@ public:
                                    ClassPathEntry* new_entry) {
     ClassLoader::add_to_list(new_entry);
   }
+  static void append_boot_classpath(ClassPathEntry* new_entry) {
+    ClassLoader::add_to_list(new_entry);
+  }
   static void setup_search_paths() {}
 
   static void init_lookup_cache(TRAPS) {}

src/share/vm/prims/jvmtiEnv.cpp

@@ -23,6 +23,7 @@
  */
 
 #include "precompiled.hpp"
+#include "classfile/classLoaderExt.hpp"
 #include "classfile/systemDictionary.hpp"
 #include "classfile/vmSymbols.hpp"
 #include "interpreter/bytecodeStream.hpp"
@@ -475,7 +476,7 @@ JvmtiEnv::AddToBootstrapClassLoaderSearch(const char* segment) {
     if (TraceClassLoading) {
       tty->print_cr("[Opened %s]", zip_entry->name());
     }
-    ClassLoader::add_to_list(zip_entry);
+    ClassLoaderExt::append_boot_classpath(zip_entry);
     return JVMTI_ERROR_NONE;
   } else {
     return JVMTI_ERROR_WRONG_PHASE;

src/share/vm/prims/whitebox.cpp

@@ -56,6 +56,7 @@
 #endif // INCLUDE_NMT
 
 #include "compiler/compileBroker.hpp"
+#include "jvmtifiles/jvmtiEnv.hpp"
 #include "runtime/compilationPolicy.hpp"
 
 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
@@ -126,6 +127,31 @@ WB_ENTRY(jintArray, WB_GetLookupCacheMatches(JNIEnv* env, jobject o, jobject loa
   return result;
 WB_END
 
+WB_ENTRY(void, WB_AddToBootstrapClassLoaderSearch(JNIEnv* env, jobject o, jstring segment)) {
+#if INCLUDE_JVMTI
+  ThreadToNativeFromVM ttnfv(thread);   // can't be in VM when we call JNI
+  const char* seg = env->GetStringUTFChars(segment, NULL);
+  JvmtiEnv* jvmti_env = JvmtiEnv::create_a_jvmti(JVMTI_VERSION);
+  jvmtiError err = jvmti_env->AddToBootstrapClassLoaderSearch(seg);
+  assert(err == JVMTI_ERROR_NONE, "must not fail");
+  env->ReleaseStringUTFChars(segment, seg);
+#endif
+}
+WB_END
+
+WB_ENTRY(void, WB_AddToSystemClassLoaderSearch(JNIEnv* env, jobject o, jstring segment)) {
+#if INCLUDE_JVMTI
+  ThreadToNativeFromVM ttnfv(thread);   // can't be in VM when we call JNI
+  const char* seg = env->GetStringUTFChars(segment, NULL);
+  JvmtiEnv* jvmti_env = JvmtiEnv::create_a_jvmti(JVMTI_VERSION);
+  jvmtiError err = jvmti_env->AddToSystemClassLoaderSearch(seg);
+  assert(err == JVMTI_ERROR_NONE, "must not fail");
+  env->ReleaseStringUTFChars(segment, seg);
+#endif
+}
+WB_END
+
+
 WB_ENTRY(jlong, WB_GetCompressedOopsMaxHeapSize(JNIEnv* env, jobject o)) {
   return (jlong)Arguments::max_heap_for_compressed_oops();
 }
@@ -958,6 +984,10 @@ static JNINativeMethod methods[] = {
       CC"(Ljava/lang/String;[Lsun/hotspot/parser/DiagnosticCommand;)[Ljava/lang/Object;",
       (void*) &WB_ParseCommandLine
   },
+  {CC"addToBootstrapClassLoaderSearch", CC"(Ljava/lang/String;)V",
+                                                      (void*)&WB_AddToBootstrapClassLoaderSearch},
+  {CC"addToSystemClassLoaderSearch",    CC"(Ljava/lang/String;)V",
+                                                      (void*)&WB_AddToSystemClassLoaderSearch},
   {CC"getCompressedOopsMaxHeapSize", CC"()J",
       (void*)&WB_GetCompressedOopsMaxHeapSize},
   {CC"printHeapSizes",     CC"()V",                   (void*)&WB_PrintHeapSizes    },

test/testlibrary/whitebox/sun/hotspot/WhiteBox.java

@@ -90,6 +90,10 @@ public class WhiteBox {
   public native URL[] getLookupCacheURLs(ClassLoader loader);
   public native int[] getLookupCacheMatches(ClassLoader loader, String name);
 
+  // JVMTI
+  public native void addToBootstrapClassLoaderSearch(String segment);
+  public native void addToSystemClassLoaderSearch(String segment);
+
   // G1
   public native boolean g1InConcurrentMark();
   public native boolean g1IsHumongous(Object o);