8038268: VM Crashes in MetaspaceShared::generate_vtable_methods while creating...

8038268: VM Crashes in MetaspaceShared::generate_vtable_methods while creating CDS archive with limiting SharedMiscCodeSize
Summary: estimate the minimum required size for the misc code region and check if the specified misc code region size meets the minimum size requirement
Reviewed-by: jiangli, dholmes

src/share/vm/memory/metaspace.cpp

@@ -3155,6 +3155,16 @@ void Metaspace::global_initialize() {
     SharedMiscDataSize  = align_size_up(SharedMiscDataSize,  max_alignment);
     SharedMiscCodeSize  = align_size_up(SharedMiscCodeSize,  max_alignment);
 
+    // the min_misc_code_size estimate is based on MetaspaceShared::generate_vtable_methods()
+    uintx min_misc_code_size = align_size_up(
+      (MetaspaceShared::num_virtuals * MetaspaceShared::vtbl_list_size) *
+        (sizeof(void*) + MetaspaceShared::vtbl_method_size) + MetaspaceShared::vtbl_common_code_size,
+          max_alignment);
+
+    if (SharedMiscCodeSize < min_misc_code_size) {
+      report_out_of_shared_space(SharedMiscCode);
+    }
+
     // Initialize with the sum of the shared space sizes.  The read-only
     // and read write metaspace chunks will be allocated out of this and the
     // remainder is the misc code and data chunks.

src/share/vm/memory/metaspaceShared.hpp

@@ -58,10 +58,15 @@ class MetaspaceShared : AllStatic {
  public:
   enum {
     vtbl_list_size         = 17,   // number of entries in the shared space vtable list.
-    num_virtuals = 200   // maximum number of virtual functions
+    num_virtuals           = 200,  // maximum number of virtual functions
                                    // If virtual functions are added to Metadata,
                                    // this number needs to be increased.  Also,
                                    // SharedMiscCodeSize will need to be increased.
+                                   // The following 2 sizes were based on
+                                   // MetaspaceShared::generate_vtable_methods()
+    vtbl_method_size       = 16,   // conservative size of the mov1 and jmp instructions
+                                   // for the x64 platform
+    vtbl_common_code_size  = (1*K) // conservative size of the "common_code" for the x64 platform
   };
 
   enum {

src/share/vm/utilities/debug.cpp

@@ -266,17 +266,19 @@ void report_out_of_shared_space(SharedSpaceType shared_space) {
     "native memory for metadata",
     "shared read only space",
     "shared read write space",
-    "shared miscellaneous data space"
+    "shared miscellaneous data space",
+    "shared miscellaneous code space"
   };
   static const char* flag[] = {
     "Metaspace",
     "SharedReadOnlySize",
     "SharedReadWriteSize",
-    "SharedMiscDataSize"
+    "SharedMiscDataSize",
+    "SharedMiscCodeSize"
   };
 
    warning("\nThe %s is not large enough\n"
-           "to preload requested classes. Use -XX:%s=\n"
+           "to preload requested classes. Use -XX:%s=<size>\n"
            "to increase the initial size of %s.\n",
            name[shared_space], flag[shared_space], name[shared_space]);
    exit(2);

src/share/vm/utilities/debug.hpp

@@ -246,7 +246,8 @@ enum SharedSpaceType {
   SharedPermGen,
   SharedReadOnly,
   SharedReadWrite,
-  SharedMiscData
+  SharedMiscData,
+  SharedMiscCode
 };
 
 void report_out_of_shared_space(SharedSpaceType space_type);

test/runtime/SharedArchiveFile/LimitSharedSizes.java

@@ -51,9 +51,12 @@ public class LimitSharedSizes {
         // Known issue, JDK-8038422 (assert() on Windows)
         // new SharedSizeTestData("-XX:SharedMiscDataSize", "500k",    "miscellaneous data"),
 
-        // This will cause a VM crash; commenting out for now; see bug JDK-8038268
-        // @ignore JDK-8038268
-        // new SharedSizeTestData("-XX:SharedMiscCodeSize", "20k",     "miscellaneous code"),
+        // Too small of a misc code size should not cause a vm crash.
+        // It should result in the following error message:
+        // The shared miscellaneous code space is not large enough
+        // to preload requested classes. Use -XX:SharedMiscCodeSize=
+        // to increase the initial size of shared miscellaneous code space.
+        new SharedSizeTestData("-XX:SharedMiscCodeSize", "20k",     "miscellaneous code"),
 
         // these values are larger than default ones, but should
         // be acceptable and not cause failure