8003705: CDS failed on Windows: can not map in the CDS.

Summary: Map memory only once to prevent 'already mapped' failures.
Reviewed-by: acorn, zgu

hotspot/src/share/vm/memory/filemap.cpp

@@ -211,7 +211,11 @@ void FileMapInfo::open_for_write() {
 
   // Remove the existing file in case another process has it open.
   remove(_full_path);
+#ifdef _WINDOWS  // if 0444 is used on Windows, then remove() will fail.
+  int fd = open(_full_path, O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0744);
+#else
   int fd = open(_full_path, O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0444);
+#endif
   if (fd < 0) {
     fail_stop("Unable to create shared archive file %s.", _full_path);
   }
@@ -370,9 +374,8 @@ ReservedSpace FileMapInfo::reserve_shared_memory() {
     return rs;
   }
   // the reserved virtual memory is for mapping class data sharing archive
-  if (MemTracker::is_on()) {
-    MemTracker::record_virtual_memory_type((address)rs.base(), mtClassShared);
-  }
+  MemTracker::record_virtual_memory_type((address)rs.base(), mtClassShared);
+
   return rs;
 }
 
@@ -394,6 +397,11 @@ char* FileMapInfo::map_region(int i) {
     fail_continue(err_msg("Unable to map %s shared space at required address.", shared_region_name[i]));
     return NULL;
   }
+#ifdef _WINDOWS
+  // This call is Windows-only because the memory_type gets recorded for the other platforms
+  // in method FileMapInfo::reserve_shared_memory(), which is not called on Windows.
+  MemTracker::record_virtual_memory_type((address)base, mtClassShared);
+#endif
   return base;
 }
 

hotspot/src/share/vm/memory/metaspaceShared.cpp

@@ -689,9 +689,15 @@ void MetaspaceShared::print_shared_spaces() {
 bool MetaspaceShared::map_shared_spaces(FileMapInfo* mapinfo) {
   size_t image_alignment = mapinfo->alignment();
 
-  // Map in the shared memory and then map the regions on top of it
+#ifndef _WINDOWS
+  // Map in the shared memory and then map the regions on top of it.
+  // On Windows, don't map the memory here because it will cause the
+  // mappings of the regions to fail.
   ReservedSpace shared_rs = mapinfo->reserve_shared_memory();
   if (!shared_rs.is_reserved()) return false;
+#endif
+
+  assert(!DumpSharedSpaces, "Should not be called with DumpSharedSpaces");
 
   // Map each shared region
   if ((_ro_base = mapinfo->map_region(ro)) != NULL &&
@@ -708,8 +714,10 @@ bool MetaspaceShared::map_shared_spaces(FileMapInfo* mapinfo) {
     if (_rw_base != NULL) mapinfo->unmap_region(rw);
     if (_md_base != NULL) mapinfo->unmap_region(md);
     if (_mc_base != NULL) mapinfo->unmap_region(mc);
+#ifndef _WINDOWS
     // Release the entire mapped region
     shared_rs.release();
+#endif
     // If -Xshare:on is specified, print out the error message and exit VM,
     // otherwise, set UseSharedSpaces to false and continue.
     if (RequireSharedSpaces) {