8176055: JMX diagnostic improvements

Reviewed-by: dfuchs, mchung, ahgross, rhalade, jwilhelm

src/share/classes/com/sun/management/HotSpotDiagnosticMXBean.java

 /*
- * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -63,9 +63,10 @@ public interface HotSpotDiagnosticMXBean extends PlatformManagedObject {
      * @param  outputFile the system-dependent filename
      * @param  live if <tt>true</tt> dump only <i>live</i> objects
      *         i.e. objects that are reachable from others
-     * @throws IOException if the <tt>outputFile</tt>
+     * @throws IOException if the <tt>outputFile</tt> already exists,
      *                     cannot be created, opened, or written to.
      * @throws UnsupportedOperationException if this operation is not supported.
+     * @throws IllegalArgumentException if <tt>outputFile</tt> does not end with ".hprof" suffix.
      * @throws NullPointerException if <tt>outputFile</tt> is <tt>null</tt>.
      * @throws SecurityException
      *         If a security manager exists and its {@link

src/share/classes/sun/management/HotSpotDiagnostic.java

 /*
- * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,8 @@ import javax.management.ObjectName;
 
 import com.sun.management.HotSpotDiagnosticMXBean;
 import com.sun.management.VMOption;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 
 /**
  * Implementation of the diagnostic MBean for Hotspot VM.
@@ -41,6 +43,14 @@ public class HotSpotDiagnostic implements HotSpotDiagnosticMXBean {
     }
 
     public void dumpHeap(String outputFile, boolean live) throws IOException {
+
+        String propertyName = "jdk.management.heapdump.allowAnyFileSuffix";
+        PrivilegedAction<Boolean> pa = () -> Boolean.parseBoolean(System.getProperty(propertyName, "false"));
+        boolean allowAnyFileSuffix = AccessController.doPrivileged(pa);
+        if (!allowAnyFileSuffix && !outputFile.endsWith(".hprof")) {
+            throw new IllegalArgumentException("heapdump file must have .hprof extention");
+        }
+
         SecurityManager security = System.getSecurityManager();
         if (security != null) {
             security.checkWrite(outputFile);