7179383: MaxDirectMemorySize argument parsing is broken for values >2G

Summary: change hotspot flag to be unsigned
Reviewed-by: dholmes, sla, fparain, brutisso
Contributed-by: NChris Dennis <cdennis@terracottatech.com>

src/share/vm/prims/jvm.cpp

@@ -345,9 +345,13 @@ JVM_ENTRY(jobject, JVM_InitProperties(JNIEnv *env, jobject properties))
   // Do this after setting user properties to prevent people
   // from setting the value with a -D option, as requested.
   {
-    char as_chars[256];
-    jio_snprintf(as_chars, sizeof(as_chars), INTX_FORMAT, MaxDirectMemorySize);
-    PUTPROP(props, "sun.nio.MaxDirectMemorySize", as_chars);
+    if (FLAG_IS_DEFAULT(MaxDirectMemorySize)) {
+      PUTPROP(props, "sun.nio.MaxDirectMemorySize", "-1");
+    } else {
+      char as_chars[256];
+      jio_snprintf(as_chars, sizeof(as_chars), UINTX_FORMAT, MaxDirectMemorySize);
+      PUTPROP(props, "sun.nio.MaxDirectMemorySize", as_chars);
+    }
   }
 
   // JVM monitoring and management support

src/share/vm/runtime/arguments.cpp

@@ -2709,6 +2709,17 @@ SOLARIS_ONLY(
         return JNI_EINVAL;
       }
       FLAG_SET_CMDLINE(uintx, ConcGCThreads, conc_threads);
+    } else if (match_option(option, "-XX:MaxDirectMemorySize=", &tail)) {
+      julong max_direct_memory_size = 0;
+      ArgsRange errcode = parse_memory_size(tail, &max_direct_memory_size, 0);
+      if (errcode != arg_in_range) {
+        jio_fprintf(defaultStream::error_stream(),
+                    "Invalid maximum direct memory size: %s\n",
+                    option->optionString);
+        describe_range_error(errcode);
+        return JNI_EINVAL;
+      }
+      FLAG_SET_CMDLINE(uintx, MaxDirectMemorySize, max_direct_memory_size);
     } else if (match_option(option, "-XX:", &tail)) { // -XX:xxxx
       // Skip -XX:Flags= since that case has already been handled
       if (strncmp(tail, "Flags=", strlen("Flags=")) != 0) {

src/share/vm/runtime/globals.hpp

@@ -3705,7 +3705,7 @@ class CommandLineFlags {
                                                                             \
   /* Properties for Java libraries  */                                      \
                                                                             \
-  product(intx, MaxDirectMemorySize, -1,                                    \
+  product(uintx, MaxDirectMemorySize, 0,                                    \
           "Maximum total size of NIO direct-buffer allocations")            \
                                                                             \
   /* temporary developer defined flags  */                                  \