7057857: SIGSEGV [libunpack.so] store_Utf8_char(signed char*, unsigned short)...

7057857: SIGSEGV [libunpack.so] store_Utf8_char(signed char*, unsigned short) in java.util.jar.pack200
Reviewed-by: jrose, asaha, hawtin

src/share/native/com/sun/java/util/jar/pack/unpack.cpp

@@ -1112,11 +1112,14 @@ void unpacker::read_Utf8_values(entry* cpMap, int len) {
     uint size3 = suffix * 3;
     if (suffix == 0)  continue;  // done with empty string
     chars.malloc(size3);
+    CHECK;
     byte* chp = chars.ptr;
     band saved_band = cp_Utf8_big_chars;
     cp_Utf8_big_chars.readData(suffix);
+    CHECK;
     for (int j = 0; j < suffix; j++) {
       unsigned short ch = cp_Utf8_big_chars.getInt();
+      CHECK;
       chp = store_Utf8_char(chp, ch);
     }
     chars.realloc(chp - chars.ptr);
@@ -1134,10 +1137,12 @@ void unpacker::read_Utf8_values(entry* cpMap, int len) {
   CHECK;
   int prevlen = 0;  // previous string length (in chars)
   tmallocs.add(bigbuf.ptr);  // free after this block
+  CHECK;
   cp_Utf8_prefix.rewind();
   for (i = 0; i < len; i++) {
     bytes& chars = allsuffixes[i];
     int prefix = (i < PREFIX_SKIP_2)? 0: cp_Utf8_prefix.getInt();
+    CHECK;
     int suffix = (int)chars.len;
     byte* fillp;
     // by induction, the buffer is already filled with the prefix

src/share/native/com/sun/java/util/jar/pack/utils.cpp

 /*
- * Copyright (c) 2001, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -52,7 +52,7 @@ void* must_malloc(size_t size) {
   if (msize >= 0 && msize < sizeof(int))
     msize = sizeof(int);  // see 0xbaadf00d below
   #endif
-  void* ptr = (msize > PSIZE_MAX) ? null : malloc(msize);
+  void* ptr = (msize > PSIZE_MAX || msize <= 0) ? null : malloc(msize);
   if (ptr != null) {
     memset(ptr, 0, size);
   } else {

src/share/native/com/sun/java/util/jar/pack/utils.h

 /*
- * Copyright (c) 2001, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,7 +33,7 @@ void mtrace(char c, void* ptr, size_t size);
 #endif
 
 // overflow management
-#define OVERFLOW ((size_t)-1)
+#define OVERFLOW ((uint)-1)
 #define PSIZE_MAX (OVERFLOW/2)  /* normal size limit */
 
 inline size_t scale_size(size_t size, size_t scale) {