8069124: runtime/NMT/MallocSiteHashOverflow.java failing in nightlies

Reviewed-by: ctornqvi, coleenp, gtriantafill, dholmes

src/share/vm/services/mallocSiteTable.cpp

@@ -136,7 +136,7 @@ bool MallocSiteTable::walk(MallocSiteWalker* walker) {
 MallocSite* MallocSiteTable::lookup_or_add(const NativeCallStack& key, size_t* bucket_idx,
   size_t* pos_idx, MEMFLAGS flags) {
   assert(flags != mtNone, "Should have a real memory type");
-  int index = hash_to_index(key.hash());
+  unsigned int index = hash_to_index(key.hash());
   assert(index >= 0, "Negative index");
   *bucket_idx = (size_t)index;
   *pos_idx = 0;

src/share/vm/services/mallocSiteTable.hpp

@@ -245,8 +245,7 @@ class MallocSiteTable : AllStatic {
   static MallocSite* malloc_site(size_t bucket_idx, size_t pos_idx);
   static bool walk(MallocSiteWalker* walker);
 
-  static inline int hash_to_index(int  hash) {
-    hash = (hash > 0) ? hash : (-hash);
+  static inline unsigned int hash_to_index(unsigned int hash) {
     return (hash % table_size);
   }
 

src/share/vm/utilities/nativeCallStack.cpp

@@ -55,6 +55,7 @@ NativeCallStack::NativeCallStack(address* pc, int frameCount) {
   for (; index < NMT_TrackingStackDepth; index ++) {
     _stack[index] = NULL;
   }
+  _hash_value = 0;
 }
 
 // number of stack frames captured
@@ -69,19 +70,16 @@ int NativeCallStack::frames() const {
 }
 
 // Hash code. Any better algorithm?
-int NativeCallStack::hash() const {
-  long hash_val = _hash_value;
+unsigned int NativeCallStack::hash() const {
+  uintptr_t hash_val = _hash_value;
   if (hash_val == 0) {
-    long pc;
-    int  index;
-    for (index = 0; index < NMT_TrackingStackDepth; index ++) {
-      pc = (long)_stack[index];
-      if (pc == 0) break;
-      hash_val += pc;
+    for (int index = 0; index < NMT_TrackingStackDepth; index++) {
+      if (_stack[index] == NULL) break;
+      hash_val += (uintptr_t)_stack[index];
     }
 
     NativeCallStack* p = const_cast<NativeCallStack*>(this);
-    p->_hash_value = (int)(hash_val & 0xFFFFFFFF);
+    p->_hash_value = (unsigned int)(hash_val & 0xFFFFFFFF);
   }
   return _hash_value;
 }

src/share/vm/utilities/nativeCallStack.hpp

@@ -56,8 +56,8 @@ class NativeCallStack : public StackObj {
   static const NativeCallStack EMPTY_STACK;
 
  private:
-  address   _stack[NMT_TrackingStackDepth];
-  int       _hash_value;
+  address       _stack[NMT_TrackingStackDepth];
+  unsigned int  _hash_value;
 
  public:
   NativeCallStack(int toSkip = 0, bool fillStack = false);
@@ -89,7 +89,7 @@ class NativeCallStack : public StackObj {
   }
 
   // Hash code. Any better algorithm?
-  int hash() const;
+  unsigned int hash() const;
 
   void print_on(outputStream* out) const;
   void print_on(outputStream* out, int indent) const;