7141259: Native stack is missing in hs_err

Summary: Code cleanup and creating a private decoder for error handler, since it can be triggered from in signal handler, where no lock can be taken
Reviewed-by: dholmes, kamg, acorn, coleenp

src/os/bsd/vm/decoder_machO.hpp

 /*
- * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,8 +29,9 @@
 
 #include "utilities/decoder.hpp"
 
-// Just a placehold for now
-class MachODecoder: public NullDecoder {
+// Just a placehold for now, a real implementation should derive
+// from AbstractDecoder
+class MachODecoder : public NullDecoder {
 public:
   MachODecoder() { }
   ~MachODecoder() { }

src/os/windows/vm/decoder_windows.hpp

@@ -36,7 +36,7 @@ typedef BOOL  (WINAPI *pfn_SymInitialize)(HANDLE, PCTSTR, BOOL);
 typedef BOOL  (WINAPI *pfn_SymGetSymFromAddr64)(HANDLE, DWORD64, PDWORD64, PIMAGEHLP_SYMBOL64);
 typedef DWORD (WINAPI *pfn_UndecorateSymbolName)(const char*, char*, DWORD, DWORD);
 
-class WindowsDecoder: public NullDecoder {
+class WindowsDecoder : public AbstractDecoder {
 
 public:
   WindowsDecoder();

src/share/vm/utilities/decoder.cpp

@@ -25,7 +25,9 @@
 #include "precompiled.hpp"
 #include "prims/jvm.h"
 #include "runtime/mutexLocker.hpp"
+#include "runtime/os.hpp"
 #include "utilities/decoder.hpp"
+#include "utilities/vmError.hpp"
 
 #if defined(_WINDOWS)
   #include "decoder_windows.hpp"
@@ -35,74 +37,94 @@
   #include "decoder_elf.hpp"
 #endif
 
-NullDecoder*  Decoder::_decoder = NULL;
-NullDecoder   Decoder::_do_nothing_decoder;
-Mutex*           Decoder::_decoder_lock = new Mutex(Mutex::safepoint,
-                                "DecoderLock");
+AbstractDecoder*  Decoder::_shared_decoder = NULL;
+AbstractDecoder*  Decoder::_error_handler_decoder = NULL;
+NullDecoder       Decoder::_do_nothing_decoder;
+Mutex*            Decoder::_shared_decoder_lock = new Mutex(Mutex::native,
+                                "SharedDecoderLock");
 
-// _decoder_lock should already acquired before enter this method
-NullDecoder* Decoder::get_decoder() {
-  assert(_decoder_lock != NULL && _decoder_lock->owned_by_self(),
+AbstractDecoder* Decoder::get_shared_instance() {
+  assert(_shared_decoder_lock != NULL && _shared_decoder_lock->owned_by_self(),
     "Require DecoderLock to enter");
 
-  if (_decoder != NULL) {
-    return _decoder;
+  if (_shared_decoder == NULL) {
+    _shared_decoder = create_decoder();
   }
+  return _shared_decoder;
+}
+
+AbstractDecoder* Decoder::get_error_handler_instance() {
+  if (_error_handler_decoder == NULL) {
+    _error_handler_decoder = create_decoder();
+  }
+  return _error_handler_decoder;
+}
 
-  // Decoder is a secondary service. Although, it is good to have,
-  // but we can live without it.
+
+AbstractDecoder* Decoder::create_decoder() {
+  AbstractDecoder* decoder;
 #if defined(_WINDOWS)
-  _decoder = new (std::nothrow) WindowsDecoder();
+  decoder = new (std::nothrow) WindowsDecoder();
 #elif defined (__APPLE__)
-    _decoder = new (std::nothrow)MachODecoder();
+  decoder = new (std::nothrow)MachODecoder();
 #else
-    _decoder = new (std::nothrow)ElfDecoder();
+  decoder = new (std::nothrow)ElfDecoder();
 #endif
 
-  if (_decoder == NULL || _decoder->has_error()) {
-    if (_decoder != NULL) {
-      delete _decoder;
+  if (decoder == NULL || decoder->has_error()) {
+    if (decoder != NULL) {
+      delete decoder;
     }
-    _decoder = &_do_nothing_decoder;
+    decoder = &_do_nothing_decoder;
   }
-  return _decoder;
+  return decoder;
 }
 
 bool Decoder::decode(address addr, char* buf, int buflen, int* offset, const char* modulepath) {
-  assert(_decoder_lock != NULL, "Just check");
-  MutexLockerEx locker(_decoder_lock, true);
-  NullDecoder* decoder = get_decoder();
+  assert(_shared_decoder_lock != NULL, "Just check");
+  bool error_handling_thread = os::current_thread_id() == VMError::first_error_tid;
+  MutexLockerEx locker(error_handling_thread ? NULL : _shared_decoder_lock, true);
+  AbstractDecoder* decoder = error_handling_thread ?
+    get_error_handler_instance(): get_shared_instance();
   assert(decoder != NULL, "null decoder");
 
   return decoder->decode(addr, buf, buflen, offset, modulepath);
 }
 
 bool Decoder::demangle(const char* symbol, char* buf, int buflen) {
-  assert(_decoder_lock != NULL, "Just check");
-  MutexLockerEx locker(_decoder_lock, true);
-  NullDecoder* decoder = get_decoder();
+  assert(_shared_decoder_lock != NULL, "Just check");
+  bool error_handling_thread = os::current_thread_id() == VMError::first_error_tid;
+  MutexLockerEx locker(error_handling_thread ? NULL : _shared_decoder_lock, true);
+  AbstractDecoder* decoder = error_handling_thread ?
+    get_error_handler_instance(): get_shared_instance();
   assert(decoder != NULL, "null decoder");
   return decoder->demangle(symbol, buf, buflen);
 }
 
 bool Decoder::can_decode_C_frame_in_vm() {
-  assert(_decoder_lock != NULL, "Just check");
-  MutexLockerEx locker(_decoder_lock, true);
-  NullDecoder* decoder = get_decoder();
+  assert(_shared_decoder_lock != NULL, "Just check");
+  bool error_handling_thread = os::current_thread_id() == VMError::first_error_tid;
+  MutexLockerEx locker(error_handling_thread ? NULL : _shared_decoder_lock, true);
+  AbstractDecoder* decoder = error_handling_thread ?
+    get_error_handler_instance(): get_shared_instance();
   assert(decoder != NULL, "null decoder");
   return decoder->can_decode_C_frame_in_vm();
 }
 
-// shutdown real decoder and replace it with
-// _do_nothing_decoder
+/*
+ * Shutdown shared decoder and replace it with
+ * _do_nothing_decoder. Do nothing with error handler
+ * instance, since the JVM is going down.
+ */
 void Decoder::shutdown() {
-  assert(_decoder_lock != NULL, "Just check");
-  MutexLockerEx locker(_decoder_lock, true);
+  assert(_shared_decoder_lock != NULL, "Just check");
+  MutexLockerEx locker(_shared_decoder_lock, true);
 
-  if (_decoder != NULL && _decoder != &_do_nothing_decoder) {
-    delete _decoder;
+  if (_shared_decoder != NULL &&
+    _shared_decoder != &_do_nothing_decoder) {
+    delete _shared_decoder;
   }
 
-  _decoder = &_do_nothing_decoder;
+  _shared_decoder = &_do_nothing_decoder;
 }
 

src/share/vm/utilities/decoder.hpp

 /*
- * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,7 +29,7 @@
 #include "memory/allocation.hpp"
 #include "runtime/mutex.hpp"
 
-class NullDecoder: public CHeapObj {
+class AbstractDecoder : public CHeapObj {
 public:
   // status code for decoding native C frame
   enum decoder_status {
@@ -43,6 +43,34 @@ public:
          helper_init_error     // SymInitialize failed (Windows only)
   };
 
+  // decode an pc address to corresponding function name and an offset from the beginning of
+  // the function
+  virtual bool decode(address pc, char* buf, int buflen, int* offset,
+    const char* modulepath = NULL) = 0;
+  // demangle a C++ symbol
+  virtual bool demangle(const char* symbol, char* buf, int buflen) = 0;
+  // if the decoder can decode symbols in vm
+  virtual bool can_decode_C_frame_in_vm() const = 0;
+
+  virtual decoder_status status() const {
+    return _decoder_status;
+  }
+
+  virtual bool has_error() const {
+    return is_error(_decoder_status);
+  }
+
+  static bool is_error(decoder_status status) {
+    return (status > 0);
+  }
+
+protected:
+  decoder_status  _decoder_status;
+};
+
+// Do nothing decoder
+class NullDecoder : public AbstractDecoder {
+public:
   NullDecoder() {
     _decoder_status = not_available;
   }
@@ -61,40 +89,34 @@ public:
   virtual bool can_decode_C_frame_in_vm() const {
     return false;
   }
-
-  virtual decoder_status status() const {
-    return _decoder_status;
-  }
-
-  virtual bool has_error() const {
-    return is_error(_decoder_status);
-  }
-
-  static bool is_error(decoder_status status) {
-    return (status > 0);
-  }
-
-protected:
-  decoder_status  _decoder_status;
 };
 
 
-class Decoder: AllStatic {
+class Decoder : AllStatic {
 public:
   static bool decode(address pc, char* buf, int buflen, int* offset, const char* modulepath = NULL);
   static bool demangle(const char* symbol, char* buf, int buflen);
   static bool can_decode_C_frame_in_vm();
 
+  // shutdown shared instance
   static void shutdown();
 protected:
-  static NullDecoder* get_decoder();
-
+  // shared decoder instance, _shared_instance_lock is needed
+  static AbstractDecoder* get_shared_instance();
+  // a private instance for error handler. Error handler can be
+  // triggered almost everywhere, including signal handler, where
+  // no lock can be taken. So the shared decoder can not be used
+  // in this scenario.
+  static AbstractDecoder* get_error_handler_instance();
+
+  static AbstractDecoder* create_decoder();
 private:
-  static NullDecoder*     _decoder;
-  static NullDecoder      _do_nothing_decoder;
+  static AbstractDecoder*     _shared_decoder;
+  static AbstractDecoder*     _error_handler_decoder;
+  static NullDecoder          _do_nothing_decoder;
 
 protected:
-  static Mutex*       _decoder_lock;
+  static Mutex*               _shared_decoder_lock;
 };
 
 #endif // SHARE_VM_UTILITIES_DECODER_HPP

src/share/vm/utilities/decoder_elf.hpp

 /*
- * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,7 +30,7 @@
 #include "utilities/decoder.hpp"
 #include "utilities/elfFile.hpp"
 
-class ElfDecoder: public NullDecoder {
+class ElfDecoder : public AbstractDecoder {
 
 public:
   ElfDecoder() {

src/share/vm/utilities/vmError.hpp

@@ -27,11 +27,12 @@
 
 #include "utilities/globalDefinitions.hpp"
 
-
+class Decoder;
 class VM_ReportJavaOutOfMemory;
 
 class VMError : public StackObj {
   friend class VM_ReportJavaOutOfMemory;
+  friend class Decoder;
 
   enum ErrorType {
     internal_error = 0xe0000000,