7021531: lock ordering problems after fix for 6354181

Reviewed-by: kvn, jrose

src/share/vm/ci/ciEnv.cpp

@@ -413,6 +413,7 @@ ciKlass* ciEnv::get_klass_by_name_impl(ciKlass* accessing_klass,
   }
   KlassHandle found_klass;
   {
+    ttyUnlocker ttyul;  // release tty lock to avoid ordering problems
     MutexLocker ml(Compile_lock);
     klassOop kls;
     if (!require_local) {

src/share/vm/utilities/ostream.cpp

@@ -699,6 +699,17 @@ void ttyLocker::release_tty(intx holder) {
   defaultStream::instance->release(holder);
 }
 
+bool ttyLocker::release_tty_if_locked() {
+  intx thread_id = os::current_thread_id();
+  if (defaultStream::instance->writer() == thread_id) {
+    // release the lock and return true so callers know if was
+    // previously held.
+    release_tty(thread_id);
+    return true;
+  }
+  return false;
+}
+
 void ttyLocker::break_tty_lock_for_safepoint(intx holder) {
   if (defaultStream::instance != NULL &&
       defaultStream::instance->writer() == holder) {

src/share/vm/utilities/ostream.hpp

@@ -123,18 +123,36 @@ extern outputStream* gclog_or_tty;  // stream for gc log if -Xloggc:<f>, or tty
 
 // advisory locking for the shared tty stream:
 class ttyLocker: StackObj {
+  friend class ttyUnlocker;
  private:
   intx _holder;
 
  public:
   static intx  hold_tty();                // returns a "holder" token
   static void  release_tty(intx holder);  // must witness same token
+  static bool  release_tty_if_locked();   // returns true if lock was released
   static void  break_tty_lock_for_safepoint(intx holder);
 
   ttyLocker()  { _holder = hold_tty(); }
   ~ttyLocker() { release_tty(_holder); }
 };
 
+// Release the tty lock if it's held and reacquire it if it was
+// locked.  Used to avoid lock ordering problems.
+class ttyUnlocker: StackObj {
+ private:
+  bool _was_locked;
+ public:
+  ttyUnlocker()  {
+    _was_locked = ttyLocker::release_tty_if_locked();
+  }
+  ~ttyUnlocker() {
+    if (_was_locked) {
+      ttyLocker::hold_tty();
+    }
+  }
+};
+
 // for writing to strings; buffer will expand automatically
 class stringStream : public outputStream {
  protected: