7187454: stack overflow in C2 compiler thread on Solaris x86

Summary: Added new FormatBufferResource class to use thread's resource area for error message buffer.
Reviewed-by: twisti

make/solaris/makefiles/fastdebug.make

@@ -39,9 +39,6 @@ OPT_CFLAGS/SLOWER = -xO2
 ifeq ($(COMPILER_REV_NUMERIC), 510)
 # CC 5.10 has bug XXXXX with -xO4
 OPT_CFLAGS/jvmtiClassFileReconstituter.o = $(OPT_CFLAGS/SLOWER)
-# jvm98 crashes on solaris-i586-fastdebug and solaris-sparc-fastdebug with stack overflow
-OPT_CFLAGS/escape.o = $(OPT_CFLAGS) -xspace
-OPT_CFLAGS/matcher.o = $(OPT_CFLAGS) -xspace
 endif # COMPILER_REV_NUMERIC == 510
 
 ifeq ($(COMPILER_REV_NUMERIC), 509)

src/share/vm/opto/callGenerator.cpp

@@ -737,7 +737,7 @@ CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod*
     break;
 
   default:
-    fatal(err_msg("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)));
+    fatal(err_msg_res("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)));
     break;
   }
   return NULL;

src/share/vm/opto/chaitin.cpp

@@ -1536,7 +1536,7 @@ Node *PhaseChaitin::find_base_for_derived( Node **derived_base_map, Node *derive
 
   // Check for AddP-related opcodes
   if( !derived->is_Phi() ) {
-    assert(derived->as_Mach()->ideal_Opcode() == Op_AddP, err_msg("but is: %s", derived->Name()));
+    assert(derived->as_Mach()->ideal_Opcode() == Op_AddP, err_msg_res("but is: %s", derived->Name()));
     Node *base = derived->in(AddPNode::Base);
     derived_base_map[derived->_idx] = base;
     return base;

src/share/vm/opto/compile.cpp

@@ -3138,7 +3138,7 @@ void Compile::ConstantTable::emit(CodeBuffer& cb) {
     default: ShouldNotReachHere();
     }
     assert(constant_addr, "consts section too small");
-    assert((constant_addr - _masm.code()->consts()->start()) == con.offset(), err_msg("must be: %d == %d", constant_addr - _masm.code()->consts()->start(), con.offset()));
+    assert((constant_addr - _masm.code()->consts()->start()) == con.offset(), err_msg_res("must be: %d == %d", constant_addr - _masm.code()->consts()->start(), con.offset()));
   }
 }
 
@@ -3199,7 +3199,7 @@ void Compile::ConstantTable::fill_jump_table(CodeBuffer& cb, MachConstantNode* n
   if (Compile::current()->in_scratch_emit_size())  return;
 
   assert(labels.is_nonempty(), "must be");
-  assert((uint) labels.length() == n->outcnt(), err_msg("must be equal: %d == %d", labels.length(), n->outcnt()));
+  assert((uint) labels.length() == n->outcnt(), err_msg_res("must be equal: %d == %d", labels.length(), n->outcnt()));
 
   // Since MachConstantNode::constant_offset() also contains
   // table_base_offset() we need to subtract the table_base_offset()
@@ -3211,7 +3211,7 @@ void Compile::ConstantTable::fill_jump_table(CodeBuffer& cb, MachConstantNode* n
 
   for (uint i = 0; i < n->outcnt(); i++) {
     address* constant_addr = &jump_table_base[i];
-    assert(*constant_addr == (((address) n) + i), err_msg("all jump-table entries must contain adjusted node pointer: " INTPTR_FORMAT " == " INTPTR_FORMAT, *constant_addr, (((address) n) + i)));
+    assert(*constant_addr == (((address) n) + i), err_msg_res("all jump-table entries must contain adjusted node pointer: " INTPTR_FORMAT " == " INTPTR_FORMAT, *constant_addr, (((address) n) + i)));
     *constant_addr = cb.consts()->target(*labels.at(i), (address) constant_addr);
     cb.consts()->relocate((address) constant_addr, relocInfo::internal_word_type);
   }

src/share/vm/opto/doCall.cpp

@@ -523,10 +523,10 @@ void Parse::do_call() {
             retnode = _gvn.transform( new (C, 3) LShiftINode(retnode, intcon(16)) );
             retnode = _gvn.transform( new (C, 3) RShiftINode(retnode, intcon(16)) );
           } else {
-            assert(ct == T_INT, err_msg("rt=%d, ct=%d", rt, ct));
+            assert(ct == T_INT, err_msg_res("rt=%d, ct=%d", rt, ct));
           }
         } else if (rt == T_OBJECT) {
-          assert(ct == T_OBJECT, err_msg("rt=T_OBJECT, ct=%d", ct));
+          assert(ct == T_OBJECT, err_msg_res("rt=T_OBJECT, ct=%d", ct));
           if (ctype->is_loaded()) {
             Node* if_fail = top();
             retnode = gen_checkcast(retnode, makecon(TypeKlassPtr::make(ctype->as_klass())), &if_fail);
@@ -539,7 +539,7 @@ void Parse::do_call() {
             push(retnode);
           }
         } else {
-          assert(ct == rt, err_msg("unexpected mismatch rt=%d, ct=%d", rt, ct));
+          assert(ct == rt, err_msg_res("unexpected mismatch rt=%d, ct=%d", rt, ct));
           // push a zero; it's better than getting an oop/int mismatch
           retnode = pop_node(rt);
           retnode = zerocon(ct);

src/share/vm/opto/escape.cpp

@@ -1055,7 +1055,7 @@ bool ConnectionGraph::complete_connection_graph(
       C->log()->text("%s", (iterations >= CG_BUILD_ITER_LIMIT) ? "iterations" : "time");
       C->log()->end_elem(" limit'");
     }
-    assert(false, err_msg("infinite EA connection graph build (%f sec, %d iterations) with %d nodes and worklist size %d",
+    assert(false, err_msg_res("infinite EA connection graph build (%f sec, %d iterations) with %d nodes and worklist size %d",
            time.seconds(), iterations, nodes_size(), ptnodes_worklist.length()));
     // Possible infinite build_connection_graph loop,
     // bailout (no changes to ideal graph were made).

src/share/vm/opto/idealGraphPrinter.cpp

@@ -155,8 +155,8 @@ IdealGraphPrinter::IdealGraphPrinter() {
     } else {
       // It would be nice if we could shut down cleanly but it should
       // be an error if we can't connect to the visualizer.
-      fatal(err_msg("Couldn't connect to visualizer at %s:%d",
-                    PrintIdealGraphAddress, PrintIdealGraphPort));
+      fatal(err_msg_res("Couldn't connect to visualizer at %s:%d",
+                        PrintIdealGraphAddress, PrintIdealGraphPort));
     }
   }
 

src/share/vm/opto/node.hpp

@@ -363,7 +363,7 @@ protected:
 #endif
 
   // Reference to the i'th input Node.  Error if out of bounds.
-  Node* in(uint i) const { assert(i < _max, err_msg("oob: i=%d, _max=%d", i, _max)); return _in[i]; }
+  Node* in(uint i) const { assert(i < _max, err_msg_res("oob: i=%d, _max=%d", i, _max)); return _in[i]; }
   // Reference to the i'th output Node.  Error if out of bounds.
   // Use this accessor sparingly.  We are going trying to use iterators instead.
   Node* raw_out(uint i) const { assert(i < _outcnt,"oob"); return _out[i]; }
@@ -394,7 +394,7 @@ protected:
   void ins_req( uint i, Node *n ); // Insert a NEW required input
   void set_req( uint i, Node *n ) {
     assert( is_not_dead(n), "can not use dead node");
-    assert( i < _cnt, err_msg("oob: i=%d, _cnt=%d", i, _cnt));
+    assert( i < _cnt, err_msg_res("oob: i=%d, _cnt=%d", i, _cnt));
     assert( !VerifyHashTableKeys || _hash_lock == 0,
             "remove node from hash table before modifying it");
     Node** p = &_in[i];    // cache this._in, across the del_out call

src/share/vm/opto/parse1.cpp

@@ -1399,7 +1399,7 @@ void Parse::do_one_block() {
     int pre_bc_sp = sp();
     int inputs, depth;
     bool have_se = !stopped() && compute_stack_effects(inputs, depth, /*for_parse*/ true);
-    assert(!have_se || pre_bc_sp >= inputs, err_msg("have enough stack to execute this BC: pre_bc_sp=%d, inputs=%d", pre_bc_sp, inputs));
+    assert(!have_se || pre_bc_sp >= inputs, err_msg_res("have enough stack to execute this BC: pre_bc_sp=%d, inputs=%d", pre_bc_sp, inputs));
 #endif //ASSERT
 
     do_one_bytecode();

src/share/vm/utilities/debug.cpp

@@ -91,6 +91,13 @@
 #  endif
 #endif // PRODUCT
 
+FormatBufferResource::FormatBufferResource(const char * format, ...)
+  : FormatBufferBase((char*)resource_allocate_bytes(RES_BUFSZ)) {
+  va_list argp;
+  va_start(argp, format);
+  jio_vsnprintf(_buf, RES_BUFSZ, format, argp);
+  va_end(argp);
+}
 
 void warning(const char* format, ...) {
   if (PrintWarnings) {

src/share/vm/utilities/debug.hpp

@@ -31,29 +31,43 @@
 #include <stdarg.h>
 
 // Simple class to format the ctor arguments into a fixed-sized buffer.
+class FormatBufferBase {
+ protected:
+  char* _buf;
+  inline FormatBufferBase(char* buf) : _buf(buf) {}
+ public:
+  operator const char *() const { return _buf; }
+};
+
+// Use resource area for buffer
+#define RES_BUFSZ 256
+class FormatBufferResource : public FormatBufferBase {
+ public:
+  FormatBufferResource(const char * format, ...);
+};
+
+// Use stack for buffer
 template <size_t bufsz = 256>
-class FormatBuffer {
+class FormatBuffer : public FormatBufferBase {
  public:
   inline FormatBuffer(const char * format, ...);
   inline void append(const char* format, ...);
   inline void print(const char* format, ...);
   inline void printv(const char* format, va_list ap);
-  operator const char *() const { return _buf; }
 
   char* buffer() { return _buf; }
   int size() { return bufsz; }
 
  private:
   FormatBuffer(const FormatBuffer &); // prevent copies
+  char _buffer[bufsz];
 
  protected:
-  char _buf[bufsz];
-
   inline FormatBuffer();
 };
 
 template <size_t bufsz>
-FormatBuffer<bufsz>::FormatBuffer(const char * format, ...) {
+FormatBuffer<bufsz>::FormatBuffer(const char * format, ...) : FormatBufferBase(_buffer) {
   va_list argp;
   va_start(argp, format);
   jio_vsnprintf(_buf, bufsz, format, argp);
@@ -61,7 +75,7 @@ FormatBuffer<bufsz>::FormatBuffer(const char * format, ...) {
 }
 
 template <size_t bufsz>
-FormatBuffer<bufsz>::FormatBuffer() {
+FormatBuffer<bufsz>::FormatBuffer() : FormatBufferBase(_buffer) {
   _buf[0] = '\0';
 }
 
@@ -93,6 +107,7 @@ void FormatBuffer<bufsz>::append(const char* format, ...) {
 
 // Used to format messages for assert(), guarantee(), fatal(), etc.
 typedef FormatBuffer<> err_msg;
+typedef FormatBufferResource err_msg_res;
 
 // assertions
 #ifdef ASSERT