macro.cpp 101.7 KB
Newer Older
D
duke 已提交
1
/*
2
 * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved.
D
duke 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
19 20 21
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
D
duke 已提交
22 23 24
 *
 */

25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
#include "precompiled.hpp"
#include "compiler/compileLog.hpp"
#include "libadt/vectset.hpp"
#include "opto/addnode.hpp"
#include "opto/callnode.hpp"
#include "opto/cfgnode.hpp"
#include "opto/compile.hpp"
#include "opto/connode.hpp"
#include "opto/locknode.hpp"
#include "opto/loopnode.hpp"
#include "opto/macro.hpp"
#include "opto/memnode.hpp"
#include "opto/node.hpp"
#include "opto/phaseX.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
#include "opto/subnode.hpp"
#include "opto/type.hpp"
#include "runtime/sharedRuntime.hpp"
D
duke 已提交
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73


//
// Replace any references to "oldref" in inputs to "use" with "newref".
// Returns the number of replacements made.
//
int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
  int nreplacements = 0;
  uint req = use->req();
  for (uint j = 0; j < use->len(); j++) {
    Node *uin = use->in(j);
    if (uin == oldref) {
      if (j < req)
        use->set_req(j, newref);
      else
        use->set_prec(j, newref);
      nreplacements++;
    } else if (j >= req && uin == NULL) {
      break;
    }
  }
  return nreplacements;
}

void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcall) {
  // Copy debug information and adjust JVMState information
  uint old_dbg_start = oldcall->tf()->domain()->cnt();
  uint new_dbg_start = newcall->tf()->domain()->cnt();
  int jvms_adj  = new_dbg_start - old_dbg_start;
  assert (new_dbg_start == newcall->req(), "argument count mismatch");
74

75 76
  // SafePointScalarObject node could be referenced several times in debug info.
  // Use Dict to record cloned nodes.
77
  Dict* sosn_map = new Dict(cmpkey,hashkey);
D
duke 已提交
78
  for (uint i = old_dbg_start; i < oldcall->req(); i++) {
79 80
    Node* old_in = oldcall->in(i);
    // Clone old SafePointScalarObjectNodes, adjusting their field contents.
81
    if (old_in != NULL && old_in->is_SafePointScalarObject()) {
82 83
      SafePointScalarObjectNode* old_sosn = old_in->as_SafePointScalarObject();
      uint old_unique = C->unique();
84 85
      Node* new_in = old_sosn->clone(sosn_map);
      if (old_unique != C->unique()) { // New node?
86
        new_in->set_req(0, C->root()); // reset control edge
87 88 89 90 91
        new_in = transform_later(new_in); // Register new node.
      }
      old_in = new_in;
    }
    newcall->add_req(old_in);
D
duke 已提交
92
  }
93

D
duke 已提交
94 95 96 97 98 99
  newcall->set_jvms(oldcall->jvms());
  for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) {
    jvms->set_map(newcall);
    jvms->set_locoff(jvms->locoff()+jvms_adj);
    jvms->set_stkoff(jvms->stkoff()+jvms_adj);
    jvms->set_monoff(jvms->monoff()+jvms_adj);
100
    jvms->set_scloff(jvms->scloff()+jvms_adj);
D
duke 已提交
101 102 103 104
    jvms->set_endoff(jvms->endoff()+jvms_adj);
  }
}

105 106 107
Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) {
  Node* cmp;
  if (mask != 0) {
108 109
    Node* and_node = transform_later(new (C) AndXNode(word, MakeConX(mask)));
    cmp = transform_later(new (C) CmpXNode(and_node, MakeConX(bits)));
110 111 112
  } else {
    cmp = word;
  }
113 114
  Node* bol = transform_later(new (C) BoolNode(cmp, BoolTest::ne));
  IfNode* iff = new (C) IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN );
115
  transform_later(iff);
D
duke 已提交
116

117
  // Fast path taken.
118
  Node *fast_taken = transform_later( new (C) IfFalseNode(iff) );
D
duke 已提交
119 120

  // Fast path not-taken, i.e. slow path
121
  Node *slow_taken = transform_later( new (C) IfTrueNode(iff) );
122 123 124 125 126 127 128 129

  if (return_fast_path) {
    region->init_req(edge, slow_taken); // Capture slow-control
    return fast_taken;
  } else {
    region->init_req(edge, fast_taken); // Capture fast-control
    return slow_taken;
  }
D
duke 已提交
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
}

//--------------------copy_predefined_input_for_runtime_call--------------------
void PhaseMacroExpand::copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call) {
  // Set fixed predefined input arguments
  call->init_req( TypeFunc::Control, ctrl );
  call->init_req( TypeFunc::I_O    , oldcall->in( TypeFunc::I_O) );
  call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ?????
  call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) );
  call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) );
}

//------------------------------make_slow_call---------------------------------
CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call, const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1) {

  // Slow-path call
 CallNode *call = leaf_name
147 148
   ? (CallNode*)new (C) CallLeafNode      ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM )
   : (CallNode*)new (C) CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), oldcall->jvms()->bci(), TypeRawPtr::BOTTOM );
D
duke 已提交
149 150 151 152 153 154 155

  // Slow path call has no side-effects, uses few values
  copy_predefined_input_for_runtime_call(slow_path, oldcall, call );
  if (parm0 != NULL)  call->init_req(TypeFunc::Parms+0, parm0);
  if (parm1 != NULL)  call->init_req(TypeFunc::Parms+1, parm1);
  copy_call_debug_info(oldcall, call);
  call->set_cnt(PROB_UNLIKELY_MAG(4));  // Same effect as RC_UNCOMMON.
156
  _igvn.replace_node(oldcall, call);
D
duke 已提交
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
  transform_later(call);

  return call;
}

void PhaseMacroExpand::extract_call_projections(CallNode *call) {
  _fallthroughproj = NULL;
  _fallthroughcatchproj = NULL;
  _ioproj_fallthrough = NULL;
  _ioproj_catchall = NULL;
  _catchallcatchproj = NULL;
  _memproj_fallthrough = NULL;
  _memproj_catchall = NULL;
  _resproj = NULL;
  for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) {
    ProjNode *pn = call->fast_out(i)->as_Proj();
    switch (pn->_con) {
      case TypeFunc::Control:
      {
        // For Control (fallthrough) and I_O (catch_all_index) we have CatchProj -> Catch -> Proj
        _fallthroughproj = pn;
        DUIterator_Fast jmax, j = pn->fast_outs(jmax);
        const Node *cn = pn->fast_out(j);
        if (cn->is_Catch()) {
          ProjNode *cpn = NULL;
          for (DUIterator_Fast kmax, k = cn->fast_outs(kmax); k < kmax; k++) {
            cpn = cn->fast_out(k)->as_Proj();
            assert(cpn->is_CatchProj(), "must be a CatchProjNode");
            if (cpn->_con == CatchProjNode::fall_through_index)
              _fallthroughcatchproj = cpn;
            else {
              assert(cpn->_con == CatchProjNode::catch_all_index, "must be correct index.");
              _catchallcatchproj = cpn;
            }
          }
        }
        break;
      }
      case TypeFunc::I_O:
        if (pn->_is_io_use)
          _ioproj_catchall = pn;
        else
          _ioproj_fallthrough = pn;
        break;
      case TypeFunc::Memory:
        if (pn->_is_io_use)
          _memproj_catchall = pn;
        else
          _memproj_fallthrough = pn;
        break;
      case TypeFunc::Parms:
        _resproj = pn;
        break;
      default:
        assert(false, "unexpected projection from allocation node.");
    }
  }

}

217
// Eliminate a card mark sequence.  p2x is a ConvP2XNode
218
void PhaseMacroExpand::eliminate_card_mark(Node* p2x) {
219
  assert(p2x->Opcode() == Op_CastP2X, "ConvP2XNode required");
220 221 222 223 224
  if (!UseG1GC) {
    // vanilla/CMS post barrier
    Node *shift = p2x->unique_out();
    Node *addp = shift->unique_out();
    for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
225 226 227 228 229 230 231 232 233 234
      Node *mem = addp->last_out(j);
      if (UseCondCardMark && mem->is_Load()) {
        assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
        // The load is checking if the card has been written so
        // replace it with zero to fold the test.
        _igvn.replace_node(mem, intcon(0));
        continue;
      }
      assert(mem->is_Store(), "store required");
      _igvn.replace_node(mem, mem->in(MemNode::Memory));
235 236 237
    }
  } else {
    // G1 pre/post barriers
238
    assert(p2x->outcnt() <= 2, "expects 1 or 2 users: Xor and URShift nodes");
239 240 241 242
    // It could be only one user, URShift node, in Object.clone() instrinsic
    // but the new allocation is passed to arraycopy stub and it could not
    // be scalar replaced. So we don't check the case.

243 244 245 246 247 248 249 250 251
    // An other case of only one user (Xor) is when the value check for NULL
    // in G1 post barrier is folded after CCP so the code which used URShift
    // is removed.

    // Take Region node before eliminating post barrier since it also
    // eliminates CastP2X node when it has only one user.
    Node* this_region = p2x->in(0);
    assert(this_region != NULL, "");

252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
    // Remove G1 post barrier.

    // Search for CastP2X->Xor->URShift->Cmp path which
    // checks if the store done to a different from the value's region.
    // And replace Cmp with #0 (false) to collapse G1 post barrier.
    Node* xorx = NULL;
    for (DUIterator_Fast imax, i = p2x->fast_outs(imax); i < imax; i++) {
      Node* u = p2x->fast_out(i);
      if (u->Opcode() == Op_XorX) {
        xorx = u;
        break;
      }
    }
    assert(xorx != NULL, "missing G1 post barrier");
    Node* shift = xorx->unique_out();
    Node* cmpx = shift->unique_out();
    assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
    cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
    "missing region check in G1 post barrier");
    _igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ));

    // Remove G1 pre barrier.

    // Search "if (marking != 0)" check and set it to "false".
    // There is no G1 pre barrier if previous stored value is NULL
    // (for example, after initialization).
    if (this_region->is_Region() && this_region->req() == 3) {
      int ind = 1;
      if (!this_region->in(ind)->is_IfFalse()) {
        ind = 2;
      }
      if (this_region->in(ind)->is_IfFalse()) {
        Node* bol = this_region->in(ind)->in(0)->in(1);
        assert(bol->is_Bool(), "");
        cmpx = bol->in(1);
        if (bol->as_Bool()->_test._test == BoolTest::ne &&
            cmpx->is_Cmp() && cmpx->in(2) == intcon(0) &&
            cmpx->in(1)->is_Load()) {
          Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
          const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() +
                                              PtrQueue::byte_offset_of_active());
          if (adr->is_AddP() && adr->in(AddPNode::Base) == top() &&
              adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
              adr->in(AddPNode::Offset) == MakeConX(marking_offset)) {
            _igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ));
          }
        }
      }
    }
    // Now CastP2X can be removed since it is used only on dead path
    // which currently still alive until igvn optimize it.
303
    assert(p2x->outcnt() == 0 || p2x->unique_out()->Opcode() == Op_URShiftX, "");
304
    _igvn.replace_node(p2x, top());
305 306 307 308
  }
}

// Search for a memory operation for the specified memory slice.
309
static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_mem, Node *alloc, PhaseGVN *phase) {
310 311
  Node *orig_mem = mem;
  Node *alloc_mem = alloc->in(TypeFunc::Memory);
312
  const TypeOopPtr *tinst = phase->C->get_adr_type(alias_idx)->isa_oopptr();
313 314
  while (true) {
    if (mem == alloc_mem || mem == start_mem ) {
T
twisti 已提交
315
      return mem;  // hit one of our sentinels
316 317 318 319 320 321 322 323
    } else if (mem->is_MergeMem()) {
      mem = mem->as_MergeMem()->memory_at(alias_idx);
    } else if (mem->is_Proj() && mem->as_Proj()->_con == TypeFunc::Memory) {
      Node *in = mem->in(0);
      // we can safely skip over safepoints, calls, locks and membars because we
      // already know that the object is safe to eliminate.
      if (in->is_Initialize() && in->as_Initialize()->allocation() == alloc) {
        return in;
324 325 326 327 328 329 330
      } else if (in->is_Call()) {
        CallNode *call = in->as_Call();
        if (!call->may_modify(tinst, phase)) {
          mem = call->in(TypeFunc::Memory);
        }
        mem = in->in(TypeFunc::Memory);
      } else if (in->is_MemBar()) {
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
        mem = in->in(TypeFunc::Memory);
      } else {
        assert(false, "unexpected projection");
      }
    } else if (mem->is_Store()) {
      const TypePtr* atype = mem->as_Store()->adr_type();
      int adr_idx = Compile::current()->get_alias_index(atype);
      if (adr_idx == alias_idx) {
        assert(atype->isa_oopptr(), "address type must be oopptr");
        int adr_offset = atype->offset();
        uint adr_iid = atype->is_oopptr()->instance_id();
        // Array elements references have the same alias_idx
        // but different offset and different instance_id.
        if (adr_offset == offset && adr_iid == alloc->_idx)
          return mem;
      } else {
        assert(adr_idx == Compile::AliasIdxRaw, "address must match or be raw");
      }
      mem = mem->in(MemNode::Memory);
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364
    } else if (mem->is_ClearArray()) {
      if (!ClearArrayNode::step_through(&mem, alloc->_idx, phase)) {
        // Can not bypass initialization of the instance
        // we are looking.
        debug_only(intptr_t offset;)
        assert(alloc == AllocateNode::Ideal_allocation(mem->in(3), phase, offset), "sanity");
        InitializeNode* init = alloc->as_Allocate()->initialization();
        // We are looking for stored value, return Initialize node
        // or memory edge from Allocate node.
        if (init != NULL)
          return init;
        else
          return alloc->in(TypeFunc::Memory); // It will produce zero value (see callers).
      }
      // Otherwise skip it (the call updated 'mem' value).
365
    } else if (mem->Opcode() == Op_SCMemProj) {
366 367 368 369 370 371 372 373 374
      mem = mem->in(0);
      Node* adr = NULL;
      if (mem->is_LoadStore()) {
        adr = mem->in(MemNode::Address);
      } else {
        assert(mem->Opcode() == Op_EncodeISOArray, "sanity");
        adr = mem->in(3); // Destination array
      }
      const TypePtr* atype = adr->bottom_type()->is_ptr();
375 376 377 378 379
      int adr_idx = Compile::current()->get_alias_index(atype);
      if (adr_idx == alias_idx) {
        assert(false, "Object is not scalar replaceable if a LoadStore node access its field");
        return NULL;
      }
380
      mem = mem->in(MemNode::Memory);
381 382 383
    } else {
      return mem;
    }
384
    assert(mem != orig_mem, "dead memory loop");
385 386 387 388 389 390 391 392
  }
}

//
// Given a Memory Phi, compute a value Phi containing the values from stores
// on the input paths.
// Note: this function is recursive, its depth is limied by the "level" argument
// Returns the computed Phi, or NULL if it cannot compute it.
393 394
Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, Node *alloc, Node_Stack *value_phis, int level) {
  assert(mem->is_Phi(), "sanity");
395 396 397 398
  int alias_idx = C->get_alias_index(adr_t);
  int offset = adr_t->offset();
  int instance_id = adr_t->instance_id();

399 400 401 402 403 404 405 406 407 408
  // Check if an appropriate value phi already exists.
  Node* region = mem->in(0);
  for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) {
    Node* phi = region->fast_out(k);
    if (phi->is_Phi() && phi != mem &&
        phi->as_Phi()->is_same_inst_field(phi_type, instance_id, alias_idx, offset)) {
      return phi;
    }
  }
  // Check if an appropriate new value phi already exists.
409 410 411
  Node* new_phi = value_phis->find(mem->_idx);
  if (new_phi != NULL)
    return new_phi;
412 413

  if (level <= 0) {
414
    return NULL; // Give up: phi tree too deep
415
  }
416 417 418 419
  Node *start_mem = C->start()->proj_out(TypeFunc::Memory);
  Node *alloc_mem = alloc->in(TypeFunc::Memory);

  uint length = mem->req();
Z
zgu 已提交
420
  GrowableArray <Node *> values(length, length, NULL, false);
421

422
  // create a new Phi for the value
423
  PhiNode *phi = new (C) PhiNode(mem->in(0), phi_type, NULL, instance_id, alias_idx, offset);
424 425 426
  transform_later(phi);
  value_phis->push(phi, mem->_idx);

427 428 429 430 431
  for (uint j = 1; j < length; j++) {
    Node *in = mem->in(j);
    if (in == NULL || in->is_top()) {
      values.at_put(j, in);
    } else  {
432
      Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
      if (val == start_mem || val == alloc_mem) {
        // hit a sentinel, return appropriate 0 value
        values.at_put(j, _igvn.zerocon(ft));
        continue;
      }
      if (val->is_Initialize()) {
        val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
      }
      if (val == NULL) {
        return NULL;  // can't find a value on this path
      }
      if (val == mem) {
        values.at_put(j, mem);
      } else if (val->is_Store()) {
        values.at_put(j, val->in(MemNode::ValueIn));
      } else if(val->is_Proj() && val->in(0) == alloc) {
        values.at_put(j, _igvn.zerocon(ft));
      } else if (val->is_Phi()) {
451 452 453
        val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1);
        if (val == NULL) {
          return NULL;
454
        }
455
        values.at_put(j, val);
456
      } else if (val->Opcode() == Op_SCMemProj) {
457
        assert(val->in(0)->is_LoadStore() || val->in(0)->Opcode() == Op_EncodeISOArray, "sanity");
458 459
        assert(false, "Object is not scalar replaceable if a LoadStore node access its field");
        return NULL;
460
      } else {
461 462
#ifdef ASSERT
        val->dump();
463
        assert(false, "unknown node on this path");
464
#endif
465
        return NULL;  // unknown node on this path
466 467 468
      }
    }
  }
469
  // Set Phi's inputs
470 471 472 473 474 475 476 477 478 479 480 481
  for (uint j = 1; j < length; j++) {
    if (values.at(j) == mem) {
      phi->init_req(j, phi);
    } else {
      phi->init_req(j, values.at(j));
    }
  }
  return phi;
}

// Search the last value stored into the object's field.
Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, Node *alloc) {
482 483 484
  assert(adr_t->is_known_instance_field(), "instance required");
  int instance_id = adr_t->instance_id();
  assert((uint)instance_id == alloc->_idx, "wrong allocation");
485 486 487 488 489 490

  int alias_idx = C->get_alias_index(adr_t);
  int offset = adr_t->offset();
  Node *start_mem = C->start()->proj_out(TypeFunc::Memory);
  Node *alloc_ctrl = alloc->in(TypeFunc::Control);
  Node *alloc_mem = alloc->in(TypeFunc::Memory);
491 492
  Arena *a = Thread::current()->resource_area();
  VectorSet visited(a);
493 494 495 496 497 498 499 500


  bool done = sfpt_mem == alloc_mem;
  Node *mem = sfpt_mem;
  while (!done) {
    if (visited.test_set(mem->_idx)) {
      return NULL;  // found a loop, give up
    }
501
    mem = scan_mem_chain(mem, alias_idx, offset, start_mem, alloc, &_igvn);
502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
    if (mem == start_mem || mem == alloc_mem) {
      done = true;  // hit a sentinel, return appropriate 0 value
    } else if (mem->is_Initialize()) {
      mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
      if (mem == NULL) {
        done = true; // Something go wrong.
      } else if (mem->is_Store()) {
        const TypePtr* atype = mem->as_Store()->adr_type();
        assert(C->get_alias_index(atype) == Compile::AliasIdxRaw, "store is correct memory slice");
        done = true;
      }
    } else if (mem->is_Store()) {
      const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr();
      assert(atype != NULL, "address type must be oopptr");
      assert(C->get_alias_index(atype) == alias_idx &&
517
             atype->is_known_instance_field() && atype->offset() == offset &&
518 519 520 521 522 523 524
             atype->instance_id() == instance_id, "store is correct memory slice");
      done = true;
    } else if (mem->is_Phi()) {
      // try to find a phi's unique input
      Node *unique_input = NULL;
      Node *top = C->top();
      for (uint i = 1; i < mem->req(); i++) {
525
        Node *n = scan_mem_chain(mem->in(i), alias_idx, offset, start_mem, alloc, &_igvn);
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
        if (n == NULL || n == top || n == mem) {
          continue;
        } else if (unique_input == NULL) {
          unique_input = n;
        } else if (unique_input != n) {
          unique_input = top;
          break;
        }
      }
      if (unique_input != NULL && unique_input != top) {
        mem = unique_input;
      } else {
        done = true;
      }
    } else {
      assert(false, "unexpected node");
    }
  }
  if (mem != NULL) {
    if (mem == start_mem || mem == alloc_mem) {
      // hit a sentinel, return appropriate 0 value
      return _igvn.zerocon(ft);
    } else if (mem->is_Store()) {
      return mem->in(MemNode::ValueIn);
    } else if (mem->is_Phi()) {
      // attempt to produce a Phi reflecting the values on the input paths of the Phi
552
      Node_Stack value_phis(a, 8);
553
      Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit);
554 555
      if (phi != NULL) {
        return phi;
556 557 558 559
      } else {
        // Kill all new Phis
        while(value_phis.is_nonempty()) {
          Node* n = value_phis.node();
560
          _igvn.replace_node(n, C->top());
561 562
          value_phis.pop();
        }
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
      }
    }
  }
  // Something go wrong.
  return NULL;
}

// Check the possibility of scalar replacement.
bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) {
  //  Scan the uses of the allocation to check for anything that would
  //  prevent us from eliminating it.
  NOT_PRODUCT( const char* fail_eliminate = NULL; )
  DEBUG_ONLY( Node* disq_node = NULL; )
  bool  can_eliminate = true;

  Node* res = alloc->result_cast();
  const TypeOopPtr* res_type = NULL;
  if (res == NULL) {
    // All users were eliminated.
  } else if (!res->is_CheckCastPP()) {
    NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";)
    can_eliminate = false;
  } else {
    res_type = _igvn.type(res)->isa_oopptr();
    if (res_type == NULL) {
      NOT_PRODUCT(fail_eliminate = "Neither instance or array allocation";)
      can_eliminate = false;
    } else if (res_type->isa_aryptr()) {
      int length = alloc->in(AllocateNode::ALength)->find_int_con(-1);
      if (length < 0) {
        NOT_PRODUCT(fail_eliminate = "Array's size is not constant";)
        can_eliminate = false;
      }
    }
  }

  if (can_eliminate && res != NULL) {
    for (DUIterator_Fast jmax, j = res->fast_outs(jmax);
                               j < jmax && can_eliminate; j++) {
      Node* use = res->fast_out(j);

      if (use->is_AddP()) {
        const TypePtr* addp_type = _igvn.type(use)->is_ptr();
        int offset = addp_type->offset();

        if (offset == Type::OffsetTop || offset == Type::OffsetBot) {
          NOT_PRODUCT(fail_eliminate = "Undefined field referrence";)
          can_eliminate = false;
          break;
        }
        for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
                                   k < kmax && can_eliminate; k++) {
          Node* n = use->fast_out(k);
          if (!n->is_Store() && n->Opcode() != Op_CastP2X) {
            DEBUG_ONLY(disq_node = n;)
618
            if (n->is_Load() || n->is_LoadStore()) {
619 620 621 622 623 624 625 626 627
              NOT_PRODUCT(fail_eliminate = "Field load";)
            } else {
              NOT_PRODUCT(fail_eliminate = "Not store field referrence";)
            }
            can_eliminate = false;
          }
        }
      } else if (use->is_SafePoint()) {
        SafePointNode* sfpt = use->as_SafePoint();
628
        if (sfpt->is_Call() && sfpt->as_Call()->has_non_debug_use(res)) {
629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670
          // Object is passed as argument.
          DEBUG_ONLY(disq_node = use;)
          NOT_PRODUCT(fail_eliminate = "Object is passed as argument";)
          can_eliminate = false;
        }
        Node* sfptMem = sfpt->memory();
        if (sfptMem == NULL || sfptMem->is_top()) {
          DEBUG_ONLY(disq_node = use;)
          NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";)
          can_eliminate = false;
        } else {
          safepoints.append_if_missing(sfpt);
        }
      } else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark
        if (use->is_Phi()) {
          if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) {
            NOT_PRODUCT(fail_eliminate = "Object is return value";)
          } else {
            NOT_PRODUCT(fail_eliminate = "Object is referenced by Phi";)
          }
          DEBUG_ONLY(disq_node = use;)
        } else {
          if (use->Opcode() == Op_Return) {
            NOT_PRODUCT(fail_eliminate = "Object is return value";)
          }else {
            NOT_PRODUCT(fail_eliminate = "Object is referenced by node";)
          }
          DEBUG_ONLY(disq_node = use;)
        }
        can_eliminate = false;
      }
    }
  }

#ifndef PRODUCT
  if (PrintEliminateAllocations) {
    if (can_eliminate) {
      tty->print("Scalar ");
      if (res == NULL)
        alloc->dump();
      else
        res->dump();
671
    } else if (alloc->_is_scalar_replaceable) {
672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695
      tty->print("NotScalar (%s)", fail_eliminate);
      if (res == NULL)
        alloc->dump();
      else
        res->dump();
#ifdef ASSERT
      if (disq_node != NULL) {
          tty->print("  >>>> ");
          disq_node->dump();
      }
#endif /*ASSERT*/
    }
  }
#endif
  return can_eliminate;
}

// Do scalar replacement.
bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) {
  GrowableArray <SafePointNode *> safepoints_done;

  ciKlass* klass = NULL;
  ciInstanceKlass* iklass = NULL;
  int nfields = 0;
696 697 698 699
  int array_base = 0;
  int element_size = 0;
  BasicType basic_elem_type = T_ILLEGAL;
  ciType* elem_type = NULL;
700 701

  Node* res = alloc->result_cast();
702
  assert(res == NULL || res->is_CheckCastPP(), "unexpected AllocateNode result");
703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730
  const TypeOopPtr* res_type = NULL;
  if (res != NULL) { // Could be NULL when there are no users
    res_type = _igvn.type(res)->isa_oopptr();
  }

  if (res != NULL) {
    klass = res_type->klass();
    if (res_type->isa_instptr()) {
      // find the fields of the class which will be needed for safepoint debug information
      assert(klass->is_instance_klass(), "must be an instance klass.");
      iklass = klass->as_instance_klass();
      nfields = iklass->nof_nonstatic_fields();
    } else {
      // find the array's elements which will be needed for safepoint debug information
      nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1);
      assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.");
      elem_type = klass->as_array_klass()->element_type();
      basic_elem_type = elem_type->basic_type();
      array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
      element_size = type2aelembytes(basic_elem_type);
    }
  }
  //
  // Process the safepoint uses
  //
  while (safepoints.length() > 0) {
    SafePointNode* sfpt = safepoints.pop();
    Node* mem = sfpt->memory();
731 732 733 734 735
    assert(sfpt->jvms() != NULL, "missed JVMS");
    // Fields of scalar objs are referenced only at the end
    // of regular debuginfo at the last (youngest) JVMS.
    // Record relative start index.
    uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
736
    SafePointScalarObjectNode* sobj = new (C) SafePointScalarObjectNode(res_type,
737 738 739 740
#ifdef ASSERT
                                                 alloc,
#endif
                                                 first_ind, nfields);
741
    sobj->init_req(0, C->root());
742 743 744 745
    transform_later(sobj);

    // Scan object's fields adding an input to the safepoint for each field.
    for (int j = 0; j < nfields; j++) {
746
      intptr_t offset;
747 748 749 750 751 752 753
      ciField* field = NULL;
      if (iklass != NULL) {
        field = iklass->nonstatic_field_at(j);
        offset = field->offset();
        elem_type = field->type();
        basic_elem_type = field->layout_type();
      } else {
754
        offset = array_base + j * (intptr_t)element_size;
755 756 757 758
      }

      const Type *field_type;
      // The next code is taken from Parse::do_get_xxx().
759
      if (basic_elem_type == T_OBJECT || basic_elem_type == T_ARRAY) {
760 761
        if (!elem_type->is_loaded()) {
          field_type = TypeInstPtr::BOTTOM;
762
        } else if (field != NULL && field->is_constant() && field->is_static()) {
763 764 765 766 767 768 769 770 771
          // This can happen if the constant oop is non-perm.
          ciObject* con = field->constant_value().as_object();
          // Do not "join" in the previous type; it doesn't add value,
          // and may yield a vacuous result if the field is of interface type.
          field_type = TypeOopPtr::make_from_constant(con)->isa_oopptr();
          assert(field_type != NULL, "field singleton type must be consistent");
        } else {
          field_type = TypeOopPtr::make_from_klass(elem_type->as_klass());
        }
772
        if (UseCompressedOops) {
773
          field_type = field_type->make_narrowoop();
774 775
          basic_elem_type = T_NARROWOOP;
        }
776 777 778 779 780 781 782 783
      } else {
        field_type = Type::get_const_basic_type(basic_elem_type);
      }

      const TypeOopPtr *field_addr_type = res_type->add_offset(offset)->isa_oopptr();

      Node *field_val = value_from_mem(mem, basic_elem_type, field_type, field_addr_type, alloc);
      if (field_val == NULL) {
784 785 786 787
        // We weren't able to find a value for this field,
        // give up on eliminating this allocation.

        // Remove any extra entries we added to the safepoint.
788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
        uint last = sfpt->req() - 1;
        for (int k = 0;  k < j; k++) {
          sfpt->del_req(last--);
        }
        // rollback processed safepoints
        while (safepoints_done.length() > 0) {
          SafePointNode* sfpt_done = safepoints_done.pop();
          // remove any extra entries we added to the safepoint
          last = sfpt_done->req() - 1;
          for (int k = 0;  k < nfields; k++) {
            sfpt_done->del_req(last--);
          }
          JVMState *jvms = sfpt_done->jvms();
          jvms->set_endoff(sfpt_done->req());
          // Now make a pass over the debug information replacing any references
          // to SafePointScalarObjectNode with the allocated object.
          int start = jvms->debug_start();
          int end   = jvms->debug_end();
          for (int i = start; i < end; i++) {
            if (sfpt_done->in(i)->is_SafePointScalarObject()) {
              SafePointScalarObjectNode* scobj = sfpt_done->in(i)->as_SafePointScalarObject();
809
              if (scobj->first_index(jvms) == sfpt_done->req() &&
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837
                  scobj->n_fields() == (uint)nfields) {
                assert(scobj->alloc() == alloc, "sanity");
                sfpt_done->set_req(i, res);
              }
            }
          }
        }
#ifndef PRODUCT
        if (PrintEliminateAllocations) {
          if (field != NULL) {
            tty->print("=== At SafePoint node %d can't find value of Field: ",
                       sfpt->_idx);
            field->print();
            int field_idx = C->get_alias_index(field_addr_type);
            tty->print(" (alias_idx=%d)", field_idx);
          } else { // Array's element
            tty->print("=== At SafePoint node %d can't find value of array element [%d]",
                       sfpt->_idx, j);
          }
          tty->print(", which prevents elimination of: ");
          if (res == NULL)
            alloc->dump();
          else
            res->dump();
        }
#endif
        return false;
      }
838 839 840
      if (UseCompressedOops && field_type->isa_narrowoop()) {
        // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
        // to be able scalar replace the allocation.
841 842 843
        if (field_val->is_EncodeP()) {
          field_val = field_val->in(1);
        } else {
844
          field_val = transform_later(new (C) DecodeNNode(field_val, field_val->get_ptr_type()));
845
        }
846
      }
847 848 849 850 851 852 853 854
      sfpt->add_req(field_val);
    }
    JVMState *jvms = sfpt->jvms();
    jvms->set_endoff(sfpt->req());
    // Now make a pass over the debug information replacing any references
    // to the allocated object with "sobj"
    int start = jvms->debug_start();
    int end   = jvms->debug_end();
855
    sfpt->replace_edges_in_range(res, sobj, start, end);
856 857 858 859 860 861
    safepoints_done.append_if_missing(sfpt); // keep it for rollback
  }
  return true;
}

// Process users of eliminated allocation.
862
void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
863 864 865 866 867 868 869 870 871 872 873
  Node* res = alloc->result_cast();
  if (res != NULL) {
    for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) {
      Node *use = res->last_out(j);
      uint oc1 = res->outcnt();

      if (use->is_AddP()) {
        for (DUIterator_Last kmin, k = use->last_outs(kmin); k >= kmin; ) {
          Node *n = use->last_out(k);
          uint oc2 = use->outcnt();
          if (n->is_Store()) {
874 875 876 877 878 879 880 881 882 883 884 885
#ifdef ASSERT
            // Verify that there is no dependent MemBarVolatile nodes,
            // they should be removed during IGVN, see MemBarNode::Ideal().
            for (DUIterator_Fast pmax, p = n->fast_outs(pmax);
                                       p < pmax; p++) {
              Node* mb = n->fast_out(p);
              assert(mb->is_Initialize() || !mb->is_MemBar() ||
                     mb->req() <= MemBarNode::Precedent ||
                     mb->in(MemBarNode::Precedent) != n,
                     "MemBarVolatile should be eliminated for non-escaping object");
            }
#endif
886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904
            _igvn.replace_node(n, n->in(MemNode::Memory));
          } else {
            eliminate_card_mark(n);
          }
          k -= (oc2 - use->outcnt());
        }
      } else {
        eliminate_card_mark(use);
      }
      j -= (oc1 - res->outcnt());
    }
    assert(res->outcnt() == 0, "all uses of allocated objects must be deleted");
    _igvn.remove_dead_node(res);
  }

  //
  // Process other users of allocation's projections
  //
  if (_resproj != NULL && _resproj->outcnt() != 0) {
905 906 907 908 909 910 911 912 913 914 915
    // First disconnect stores captured by Initialize node.
    // If Initialize node is eliminated first in the following code,
    // it will kill such stores and DUIterator_Last will assert.
    for (DUIterator_Fast jmax, j = _resproj->fast_outs(jmax);  j < jmax; j++) {
      Node *use = _resproj->fast_out(j);
      if (use->is_AddP()) {
        // raw memory addresses used only by the initialization
        _igvn.replace_node(use, C->top());
        --j; --jmax;
      }
    }
916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966
    for (DUIterator_Last jmin, j = _resproj->last_outs(jmin); j >= jmin; ) {
      Node *use = _resproj->last_out(j);
      uint oc1 = _resproj->outcnt();
      if (use->is_Initialize()) {
        // Eliminate Initialize node.
        InitializeNode *init = use->as_Initialize();
        assert(init->outcnt() <= 2, "only a control and memory projection expected");
        Node *ctrl_proj = init->proj_out(TypeFunc::Control);
        if (ctrl_proj != NULL) {
           assert(init->in(TypeFunc::Control) == _fallthroughcatchproj, "allocation control projection");
          _igvn.replace_node(ctrl_proj, _fallthroughcatchproj);
        }
        Node *mem_proj = init->proj_out(TypeFunc::Memory);
        if (mem_proj != NULL) {
          Node *mem = init->in(TypeFunc::Memory);
#ifdef ASSERT
          if (mem->is_MergeMem()) {
            assert(mem->in(TypeFunc::Memory) == _memproj_fallthrough, "allocation memory projection");
          } else {
            assert(mem == _memproj_fallthrough, "allocation memory projection");
          }
#endif
          _igvn.replace_node(mem_proj, mem);
        }
      } else  {
        assert(false, "only Initialize or AddP expected");
      }
      j -= (oc1 - _resproj->outcnt());
    }
  }
  if (_fallthroughcatchproj != NULL) {
    _igvn.replace_node(_fallthroughcatchproj, alloc->in(TypeFunc::Control));
  }
  if (_memproj_fallthrough != NULL) {
    _igvn.replace_node(_memproj_fallthrough, alloc->in(TypeFunc::Memory));
  }
  if (_memproj_catchall != NULL) {
    _igvn.replace_node(_memproj_catchall, C->top());
  }
  if (_ioproj_fallthrough != NULL) {
    _igvn.replace_node(_ioproj_fallthrough, alloc->in(TypeFunc::I_O));
  }
  if (_ioproj_catchall != NULL) {
    _igvn.replace_node(_ioproj_catchall, C->top());
  }
  if (_catchallcatchproj != NULL) {
    _igvn.replace_node(_catchallcatchproj, C->top());
  }
}

bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) {
967 968 969 970 971
  // Don't do scalar replacement if the frame can be popped by JVMTI:
  // if reallocation fails during deoptimization we'll pop all
  // interpreter frames for this compiled frame and that won't play
  // nice with JVMTI popframe.
  if (!EliminateAllocations || JvmtiExport::can_pop_frame() || !alloc->_is_non_escaping) {
972 973 974 975 976 977 978 979 980 981 982
    return false;
  }
  Node* klass = alloc->in(AllocateNode::KlassNode);
  const TypeKlassPtr* tklass = _igvn.type(klass)->is_klassptr();
  Node* res = alloc->result_cast();
  // Eliminate boxing allocations which are not used
  // regardless scalar replacable status.
  bool boxing_alloc = C->eliminate_boxing() &&
                      tklass->klass()->is_instance_klass()  &&
                      tklass->klass()->as_instance_klass()->is_box_klass();
  if (!alloc->_is_scalar_replaceable && (!boxing_alloc || (res != NULL))) {
983 984 985 986 987 988 989 990 991 992
    return false;
  }

  extract_call_projections(alloc);

  GrowableArray <SafePointNode *> safepoints;
  if (!can_eliminate_allocation(alloc, safepoints)) {
    return false;
  }

993 994 995 996 997 998 999 1000 1001 1002
  if (!alloc->_is_scalar_replaceable) {
    assert(res == NULL, "sanity");
    // We can only eliminate allocation if all debug info references
    // are already replaced with SafePointScalarObject because
    // we can't search for a fields value without instance_id.
    if (safepoints.length() > 0) {
      return false;
    }
  }

1003 1004 1005 1006
  if (!scalar_replacement(alloc, safepoints)) {
    return false;
  }

1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
  CompileLog* log = C->log();
  if (log != NULL) {
    log->head("eliminate_allocation type='%d'",
              log->identify(tklass->klass()));
    JVMState* p = alloc->jvms();
    while (p != NULL) {
      log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method()));
      p = p->caller();
    }
    log->tail("eliminate_allocation");
  }

1019 1020 1021
  process_users_of_allocation(alloc);

#ifndef PRODUCT
1022 1023 1024 1025 1026 1027
  if (PrintEliminateAllocations) {
    if (alloc->is_AllocateArray())
      tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx);
    else
      tty->print_cr("++++ Eliminated: %d Allocate", alloc->_idx);
  }
1028 1029 1030 1031 1032
#endif

  return true;
}

1033 1034 1035 1036 1037 1038
bool PhaseMacroExpand::eliminate_boxing_node(CallStaticJavaNode *boxing) {
  // EA should remove all uses of non-escaping boxing node.
  if (!C->eliminate_boxing() || boxing->proj_out(TypeFunc::Parms) != NULL) {
    return false;
  }

1039 1040
  assert(boxing->result_cast() == NULL, "unexpected boxing node result");

1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
  extract_call_projections(boxing);

  const TypeTuple* r = boxing->tf()->range();
  assert(r->cnt() > TypeFunc::Parms, "sanity");
  const TypeInstPtr* t = r->field_at(TypeFunc::Parms)->isa_instptr();
  assert(t != NULL, "sanity");

  CompileLog* log = C->log();
  if (log != NULL) {
    log->head("eliminate_boxing type='%d'",
              log->identify(t->klass()));
    JVMState* p = boxing->jvms();
    while (p != NULL) {
      log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method()));
      p = p->caller();
    }
    log->tail("eliminate_boxing");
  }

  process_users_of_allocation(boxing);

#ifndef PRODUCT
  if (PrintEliminateAllocations) {
    tty->print("++++ Eliminated: %d ", boxing->_idx);
    boxing->method()->print_short_name(tty);
    tty->cr();
  }
#endif

  return true;
}
D
duke 已提交
1072 1073 1074 1075

//---------------------------set_eden_pointers-------------------------
void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) {
  if (UseTLAB) {                // Private allocation: load from TLS
1076
    Node* thread = transform_later(new (C) ThreadLocalNode());
D
duke 已提交
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
    int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset());
    int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset());
    eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset);
    eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset);
  } else {                      // Shared allocation: load from globals
    CollectedHeap* ch = Universe::heap();
    address top_adr = (address)ch->top_addr();
    address end_adr = (address)ch->end_addr();
    eden_top_adr = makecon(TypeRawPtr::make(top_adr));
    eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr);
  }
}


Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) {
  Node* adr = basic_plus_adr(base, offset);
1093
  const TypePtr* adr_type = adr->bottom_type()->is_ptr();
1094
  Node* value = LoadNode::make(_igvn, ctl, mem, adr, adr_type, value_type, bt, MemNode::unordered);
D
duke 已提交
1095 1096 1097 1098 1099 1100 1101
  transform_later(value);
  return value;
}


Node* PhaseMacroExpand::make_store(Node* ctl, Node* mem, Node* base, int offset, Node* value, BasicType bt) {
  Node* adr = basic_plus_adr(base, offset);
1102
  mem = StoreNode::make(_igvn, ctl, mem, adr, NULL, value, bt, MemNode::unordered);
D
duke 已提交
1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
  transform_later(mem);
  return mem;
}

//=============================================================================
//
//                              A L L O C A T I O N
//
// Allocation attempts to be fast in the case of frequent small objects.
// It breaks down like this:
//
// 1) Size in doublewords is computed.  This is a constant for objects and
// variable for most arrays.  Doubleword units are used to avoid size
// overflow of huge doubleword arrays.  We need doublewords in the end for
// rounding.
//
// 2) Size is checked for being 'too large'.  Too-large allocations will go
// the slow path into the VM.  The slow path can throw any required
// exceptions, and does all the special checks for very large arrays.  The
// size test can constant-fold away for objects.  For objects with
// finalizers it constant-folds the otherway: you always go slow with
// finalizers.
//
// 3) If NOT using TLABs, this is the contended loop-back point.
// Load-Locked the heap top.  If using TLABs normal-load the heap top.
//
// 4) Check that heap top + size*8 < max.  If we fail go the slow ` route.
// NOTE: "top+size*8" cannot wrap the 4Gig line!  Here's why: for largish
// "size*8" we always enter the VM, where "largish" is a constant picked small
// enough that there's always space between the eden max and 4Gig (old space is
// there so it's quite large) and large enough that the cost of entering the VM
// is dwarfed by the cost to initialize the space.
//
// 5) If NOT using TLABs, Store-Conditional the adjusted heap top back
// down.  If contended, repeat at step 3.  If using TLABs normal-store
// adjusted heap top back down; there is no contention.
//
// 6) If !ZeroTLAB then Bulk-clear the object/array.  Fill in klass & mark
// fields.
//
// 7) Merge with the slow-path; cast the raw memory pointer to the correct
// oop flavor.
//
//=============================================================================
// FastAllocateSizeLimit value is in DOUBLEWORDS.
// Allocations bigger than this always go the slow route.
// This value must be small enough that allocation attempts that need to
// trigger exceptions go the slow route.  Also, it must be small enough so
// that heap_top + size_in_bytes does not wrap around the 4Gig limit.
//=============================================================================j//
// %%% Here is an old comment from parseHelper.cpp; is it outdated?
// The allocator will coalesce int->oop copies away.  See comment in
// coalesce.cpp about how this works.  It depends critically on the exact
// code shape produced here, so if you are changing this code shape
// make sure the GC info for the heap-top is correct in and around the
// slow-path call.
//

void PhaseMacroExpand::expand_allocate_common(
            AllocateNode* alloc, // allocation node to be expanded
            Node* length,  // array length for an array allocation
            const TypeFunc* slow_call_type, // Type of slow call
            address slow_call_address  // Address of slow call
    )
{

  Node* ctrl = alloc->in(TypeFunc::Control);
  Node* mem  = alloc->in(TypeFunc::Memory);
  Node* i_o  = alloc->in(TypeFunc::I_O);
  Node* size_in_bytes     = alloc->in(AllocateNode::AllocSize);
  Node* klass_node        = alloc->in(AllocateNode::KlassNode);
  Node* initial_slow_test = alloc->in(AllocateNode::InitialTest);

  assert(ctrl != NULL, "must have control");
  // We need a Region and corresponding Phi's to merge the slow-path and fast-path results.
  // they will not be used if "always_slow" is set
  enum { slow_result_path = 1, fast_result_path = 2 };
1180 1181 1182 1183
  Node *result_region = NULL;
  Node *result_phi_rawmem = NULL;
  Node *result_phi_rawoop = NULL;
  Node *result_phi_i_o = NULL;
D
duke 已提交
1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195

  // The initial slow comparison is a size check, the comparison
  // we want to do is a BoolTest::gt
  bool always_slow = false;
  int tv = _igvn.find_int_con(initial_slow_test, -1);
  if (tv >= 0) {
    always_slow = (tv == 1);
    initial_slow_test = NULL;
  } else {
    initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn);
  }

1196
  if (C->env()->dtrace_alloc_probes() ||
1197 1198
      !UseTLAB && (!Universe::heap()->supports_inline_contig_alloc() ||
                   (UseConcMarkSweepGC && CMSIncrementalMode))) {
D
duke 已提交
1199 1200 1201 1202 1203
    // Force slow-path allocation
    always_slow = true;
    initial_slow_test = NULL;
  }

1204

D
duke 已提交
1205 1206 1207 1208 1209 1210 1211
  enum { too_big_or_final_path = 1, need_gc_path = 2 };
  Node *slow_region = NULL;
  Node *toobig_false = ctrl;

  assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent");
  // generate the initial test if necessary
  if (initial_slow_test != NULL ) {
1212
    slow_region = new (C) RegionNode(3);
D
duke 已提交
1213 1214 1215 1216

    // Now make the initial failure test.  Usually a too-big test but
    // might be a TRUE for finalizers or a fancy class check for
    // newInstance0.
1217
    IfNode *toobig_iff = new (C) IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
D
duke 已提交
1218 1219
    transform_later(toobig_iff);
    // Plug the failing-too-big test into the slow-path region
1220
    Node *toobig_true = new (C) IfTrueNode( toobig_iff );
D
duke 已提交
1221 1222
    transform_later(toobig_true);
    slow_region    ->init_req( too_big_or_final_path, toobig_true );
1223
    toobig_false = new (C) IfFalseNode( toobig_iff );
D
duke 已提交
1224 1225 1226 1227 1228 1229 1230 1231 1232
    transform_later(toobig_false);
  } else {         // No initial test, just fall into next case
    toobig_false = ctrl;
    debug_only(slow_region = NodeSentinel);
  }

  Node *slow_mem = mem;  // save the current memory state for slow path
  // generate the fast allocation code unless we know that the initial test will always go slow
  if (!always_slow) {
1233 1234 1235 1236 1237
    // Fast path modifies only raw memory.
    if (mem->is_MergeMem()) {
      mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
    }

1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
    Node* eden_top_adr;
    Node* eden_end_adr;

    set_eden_pointers(eden_top_adr, eden_end_adr);

    // Load Eden::end.  Loop invariant and hoisted.
    //
    // Note: We set the control input on "eden_end" and "old_eden_top" when using
    //       a TLAB to work around a bug where these values were being moved across
    //       a safepoint.  These are not oops, so they cannot be include in the oop
1248
    //       map, but they can be changed by a GC.   The proper way to fix this would
1249 1250 1251 1252 1253 1254
    //       be to set the raw memory state when generating a  SafepointNode.  However
    //       this will require extensive changes to the loop optimization in order to
    //       prevent a degradation of the optimization.
    //       See comment in memnode.hpp, around line 227 in class LoadPNode.
    Node *eden_end = make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);

D
duke 已提交
1255
    // allocate the Region and Phi nodes for the result
1256 1257 1258 1259
    result_region = new (C) RegionNode(3);
    result_phi_rawmem = new (C) PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM);
    result_phi_rawoop = new (C) PhiNode(result_region, TypeRawPtr::BOTTOM);
    result_phi_i_o    = new (C) PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch
D
duke 已提交
1260 1261 1262 1263 1264

    // We need a Region for the loop-back contended case.
    enum { fall_in_path = 1, contended_loopback_path = 2 };
    Node *contended_region;
    Node *contended_phi_rawmem;
1265
    if (UseTLAB) {
D
duke 已提交
1266 1267 1268
      contended_region = toobig_false;
      contended_phi_rawmem = mem;
    } else {
1269 1270
      contended_region = new (C) RegionNode(3);
      contended_phi_rawmem = new (C) PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
D
duke 已提交
1271 1272
      // Now handle the passing-too-big test.  We fall into the contended
      // loop-back merge point.
1273 1274
      contended_region    ->init_req(fall_in_path, toobig_false);
      contended_phi_rawmem->init_req(fall_in_path, mem);
D
duke 已提交
1275 1276 1277 1278 1279 1280 1281
      transform_later(contended_region);
      transform_later(contended_phi_rawmem);
    }

    // Load(-locked) the heap top.
    // See note above concerning the control input when using a TLAB
    Node *old_eden_top = UseTLAB
1282 1283
      ? new (C) LoadPNode      (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
      : new (C) LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
D
duke 已提交
1284 1285 1286

    transform_later(old_eden_top);
    // Add to heap top to get a new heap top
1287
    Node *new_eden_top = new (C) AddPNode(top(), old_eden_top, size_in_bytes);
D
duke 已提交
1288 1289
    transform_later(new_eden_top);
    // Check for needing a GC; compare against heap end
1290
    Node *needgc_cmp = new (C) CmpPNode(new_eden_top, eden_end);
D
duke 已提交
1291
    transform_later(needgc_cmp);
1292
    Node *needgc_bol = new (C) BoolNode(needgc_cmp, BoolTest::ge);
D
duke 已提交
1293
    transform_later(needgc_bol);
1294
    IfNode *needgc_iff = new (C) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
D
duke 已提交
1295 1296 1297
    transform_later(needgc_iff);

    // Plug the failing-heap-space-need-gc test into the slow-path region
1298
    Node *needgc_true = new (C) IfTrueNode(needgc_iff);
D
duke 已提交
1299
    transform_later(needgc_true);
1300 1301
    if (initial_slow_test) {
      slow_region->init_req(need_gc_path, needgc_true);
D
duke 已提交
1302 1303 1304 1305
      // This completes all paths into the slow merge point
      transform_later(slow_region);
    } else {                      // No initial slow path needed!
      // Just fall from the need-GC path straight into the VM call.
1306
      slow_region = needgc_true;
D
duke 已提交
1307 1308
    }
    // No need for a GC.  Setup for the Store-Conditional
1309
    Node *needgc_false = new (C) IfFalseNode(needgc_iff);
D
duke 已提交
1310 1311 1312 1313
    transform_later(needgc_false);

    // Grab regular I/O before optional prefetch may change it.
    // Slow-path does no I/O so just set it to the original I/O.
1314
    result_phi_i_o->init_req(slow_result_path, i_o);
D
duke 已提交
1315 1316 1317 1318

    i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
                              old_eden_top, new_eden_top, length);

1319 1320 1321 1322 1323
    // Name successful fast-path variables
    Node* fast_oop = old_eden_top;
    Node* fast_oop_ctrl;
    Node* fast_oop_rawmem;

D
duke 已提交
1324 1325 1326
    // Store (-conditional) the modified eden top back down.
    // StorePConditional produces flags for a test PLUS a modified raw
    // memory state.
1327 1328
    if (UseTLAB) {
      Node* store_eden_top =
1329
        new (C) StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1330
                              TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered);
D
duke 已提交
1331 1332
      transform_later(store_eden_top);
      fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
1333
      fast_oop_rawmem = store_eden_top;
D
duke 已提交
1334
    } else {
1335
      Node* store_eden_top =
1336
        new (C) StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1337
                                         new_eden_top, fast_oop/*old_eden_top*/);
D
duke 已提交
1338
      transform_later(store_eden_top);
1339
      Node *contention_check = new (C) BoolNode(store_eden_top, BoolTest::ne);
D
duke 已提交
1340
      transform_later(contention_check);
1341
      store_eden_top = new (C) SCMemProjNode(store_eden_top);
D
duke 已提交
1342 1343 1344
      transform_later(store_eden_top);

      // If not using TLABs, check to see if there was contention.
1345
      IfNode *contention_iff = new (C) IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
D
duke 已提交
1346
      transform_later(contention_iff);
1347
      Node *contention_true = new (C) IfTrueNode(contention_iff);
D
duke 已提交
1348 1349
      transform_later(contention_true);
      // If contention, loopback and try again.
1350 1351
      contended_region->init_req(contended_loopback_path, contention_true);
      contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
D
duke 已提交
1352 1353

      // Fast-path succeeded with no contention!
1354
      Node *contention_false = new (C) IfFalseNode(contention_iff);
D
duke 已提交
1355 1356
      transform_later(contention_false);
      fast_oop_ctrl = contention_false;
1357 1358

      // Bump total allocated bytes for this thread
1359
      Node* thread = new (C) ThreadLocalNode();
1360 1361 1362 1363 1364 1365 1366 1367
      transform_later(thread);
      Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
                                             in_bytes(JavaThread::allocated_bytes_offset()));
      Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
                                    0, TypeLong::LONG, T_LONG);
#ifdef _LP64
      Node* alloc_size = size_in_bytes;
#else
1368
      Node* alloc_size = new (C) ConvI2LNode(size_in_bytes);
1369 1370
      transform_later(alloc_size);
#endif
1371
      Node* new_alloc_bytes = new (C) AddLNode(alloc_bytes, alloc_size);
1372 1373 1374
      transform_later(new_alloc_bytes);
      fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
                                   0, new_alloc_bytes, T_LONG);
D
duke 已提交
1375 1376
    }

1377
    InitializeNode* init = alloc->initialization();
D
duke 已提交
1378 1379 1380 1381
    fast_oop_rawmem = initialize_object(alloc,
                                        fast_oop_ctrl, fast_oop_rawmem, fast_oop,
                                        klass_node, length, size_in_bytes);

1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
    // If initialization is performed by an array copy, any required
    // MemBarStoreStore was already added. If the object does not
    // escape no need for a MemBarStoreStore. Otherwise we need a
    // MemBarStoreStore so that stores that initialize this object
    // can't be reordered with a subsequent store that makes this
    // object accessible by other threads.
    if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
      if (init == NULL || init->req() < InitializeNode::RawStores) {
        // No InitializeNode or no stores captured by zeroing
        // elimination. Simply add the MemBarStoreStore after object
        // initialization.
1393
        MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1394 1395 1396 1397
        transform_later(mb);

        mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
        mb->init_req(TypeFunc::Control, fast_oop_ctrl);
1398
        fast_oop_ctrl = new (C) ProjNode(mb,TypeFunc::Control);
1399
        transform_later(fast_oop_ctrl);
1400
        fast_oop_rawmem = new (C) ProjNode(mb,TypeFunc::Memory);
1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
        transform_later(fast_oop_rawmem);
      } else {
        // Add the MemBarStoreStore after the InitializeNode so that
        // all stores performing the initialization that were moved
        // before the InitializeNode happen before the storestore
        // barrier.

        Node* init_ctrl = init->proj_out(TypeFunc::Control);
        Node* init_mem = init->proj_out(TypeFunc::Memory);

        MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
        transform_later(mb);

1414
        Node* ctrl = new (C) ProjNode(init,TypeFunc::Control);
1415
        transform_later(ctrl);
1416
        Node* mem = new (C) ProjNode(init,TypeFunc::Memory);
1417 1418 1419 1420 1421 1422 1423
        transform_later(mem);

        // The MemBarStoreStore depends on control and memory coming
        // from the InitializeNode
        mb->init_req(TypeFunc::Memory, mem);
        mb->init_req(TypeFunc::Control, ctrl);

1424
        ctrl = new (C) ProjNode(mb,TypeFunc::Control);
1425
        transform_later(ctrl);
1426
        mem = new (C) ProjNode(mb,TypeFunc::Memory);
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436
        transform_later(mem);

        // All nodes that depended on the InitializeNode for control
        // and memory must now depend on the MemBarNode that itself
        // depends on the InitializeNode
        _igvn.replace_node(init_ctrl, ctrl);
        _igvn.replace_node(init_mem, mem);
      }
    }

1437
    if (C->env()->dtrace_extended_probes()) {
D
duke 已提交
1438 1439
      // Slow-path call
      int size = TypeFunc::Parms + 2;
1440 1441 1442 1443
      CallLeafNode *call = new (C) CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(),
                                                CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc_base),
                                                "dtrace_object_alloc",
                                                TypeRawPtr::BOTTOM);
D
duke 已提交
1444 1445

      // Get base of thread-local storage area
1446
      Node* thread = new (C) ThreadLocalNode();
D
duke 已提交
1447 1448 1449 1450
      transform_later(thread);

      call->init_req(TypeFunc::Parms+0, thread);
      call->init_req(TypeFunc::Parms+1, fast_oop);
1451 1452 1453 1454 1455
      call->init_req(TypeFunc::Control, fast_oop_ctrl);
      call->init_req(TypeFunc::I_O    , top()); // does no i/o
      call->init_req(TypeFunc::Memory , fast_oop_rawmem);
      call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr));
      call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr));
D
duke 已提交
1456
      transform_later(call);
1457
      fast_oop_ctrl = new (C) ProjNode(call,TypeFunc::Control);
D
duke 已提交
1458
      transform_later(fast_oop_ctrl);
1459
      fast_oop_rawmem = new (C) ProjNode(call,TypeFunc::Memory);
D
duke 已提交
1460 1461 1462 1463
      transform_later(fast_oop_rawmem);
    }

    // Plug in the successful fast-path into the result merge point
1464 1465 1466 1467
    result_region    ->init_req(fast_result_path, fast_oop_ctrl);
    result_phi_rawoop->init_req(fast_result_path, fast_oop);
    result_phi_i_o   ->init_req(fast_result_path, i_o);
    result_phi_rawmem->init_req(fast_result_path, fast_oop_rawmem);
D
duke 已提交
1468 1469
  } else {
    slow_region = ctrl;
K
kvn 已提交
1470
    result_phi_i_o = i_o; // Rename it to use in the following code.
D
duke 已提交
1471 1472 1473
  }

  // Generate slow-path call
1474 1475 1476 1477
  CallNode *call = new (C) CallStaticJavaNode(slow_call_type, slow_call_address,
                               OptoRuntime::stub_name(slow_call_address),
                               alloc->jvms()->bci(),
                               TypePtr::BOTTOM);
D
duke 已提交
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
  call->init_req( TypeFunc::Control, slow_region );
  call->init_req( TypeFunc::I_O    , top() )     ;   // does no i/o
  call->init_req( TypeFunc::Memory , slow_mem ); // may gc ptrs
  call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) );
  call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) );

  call->init_req(TypeFunc::Parms+0, klass_node);
  if (length != NULL) {
    call->init_req(TypeFunc::Parms+1, length);
  }

  // Copy debug information and adjust JVMState information, then replace
  // allocate node with the call
  copy_call_debug_info((CallNode *) alloc,  call);
  if (!always_slow) {
    call->set_cnt(PROB_UNLIKELY_MAG(4));  // Same effect as RC_UNCOMMON.
K
kvn 已提交
1494 1495 1496 1497
  } else {
    // Hook i_o projection to avoid its elimination during allocation
    // replacement (when only a slow call is generated).
    call->set_req(TypeFunc::I_O, result_phi_i_o);
D
duke 已提交
1498
  }
1499
  _igvn.replace_node(alloc, call);
D
duke 已提交
1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
  transform_later(call);

  // Identify the output projections from the allocate node and
  // adjust any references to them.
  // The control and io projections look like:
  //
  //        v---Proj(ctrl) <-----+   v---CatchProj(ctrl)
  //  Allocate                   Catch
  //        ^---Proj(io) <-------+   ^---CatchProj(io)
  //
  //  We are interested in the CatchProj nodes.
  //
  extract_call_projections(call);

K
kvn 已提交
1514 1515 1516 1517
  // An allocate node has separate memory projections for the uses on
  // the control and i_o paths. Replace the control memory projection with
  // result_phi_rawmem (unless we are only generating a slow call when
  // both memory projections are combined)
D
duke 已提交
1518 1519 1520
  if (!always_slow && _memproj_fallthrough != NULL) {
    for (DUIterator_Fast imax, i = _memproj_fallthrough->fast_outs(imax); i < imax; i++) {
      Node *use = _memproj_fallthrough->fast_out(i);
1521
      _igvn.rehash_node_delayed(use);
D
duke 已提交
1522 1523 1524 1525 1526
      imax -= replace_input(use, _memproj_fallthrough, result_phi_rawmem);
      // back up iterator
      --i;
    }
  }
K
kvn 已提交
1527 1528
  // Now change uses of _memproj_catchall to use _memproj_fallthrough and delete
  // _memproj_catchall so we end up with a call that has only 1 memory projection.
D
duke 已提交
1529 1530
  if (_memproj_catchall != NULL ) {
    if (_memproj_fallthrough == NULL) {
1531
      _memproj_fallthrough = new (C) ProjNode(call, TypeFunc::Memory);
D
duke 已提交
1532 1533 1534 1535
      transform_later(_memproj_fallthrough);
    }
    for (DUIterator_Fast imax, i = _memproj_catchall->fast_outs(imax); i < imax; i++) {
      Node *use = _memproj_catchall->fast_out(i);
1536
      _igvn.rehash_node_delayed(use);
D
duke 已提交
1537 1538 1539 1540
      imax -= replace_input(use, _memproj_catchall, _memproj_fallthrough);
      // back up iterator
      --i;
    }
K
kvn 已提交
1541 1542
    assert(_memproj_catchall->outcnt() == 0, "all uses must be deleted");
    _igvn.remove_dead_node(_memproj_catchall);
D
duke 已提交
1543 1544
  }

K
kvn 已提交
1545 1546 1547 1548 1549 1550
  // An allocate node has separate i_o projections for the uses on the control
  // and i_o paths. Always replace the control i_o projection with result i_o
  // otherwise incoming i_o become dead when only a slow call is generated
  // (it is different from memory projections where both projections are
  // combined in such case).
  if (_ioproj_fallthrough != NULL) {
D
duke 已提交
1551 1552
    for (DUIterator_Fast imax, i = _ioproj_fallthrough->fast_outs(imax); i < imax; i++) {
      Node *use = _ioproj_fallthrough->fast_out(i);
1553
      _igvn.rehash_node_delayed(use);
D
duke 已提交
1554 1555 1556 1557 1558
      imax -= replace_input(use, _ioproj_fallthrough, result_phi_i_o);
      // back up iterator
      --i;
    }
  }
K
kvn 已提交
1559 1560
  // Now change uses of _ioproj_catchall to use _ioproj_fallthrough and delete
  // _ioproj_catchall so we end up with a call that has only 1 i_o projection.
D
duke 已提交
1561
  if (_ioproj_catchall != NULL ) {
K
kvn 已提交
1562
    if (_ioproj_fallthrough == NULL) {
1563
      _ioproj_fallthrough = new (C) ProjNode(call, TypeFunc::I_O);
K
kvn 已提交
1564 1565
      transform_later(_ioproj_fallthrough);
    }
D
duke 已提交
1566 1567
    for (DUIterator_Fast imax, i = _ioproj_catchall->fast_outs(imax); i < imax; i++) {
      Node *use = _ioproj_catchall->fast_out(i);
1568
      _igvn.rehash_node_delayed(use);
D
duke 已提交
1569 1570 1571 1572
      imax -= replace_input(use, _ioproj_catchall, _ioproj_fallthrough);
      // back up iterator
      --i;
    }
K
kvn 已提交
1573 1574
    assert(_ioproj_catchall->outcnt() == 0, "all uses must be deleted");
    _igvn.remove_dead_node(_ioproj_catchall);
D
duke 已提交
1575 1576 1577
  }

  // if we generated only a slow call, we are done
K
kvn 已提交
1578 1579
  if (always_slow) {
    // Now we can unhook i_o.
1580 1581 1582 1583 1584 1585 1586 1587 1588 1589
    if (result_phi_i_o->outcnt() > 1) {
      call->set_req(TypeFunc::I_O, top());
    } else {
      assert(result_phi_i_o->unique_ctrl_out() == call, "");
      // Case of new array with negative size known during compilation.
      // AllocateArrayNode::Ideal() optimization disconnect unreachable
      // following code since call to runtime will throw exception.
      // As result there will be no users of i_o after the call.
      // Leave i_o attached to this call to avoid problems in preceding graph.
    }
D
duke 已提交
1590
    return;
K
kvn 已提交
1591
  }
D
duke 已提交
1592 1593 1594 1595 1596


  if (_fallthroughcatchproj != NULL) {
    ctrl = _fallthroughcatchproj->clone();
    transform_later(ctrl);
1597
    _igvn.replace_node(_fallthroughcatchproj, result_region);
D
duke 已提交
1598 1599 1600 1601 1602 1603 1604 1605 1606 1607
  } else {
    ctrl = top();
  }
  Node *slow_result;
  if (_resproj == NULL) {
    // no uses of the allocation result
    slow_result = top();
  } else {
    slow_result = _resproj->clone();
    transform_later(slow_result);
1608
    _igvn.replace_node(_resproj, result_phi_rawoop);
D
duke 已提交
1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
  }

  // Plug slow-path into result merge point
  result_region    ->init_req( slow_result_path, ctrl );
  result_phi_rawoop->init_req( slow_result_path, slow_result);
  result_phi_rawmem->init_req( slow_result_path, _memproj_fallthrough );
  transform_later(result_region);
  transform_later(result_phi_rawoop);
  transform_later(result_phi_rawmem);
  transform_later(result_phi_i_o);
  // This completes all paths into the result merge point
}


// Helper for PhaseMacroExpand::expand_allocate_common.
// Initializes the newly-allocated storage.
Node*
PhaseMacroExpand::initialize_object(AllocateNode* alloc,
                                    Node* control, Node* rawmem, Node* object,
                                    Node* klass_node, Node* length,
                                    Node* size_in_bytes) {
  InitializeNode* init = alloc->initialization();
  // Store the klass & mark bits
  Node* mark_node = NULL;
  // For now only enable fast locking for non-array types
  if (UseBiasedLocking && (length == NULL)) {
1635
    mark_node = make_load(control, rawmem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
D
duke 已提交
1636 1637 1638 1639
  } else {
    mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype()));
  }
  rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
1640

1641
  rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
D
duke 已提交
1642 1643 1644 1645 1646 1647
  int header_size = alloc->minimum_header_size();  // conservatively small

  // Array length
  if (length != NULL) {         // Arrays need length field
    rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT);
    // conservatively small header size:
1648
    header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE);
D
duke 已提交
1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690
    ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
    if (k->is_array_klass())    // we know the exact header size in most cases:
      header_size = Klass::layout_helper_header_size(k->layout_helper());
  }

  // Clear the object body, if necessary.
  if (init == NULL) {
    // The init has somehow disappeared; be cautious and clear everything.
    //
    // This can happen if a node is allocated but an uncommon trap occurs
    // immediately.  In this case, the Initialize gets associated with the
    // trap, and may be placed in a different (outer) loop, if the Allocate
    // is in a loop.  If (this is rare) the inner loop gets unrolled, then
    // there can be two Allocates to one Initialize.  The answer in all these
    // edge cases is safety first.  It is always safe to clear immediately
    // within an Allocate, and then (maybe or maybe not) clear some more later.
    if (!ZeroTLAB)
      rawmem = ClearArrayNode::clear_memory(control, rawmem, object,
                                            header_size, size_in_bytes,
                                            &_igvn);
  } else {
    if (!init->is_complete()) {
      // Try to win by zeroing only what the init does not store.
      // We can also try to do some peephole optimizations,
      // such as combining some adjacent subword stores.
      rawmem = init->complete_stores(control, rawmem, object,
                                     header_size, size_in_bytes, &_igvn);
    }
    // We have no more use for this link, since the AllocateNode goes away:
    init->set_req(InitializeNode::RawAddress, top());
    // (If we keep the link, it just confuses the register allocator,
    // who thinks he sees a real use of the address by the membar.)
  }

  return rawmem;
}

// Generate prefetch instructions for next allocations.
Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false,
                                        Node*& contended_phi_rawmem,
                                        Node* old_eden_top, Node* new_eden_top,
                                        Node* length) {
1691
   enum { fall_in_path = 1, pf_path = 2 };
D
duke 已提交
1692 1693 1694 1695 1696
   if( UseTLAB && AllocatePrefetchStyle == 2 ) {
      // Generate prefetch allocation with watermark check.
      // As an allocation hits the watermark, we will prefetch starting
      // at a "distance" away from watermark.

1697 1698
      Node *pf_region = new (C) RegionNode(3);
      Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
D
duke 已提交
1699 1700
                                                TypeRawPtr::BOTTOM );
      // I/O is used for Prefetch
1701
      Node *pf_phi_abio = new (C) PhiNode( pf_region, Type::ABIO );
D
duke 已提交
1702

1703
      Node *thread = new (C) ThreadLocalNode();
D
duke 已提交
1704 1705
      transform_later(thread);

1706
      Node *eden_pf_adr = new (C) AddPNode( top()/*not oop*/, thread,
D
duke 已提交
1707 1708 1709
                   _igvn.MakeConX(in_bytes(JavaThread::tlab_pf_top_offset())) );
      transform_later(eden_pf_adr);

1710
      Node *old_pf_wm = new (C) LoadPNode(needgc_false,
D
duke 已提交
1711
                                   contended_phi_rawmem, eden_pf_adr,
1712 1713
                                   TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM,
                                   MemNode::unordered);
D
duke 已提交
1714 1715 1716
      transform_later(old_pf_wm);

      // check against new_eden_top
1717
      Node *need_pf_cmp = new (C) CmpPNode( new_eden_top, old_pf_wm );
D
duke 已提交
1718
      transform_later(need_pf_cmp);
1719
      Node *need_pf_bol = new (C) BoolNode( need_pf_cmp, BoolTest::ge );
D
duke 已提交
1720
      transform_later(need_pf_bol);
1721
      IfNode *need_pf_iff = new (C) IfNode( needgc_false, need_pf_bol,
D
duke 已提交
1722 1723 1724 1725
                                       PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN );
      transform_later(need_pf_iff);

      // true node, add prefetchdistance
1726
      Node *need_pf_true = new (C) IfTrueNode( need_pf_iff );
D
duke 已提交
1727 1728
      transform_later(need_pf_true);

1729
      Node *need_pf_false = new (C) IfFalseNode( need_pf_iff );
D
duke 已提交
1730 1731
      transform_later(need_pf_false);

1732
      Node *new_pf_wmt = new (C) AddPNode( top(), old_pf_wm,
D
duke 已提交
1733 1734 1735 1736
                                    _igvn.MakeConX(AllocatePrefetchDistance) );
      transform_later(new_pf_wmt );
      new_pf_wmt->set_req(0, need_pf_true);

1737
      Node *store_new_wmt = new (C) StorePNode(need_pf_true,
D
duke 已提交
1738
                                       contended_phi_rawmem, eden_pf_adr,
1739 1740
                                       TypeRawPtr::BOTTOM, new_pf_wmt,
                                       MemNode::unordered);
D
duke 已提交
1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
      transform_later(store_new_wmt);

      // adding prefetches
      pf_phi_abio->init_req( fall_in_path, i_o );

      Node *prefetch_adr;
      Node *prefetch;
      uint lines = AllocatePrefetchDistance / AllocatePrefetchStepSize;
      uint step_size = AllocatePrefetchStepSize;
      uint distance = 0;

      for ( uint i = 0; i < lines; i++ ) {
1753
        prefetch_adr = new (C) AddPNode( old_pf_wm, new_pf_wmt,
D
duke 已提交
1754 1755
                                            _igvn.MakeConX(distance) );
        transform_later(prefetch_adr);
1756
        prefetch = new (C) PrefetchAllocationNode( i_o, prefetch_adr );
D
duke 已提交
1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775
        transform_later(prefetch);
        distance += step_size;
        i_o = prefetch;
      }
      pf_phi_abio->set_req( pf_path, i_o );

      pf_region->init_req( fall_in_path, need_pf_false );
      pf_region->init_req( pf_path, need_pf_true );

      pf_phi_rawmem->init_req( fall_in_path, contended_phi_rawmem );
      pf_phi_rawmem->init_req( pf_path, store_new_wmt );

      transform_later(pf_region);
      transform_later(pf_phi_rawmem);
      transform_later(pf_phi_abio);

      needgc_false = pf_region;
      contended_phi_rawmem = pf_phi_rawmem;
      i_o = pf_phi_abio;
1776
   } else if( UseTLAB && AllocatePrefetchStyle == 3 ) {
1777 1778
      // Insert a prefetch for each allocation.
      // This code is used for Sparc with BIS.
1779 1780 1781
      Node *pf_region = new (C) RegionNode(3);
      Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
                                             TypeRawPtr::BOTTOM );
1782

1783 1784
      // Generate several prefetch instructions.
      uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1785 1786 1787 1788
      uint step_size = AllocatePrefetchStepSize;
      uint distance = AllocatePrefetchDistance;

      // Next cache address.
1789
      Node *cache_adr = new (C) AddPNode(old_eden_top, old_eden_top,
1790 1791
                                            _igvn.MakeConX(distance));
      transform_later(cache_adr);
1792
      cache_adr = new (C) CastP2XNode(needgc_false, cache_adr);
1793 1794
      transform_later(cache_adr);
      Node* mask = _igvn.MakeConX(~(intptr_t)(step_size-1));
1795
      cache_adr = new (C) AndXNode(cache_adr, mask);
1796
      transform_later(cache_adr);
1797
      cache_adr = new (C) CastX2PNode(cache_adr);
1798 1799 1800
      transform_later(cache_adr);

      // Prefetch
1801
      Node *prefetch = new (C) PrefetchAllocationNode( contended_phi_rawmem, cache_adr );
1802 1803 1804 1805 1806 1807
      prefetch->set_req(0, needgc_false);
      transform_later(prefetch);
      contended_phi_rawmem = prefetch;
      Node *prefetch_adr;
      distance = step_size;
      for ( uint i = 1; i < lines; i++ ) {
1808
        prefetch_adr = new (C) AddPNode( cache_adr, cache_adr,
1809 1810
                                            _igvn.MakeConX(distance) );
        transform_later(prefetch_adr);
1811
        prefetch = new (C) PrefetchAllocationNode( contended_phi_rawmem, prefetch_adr );
1812 1813 1814 1815
        transform_later(prefetch);
        distance += step_size;
        contended_phi_rawmem = prefetch;
      }
D
duke 已提交
1816 1817 1818 1819
   } else if( AllocatePrefetchStyle > 0 ) {
      // Insert a prefetch for each allocation only on the fast-path
      Node *prefetch_adr;
      Node *prefetch;
1820 1821
      // Generate several prefetch instructions.
      uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
D
duke 已提交
1822 1823 1824
      uint step_size = AllocatePrefetchStepSize;
      uint distance = AllocatePrefetchDistance;
      for ( uint i = 0; i < lines; i++ ) {
1825
        prefetch_adr = new (C) AddPNode( old_eden_top, new_eden_top,
D
duke 已提交
1826 1827
                                            _igvn.MakeConX(distance) );
        transform_later(prefetch_adr);
1828
        prefetch = new (C) PrefetchAllocationNode( i_o, prefetch_adr );
D
duke 已提交
1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850
        // Do not let it float too high, since if eden_top == eden_end,
        // both might be null.
        if( i == 0 ) { // Set control for first prefetch, next follows it
          prefetch->init_req(0, needgc_false);
        }
        transform_later(prefetch);
        distance += step_size;
        i_o = prefetch;
      }
   }
   return i_o;
}


void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) {
  expand_allocate_common(alloc, NULL,
                         OptoRuntime::new_instance_Type(),
                         OptoRuntime::new_instance_Java());
}

void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) {
  Node* length = alloc->in(AllocateNode::ALength);
1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862
  InitializeNode* init = alloc->initialization();
  Node* klass_node = alloc->in(AllocateNode::KlassNode);
  ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
  address slow_call_address;  // Address of slow call
  if (init != NULL && init->is_complete_with_arraycopy() &&
      k->is_type_array_klass()) {
    // Don't zero type array during slow allocation in VM since
    // it will be initialized later by arraycopy in compiled code.
    slow_call_address = OptoRuntime::new_array_nozero_Java();
  } else {
    slow_call_address = OptoRuntime::new_array_Java();
  }
D
duke 已提交
1863 1864
  expand_allocate_common(alloc, length,
                         OptoRuntime::new_array_Type(),
1865
                         slow_call_address);
D
duke 已提交
1866 1867
}

K
kvn 已提交
1868 1869
//-------------------mark_eliminated_box----------------------------------
//
1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
// During EA obj may point to several objects but after few ideal graph
// transformations (CCP) it may point to only one non escaping object
// (but still using phi), corresponding locks and unlocks will be marked
// for elimination. Later obj could be replaced with a new node (new phi)
// and which does not have escape information. And later after some graph
// reshape other locks and unlocks (which were not marked for elimination
// before) are connected to this new obj (phi) but they still will not be
// marked for elimination since new obj has no escape information.
// Mark all associated (same box and obj) lock and unlock nodes for
// elimination if some of them marked already.
K
kvn 已提交
1880
void PhaseMacroExpand::mark_eliminated_box(Node* oldbox, Node* obj) {
1881 1882 1883 1884 1885 1886 1887 1888
  if (oldbox->as_BoxLock()->is_eliminated())
    return; // This BoxLock node was processed already.

  // New implementation (EliminateNestedLocks) has separate BoxLock
  // node for each locked region so mark all associated locks/unlocks as
  // eliminated even if different objects are referenced in one locked region
  // (for example, OSR compilation of nested loop inside locked scope).
  if (EliminateNestedLocks ||
K
kvn 已提交
1889 1890 1891 1892
      oldbox->as_BoxLock()->is_simple_lock_region(NULL, obj)) {
    // Box is used only in one lock region. Mark this box as eliminated.
    _igvn.hash_delete(oldbox);
    oldbox->as_BoxLock()->set_eliminated(); // This changes box's hash value
1893
     _igvn.hash_insert(oldbox);
K
kvn 已提交
1894 1895 1896 1897 1898 1899 1900 1901

    for (uint i = 0; i < oldbox->outcnt(); i++) {
      Node* u = oldbox->raw_out(i);
      if (u->is_AbstractLock() && !u->as_AbstractLock()->is_non_esc_obj()) {
        AbstractLockNode* alock = u->as_AbstractLock();
        // Check lock's box since box could be referenced by Lock's debug info.
        if (alock->box_node() == oldbox) {
          // Mark eliminated all related locks and unlocks.
1902 1903 1904
#ifdef ASSERT
          alock->log_lock_optimization(C, "eliminate_lock_set_non_esc4");
#endif
K
kvn 已提交
1905 1906 1907 1908
          alock->set_non_esc_obj();
        }
      }
    }
1909
    return;
1910
  }
K
kvn 已提交
1911 1912 1913

  // Create new "eliminated" BoxLock node and use it in monitor debug info
  // instead of oldbox for the same object.
1914
  BoxLockNode* newbox = oldbox->clone()->as_BoxLock();
K
kvn 已提交
1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928

  // Note: BoxLock node is marked eliminated only here and it is used
  // to indicate that all associated lock and unlock nodes are marked
  // for elimination.
  newbox->set_eliminated();
  transform_later(newbox);

  // Replace old box node with new box for all users of the same object.
  for (uint i = 0; i < oldbox->outcnt();) {
    bool next_edge = true;

    Node* u = oldbox->raw_out(i);
    if (u->is_AbstractLock()) {
      AbstractLockNode* alock = u->as_AbstractLock();
K
kvn 已提交
1929
      if (alock->box_node() == oldbox && alock->obj_node()->eqv_uncast(obj)) {
K
kvn 已提交
1930
        // Replace Box and mark eliminated all related locks and unlocks.
1931 1932 1933
#ifdef ASSERT
        alock->log_lock_optimization(C, "eliminate_lock_set_non_esc5");
#endif
K
kvn 已提交
1934
        alock->set_non_esc_obj();
1935
        _igvn.rehash_node_delayed(alock);
K
kvn 已提交
1936 1937 1938 1939
        alock->set_box_node(newbox);
        next_edge = false;
      }
    }
K
kvn 已提交
1940
    if (u->is_FastLock() && u->as_FastLock()->obj_node()->eqv_uncast(obj)) {
K
kvn 已提交
1941 1942
      FastLockNode* flock = u->as_FastLock();
      assert(flock->box_node() == oldbox, "sanity");
1943
      _igvn.rehash_node_delayed(flock);
K
kvn 已提交
1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959
      flock->set_box_node(newbox);
      next_edge = false;
    }

    // Replace old box in monitor debug info.
    if (u->is_SafePoint() && u->as_SafePoint()->jvms()) {
      SafePointNode* sfn = u->as_SafePoint();
      JVMState* youngest_jvms = sfn->jvms();
      int max_depth = youngest_jvms->depth();
      for (int depth = 1; depth <= max_depth; depth++) {
        JVMState* jvms = youngest_jvms->of_depth(depth);
        int num_mon  = jvms->nof_monitors();
        // Loop over monitors
        for (int idx = 0; idx < num_mon; idx++) {
          Node* obj_node = sfn->monitor_obj(jvms, idx);
          Node* box_node = sfn->monitor_box(jvms, idx);
K
kvn 已提交
1960
          if (box_node == oldbox && obj_node->eqv_uncast(obj)) {
K
kvn 已提交
1961
            int j = jvms->monitor_box_offset(idx);
1962
            _igvn.replace_input_of(u, j, newbox);
K
kvn 已提交
1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979
            next_edge = false;
          }
        }
      }
    }
    if (next_edge) i++;
  }
}

//-----------------------mark_eliminated_locking_nodes-----------------------
void PhaseMacroExpand::mark_eliminated_locking_nodes(AbstractLockNode *alock) {
  if (EliminateNestedLocks) {
    if (alock->is_nested()) {
       assert(alock->box_node()->as_BoxLock()->is_eliminated(), "sanity");
       return;
    } else if (!alock->is_non_esc_obj()) { // Not eliminated or coarsened
      // Only Lock node has JVMState needed here.
1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003
      // Not that preceding claim is documented anywhere else.
      if (alock->jvms() != NULL) {
        if (alock->as_Lock()->is_nested_lock_region()) {
          // Mark eliminated related nested locks and unlocks.
          Node* obj = alock->obj_node();
          BoxLockNode* box_node = alock->box_node()->as_BoxLock();
          assert(!box_node->is_eliminated(), "should not be marked yet");
          // Note: BoxLock node is marked eliminated only here
          // and it is used to indicate that all associated lock
          // and unlock nodes are marked for elimination.
          box_node->set_eliminated(); // Box's hash is always NO_HASH here
          for (uint i = 0; i < box_node->outcnt(); i++) {
            Node* u = box_node->raw_out(i);
            if (u->is_AbstractLock()) {
              alock = u->as_AbstractLock();
              if (alock->box_node() == box_node) {
                // Verify that this Box is referenced only by related locks.
                assert(alock->obj_node()->eqv_uncast(obj), "");
                // Mark all related locks and unlocks.
#ifdef ASSERT
                alock->log_lock_optimization(C, "eliminate_lock_set_nested");
#endif
                alock->set_nested();
              }
K
kvn 已提交
2004
            }
2005
          }
2006 2007 2008 2009 2010 2011
        } else {
#ifdef ASSERT
          alock->log_lock_optimization(C, "eliminate_lock_NOT_nested_lock_region");
          if (C->log() != NULL)
            alock->as_Lock()->is_nested_lock_region(C); // rerun for debugging output
#endif
K
kvn 已提交
2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025
        }
      }
      return;
    }
    // Process locks for non escaping object
    assert(alock->is_non_esc_obj(), "");
  } // EliminateNestedLocks

  if (alock->is_non_esc_obj()) { // Lock is used for non escaping object
    // Look for all locks of this object and mark them and
    // corresponding BoxLock nodes as eliminated.
    Node* obj = alock->obj_node();
    for (uint j = 0; j < obj->outcnt(); j++) {
      Node* o = obj->raw_out(j);
K
kvn 已提交
2026 2027
      if (o->is_AbstractLock() &&
          o->as_AbstractLock()->obj_node()->eqv_uncast(obj)) {
K
kvn 已提交
2028 2029 2030 2031 2032 2033 2034 2035
        alock = o->as_AbstractLock();
        Node* box = alock->box_node();
        // Replace old box node with new eliminated box for all users
        // of the same object and mark related locks as eliminated.
        mark_eliminated_box(box, obj);
      }
    }
  }
2036
}
2037

2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
// we have determined that this lock/unlock can be eliminated, we simply
// eliminate the node without expanding it.
//
// Note:  The membar's associated with the lock/unlock are currently not
//        eliminated.  This should be investigated as a future enhancement.
//
bool PhaseMacroExpand::eliminate_locking_node(AbstractLockNode *alock) {

  if (!alock->is_eliminated()) {
    return false;
  }
#ifdef ASSERT
K
kvn 已提交
2050
  if (!alock->is_coarsened()) {
2051 2052 2053 2054 2055
    // Check that new "eliminated" BoxLock node is created.
    BoxLockNode* oldbox = alock->box_node()->as_BoxLock();
    assert(oldbox->is_eliminated(), "should be done already");
  }
#endif
2056

2057 2058 2059
  alock->log_lock_optimization(C, "eliminate_lock");

#ifndef PRODUCT
2060 2061
  if (PrintEliminateLocks) {
    if (alock->is_Lock()) {
2062
      tty->print_cr("++++ Eliminated: %d Lock", alock->_idx);
2063
    } else {
2064
      tty->print_cr("++++ Eliminated: %d Unlock", alock->_idx);
2065 2066
    }
  }
2067
#endif
2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081

  Node* mem  = alock->in(TypeFunc::Memory);
  Node* ctrl = alock->in(TypeFunc::Control);

  extract_call_projections(alock);
  // There are 2 projections from the lock.  The lock node will
  // be deleted when its last use is subsumed below.
  assert(alock->outcnt() == 2 &&
         _fallthroughproj != NULL &&
         _memproj_fallthrough != NULL,
         "Unexpected projections from Lock/Unlock");

  Node* fallthroughproj = _fallthroughproj;
  Node* memproj_fallthrough = _memproj_fallthrough;
D
duke 已提交
2082 2083 2084 2085 2086

  // The memory projection from a lock/unlock is RawMem
  // The input to a Lock is merged memory, so extract its RawMem input
  // (unless the MergeMem has been optimized away.)
  if (alock->is_Lock()) {
2087
    // Seach for MemBarAcquireLock node and delete it also.
2088
    MemBarNode* membar = fallthroughproj->unique_ctrl_out()->as_MemBar();
2089
    assert(membar != NULL && membar->Opcode() == Op_MemBarAcquireLock, "");
2090 2091
    Node* ctrlproj = membar->proj_out(TypeFunc::Control);
    Node* memproj = membar->proj_out(TypeFunc::Memory);
2092 2093
    _igvn.replace_node(ctrlproj, fallthroughproj);
    _igvn.replace_node(memproj, memproj_fallthrough);
2094 2095 2096 2097 2098 2099

    // Delete FastLock node also if this Lock node is unique user
    // (a loop peeling may clone a Lock node).
    Node* flock = alock->as_Lock()->fastlock_node();
    if (flock->outcnt() == 1) {
      assert(flock->unique_out() == alock, "sanity");
2100
      _igvn.replace_node(flock, top());
2101
    }
D
duke 已提交
2102 2103
  }

2104
  // Seach for MemBarReleaseLock node and delete it also.
2105 2106 2107
  if (alock->is_Unlock() && ctrl != NULL && ctrl->is_Proj() &&
      ctrl->in(0)->is_MemBar()) {
    MemBarNode* membar = ctrl->in(0)->as_MemBar();
2108
    assert(membar->Opcode() == Op_MemBarReleaseLock &&
2109
           mem->is_Proj() && membar == mem->in(0), "");
2110 2111
    _igvn.replace_node(fallthroughproj, ctrl);
    _igvn.replace_node(memproj_fallthrough, mem);
2112 2113 2114 2115 2116 2117
    fallthroughproj = ctrl;
    memproj_fallthrough = mem;
    ctrl = membar->in(TypeFunc::Control);
    mem  = membar->in(TypeFunc::Memory);
  }

2118 2119
  _igvn.replace_node(fallthroughproj, ctrl);
  _igvn.replace_node(memproj_fallthrough, mem);
2120
  return true;
D
duke 已提交
2121 2122 2123 2124 2125 2126 2127 2128 2129 2130
}


//------------------------------expand_lock_node----------------------
void PhaseMacroExpand::expand_lock_node(LockNode *lock) {

  Node* ctrl = lock->in(TypeFunc::Control);
  Node* mem = lock->in(TypeFunc::Memory);
  Node* obj = lock->obj_node();
  Node* box = lock->box_node();
2131
  Node* flock = lock->fastlock_node();
D
duke 已提交
2132

2133
  assert(!box->as_BoxLock()->is_eliminated(), "sanity");
K
kvn 已提交
2134

D
duke 已提交
2135
  // Make the merge point
2136 2137 2138 2139 2140 2141
  Node *region;
  Node *mem_phi;
  Node *slow_path;

  if (UseOptoBiasInlining) {
    /*
T
twisti 已提交
2142
     *  See the full description in MacroAssembler::biased_locking_enter().
2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185
     *
     *  if( (mark_word & biased_lock_mask) == biased_lock_pattern ) {
     *    // The object is biased.
     *    proto_node = klass->prototype_header;
     *    o_node = thread | proto_node;
     *    x_node = o_node ^ mark_word;
     *    if( (x_node & ~age_mask) == 0 ) { // Biased to the current thread ?
     *      // Done.
     *    } else {
     *      if( (x_node & biased_lock_mask) != 0 ) {
     *        // The klass's prototype header is no longer biased.
     *        cas(&mark_word, mark_word, proto_node)
     *        goto cas_lock;
     *      } else {
     *        // The klass's prototype header is still biased.
     *        if( (x_node & epoch_mask) != 0 ) { // Expired epoch?
     *          old = mark_word;
     *          new = o_node;
     *        } else {
     *          // Different thread or anonymous biased.
     *          old = mark_word & (epoch_mask | age_mask | biased_lock_mask);
     *          new = thread | old;
     *        }
     *        // Try to rebias.
     *        if( cas(&mark_word, old, new) == 0 ) {
     *          // Done.
     *        } else {
     *          goto slow_path; // Failed.
     *        }
     *      }
     *    }
     *  } else {
     *    // The object is not biased.
     *    cas_lock:
     *    if( FastLock(obj) == 0 ) {
     *      // Done.
     *    } else {
     *      slow_path:
     *      OptoRuntime::complete_monitor_locking_Java(obj);
     *    }
     *  }
     */

2186
    region  = new (C) RegionNode(5);
2187
    // create a Phi for the memory state
2188
    mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2189

2190 2191
    Node* fast_lock_region  = new (C) RegionNode(3);
    Node* fast_lock_mem_phi = new (C) PhiNode( fast_lock_region, Type::MEMORY, TypeRawPtr::BOTTOM);
2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209

    // First, check mark word for the biased lock pattern.
    Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());

    // Get fast path - mark word has the biased lock pattern.
    ctrl = opt_bits_test(ctrl, fast_lock_region, 1, mark_node,
                         markOopDesc::biased_lock_mask_in_place,
                         markOopDesc::biased_lock_pattern, true);
    // fast_lock_region->in(1) is set to slow path.
    fast_lock_mem_phi->init_req(1, mem);

    // Now check that the lock is biased to the current thread and has
    // the same epoch and bias as Klass::_prototype_header.

    // Special-case a fresh allocation to avoid building nodes:
    Node* klass_node = AllocateNode::Ideal_klass(obj, &_igvn);
    if (klass_node == NULL) {
      Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
Z
zmajo 已提交
2210
      klass_node = transform_later(LoadKlassNode::make(_igvn, NULL, mem, k_adr, _igvn.type(k_adr)->is_ptr()));
2211
#ifdef _LP64
2212
      if (UseCompressedClassPointers && klass_node->is_DecodeNKlass()) {
2213 2214 2215 2216 2217
        assert(klass_node->in(1)->Opcode() == Op_LoadNKlass, "sanity");
        klass_node->in(1)->init_req(0, ctrl);
      } else
#endif
      klass_node->init_req(0, ctrl);
2218
    }
2219
    Node *proto_node = make_load(ctrl, mem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeX_X, TypeX_X->basic_type());
D
duke 已提交
2220

2221 2222 2223 2224
    Node* thread = transform_later(new (C) ThreadLocalNode());
    Node* cast_thread = transform_later(new (C) CastP2XNode(ctrl, thread));
    Node* o_node = transform_later(new (C) OrXNode(cast_thread, proto_node));
    Node* x_node = transform_later(new (C) XorXNode(o_node, mark_node));
2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246

    // Get slow path - mark word does NOT match the value.
    Node* not_biased_ctrl =  opt_bits_test(ctrl, region, 3, x_node,
                                      (~markOopDesc::age_mask_in_place), 0);
    // region->in(3) is set to fast path - the object is biased to the current thread.
    mem_phi->init_req(3, mem);


    // Mark word does NOT match the value (thread | Klass::_prototype_header).


    // First, check biased pattern.
    // Get fast path - _prototype_header has the same biased lock pattern.
    ctrl =  opt_bits_test(not_biased_ctrl, fast_lock_region, 2, x_node,
                          markOopDesc::biased_lock_mask_in_place, 0, true);

    not_biased_ctrl = fast_lock_region->in(2); // Slow path
    // fast_lock_region->in(2) - the prototype header is no longer biased
    // and we have to revoke the bias on this object.
    // We are going to try to reset the mark of this object to the prototype
    // value and fall through to the CAS-based locking scheme.
    Node* adr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
2247 2248
    Node* cas = new (C) StoreXConditionalNode(not_biased_ctrl, mem, adr,
                                              proto_node, mark_node);
2249
    transform_later(cas);
2250
    Node* proj = transform_later( new (C) SCMemProjNode(cas));
2251 2252 2253 2254
    fast_lock_mem_phi->init_req(2, proj);


    // Second, check epoch bits.
2255 2256 2257
    Node* rebiased_region  = new (C) RegionNode(3);
    Node* old_phi = new (C) PhiNode( rebiased_region, TypeX_X);
    Node* new_phi = new (C) PhiNode( rebiased_region, TypeX_X);
2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273

    // Get slow path - mark word does NOT match epoch bits.
    Node* epoch_ctrl =  opt_bits_test(ctrl, rebiased_region, 1, x_node,
                                      markOopDesc::epoch_mask_in_place, 0);
    // The epoch of the current bias is not valid, attempt to rebias the object
    // toward the current thread.
    rebiased_region->init_req(2, epoch_ctrl);
    old_phi->init_req(2, mark_node);
    new_phi->init_req(2, o_node);

    // rebiased_region->in(1) is set to fast path.
    // The epoch of the current bias is still valid but we know
    // nothing about the owner; it might be set or it might be clear.
    Node* cmask   = MakeConX(markOopDesc::biased_lock_mask_in_place |
                             markOopDesc::age_mask_in_place |
                             markOopDesc::epoch_mask_in_place);
2274 2275 2276
    Node* old = transform_later(new (C) AndXNode(mark_node, cmask));
    cast_thread = transform_later(new (C) CastP2XNode(ctrl, thread));
    Node* new_mark = transform_later(new (C) OrXNode(cast_thread, old));
2277 2278 2279 2280 2281 2282 2283 2284 2285
    old_phi->init_req(1, old);
    new_phi->init_req(1, new_mark);

    transform_later(rebiased_region);
    transform_later(old_phi);
    transform_later(new_phi);

    // Try to acquire the bias of the object using an atomic operation.
    // If this fails we will go in to the runtime to revoke the object's bias.
2286
    cas = new (C) StoreXConditionalNode(rebiased_region, mem, adr,
2287 2288
                                           new_phi, old_phi);
    transform_later(cas);
2289
    proj = transform_later( new (C) SCMemProjNode(cas));
2290 2291 2292 2293 2294 2295 2296

    // Get slow path - Failed to CAS.
    not_biased_ctrl = opt_bits_test(rebiased_region, region, 4, cas, 0, 0);
    mem_phi->init_req(4, proj);
    // region->in(4) is set to fast path - the object is rebiased to the current thread.

    // Failed to CAS.
2297 2298
    slow_path  = new (C) RegionNode(3);
    Node *slow_mem = new (C) PhiNode( slow_path, Type::MEMORY, TypeRawPtr::BOTTOM);
2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321

    slow_path->init_req(1, not_biased_ctrl); // Capture slow-control
    slow_mem->init_req(1, proj);

    // Call CAS-based locking scheme (FastLock node).

    transform_later(fast_lock_region);
    transform_later(fast_lock_mem_phi);

    // Get slow path - FastLock failed to lock the object.
    ctrl = opt_bits_test(fast_lock_region, region, 2, flock, 0, 0);
    mem_phi->init_req(2, fast_lock_mem_phi);
    // region->in(2) is set to fast path - the object is locked to the current thread.

    slow_path->init_req(2, ctrl); // Capture slow-control
    slow_mem->init_req(2, fast_lock_mem_phi);

    transform_later(slow_path);
    transform_later(slow_mem);
    // Reset lock's memory edge.
    lock->set_req(TypeFunc::Memory, slow_mem);

  } else {
2322
    region  = new (C) RegionNode(3);
2323
    // create a Phi for the memory state
2324
    mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2325 2326 2327 2328 2329

    // Optimize test; set region slot 2
    slow_path = opt_bits_test(ctrl, region, 2, flock, 0, 0);
    mem_phi->init_req(2, mem);
  }
D
duke 已提交
2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348

  // Make slow path call
  CallNode *call = make_slow_call( (CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(), OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path, obj, box );

  extract_call_projections(call);

  // Slow path can only throw asynchronous exceptions, which are always
  // de-opted.  So the compiler thinks the slow-call can never throw an
  // exception.  If it DOES throw an exception we would need the debug
  // info removed first (since if it throws there is no monitor).
  assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
           _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");

  // Capture slow path
  // disconnect fall-through projection from call and create a new one
  // hook up users of fall-through projection to region
  Node *slow_ctrl = _fallthroughproj->clone();
  transform_later(slow_ctrl);
  _igvn.hash_delete(_fallthroughproj);
2349
  _fallthroughproj->disconnect_inputs(NULL, C);
D
duke 已提交
2350 2351 2352
  region->init_req(1, slow_ctrl);
  // region inputs are now complete
  transform_later(region);
2353
  _igvn.replace_node(_fallthroughproj, region);
D
duke 已提交
2354

2355
  Node *memproj = transform_later( new(C) ProjNode(call, TypeFunc::Memory) );
D
duke 已提交
2356 2357
  mem_phi->init_req(1, memproj );
  transform_later(mem_phi);
2358
  _igvn.replace_node(_memproj_fallthrough, mem_phi);
D
duke 已提交
2359 2360 2361 2362 2363
}

//------------------------------expand_unlock_node----------------------
void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) {

2364
  Node* ctrl = unlock->in(TypeFunc::Control);
D
duke 已提交
2365 2366 2367 2368
  Node* mem = unlock->in(TypeFunc::Memory);
  Node* obj = unlock->obj_node();
  Node* box = unlock->box_node();

2369
  assert(!box->as_BoxLock()->is_eliminated(), "sanity");
K
kvn 已提交
2370

D
duke 已提交
2371 2372 2373
  // No need for a null check on unlock

  // Make the merge point
2374 2375 2376 2377 2378
  Node *region;
  Node *mem_phi;

  if (UseOptoBiasInlining) {
    // Check for biased locking unlock case, which is a no-op.
T
twisti 已提交
2379
    // See the full description in MacroAssembler::biased_locking_exit().
2380
    region  = new (C) RegionNode(4);
2381
    // create a Phi for the memory state
2382
    mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2383 2384 2385 2386 2387 2388 2389
    mem_phi->init_req(3, mem);

    Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
    ctrl = opt_bits_test(ctrl, region, 3, mark_node,
                         markOopDesc::biased_lock_mask_in_place,
                         markOopDesc::biased_lock_pattern);
  } else {
2390
    region  = new (C) RegionNode(3);
2391
    // create a Phi for the memory state
2392
    mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2393
  }
D
duke 已提交
2394

2395
  FastUnlockNode *funlock = new (C) FastUnlockNode( ctrl, obj, box );
D
duke 已提交
2396 2397
  funlock = transform_later( funlock )->as_FastUnlock();
  // Optimize test; set region slot 2
2398
  Node *slow_path = opt_bits_test(ctrl, region, 2, funlock, 0, 0);
D
duke 已提交
2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413

  CallNode *call = make_slow_call( (CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), "complete_monitor_unlocking_C", slow_path, obj, box );

  extract_call_projections(call);

  assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
           _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");

  // No exceptions for unlocking
  // Capture slow path
  // disconnect fall-through projection from call and create a new one
  // hook up users of fall-through projection to region
  Node *slow_ctrl = _fallthroughproj->clone();
  transform_later(slow_ctrl);
  _igvn.hash_delete(_fallthroughproj);
2414
  _fallthroughproj->disconnect_inputs(NULL, C);
D
duke 已提交
2415 2416 2417
  region->init_req(1, slow_ctrl);
  // region inputs are now complete
  transform_later(region);
2418
  _igvn.replace_node(_fallthroughproj, region);
D
duke 已提交
2419

2420
  Node *memproj = transform_later( new(C) ProjNode(call, TypeFunc::Memory) );
D
duke 已提交
2421 2422 2423
  mem_phi->init_req(1, memproj );
  mem_phi->init_req(2, mem);
  transform_later(mem_phi);
2424
  _igvn.replace_node(_memproj_fallthrough, mem_phi);
D
duke 已提交
2425 2426
}

2427 2428 2429
//---------------------------eliminate_macro_nodes----------------------
// Eliminate scalar replaced allocations and associated locks.
void PhaseMacroExpand::eliminate_macro_nodes() {
D
duke 已提交
2430
  if (C->macro_count() == 0)
2431 2432
    return;

2433
  // First, attempt to eliminate locks
2434 2435 2436 2437 2438 2439 2440 2441 2442
  int cnt = C->macro_count();
  for (int i=0; i < cnt; i++) {
    Node *n = C->macro_node(i);
    if (n->is_AbstractLock()) { // Lock and Unlock nodes
      // Before elimination mark all associated (same box and obj)
      // lock and unlock nodes.
      mark_eliminated_locking_nodes(n->as_AbstractLock());
    }
  }
2443
  bool progress = true;
2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457
  while (progress) {
    progress = false;
    for (int i = C->macro_count(); i > 0; i--) {
      Node * n = C->macro_node(i-1);
      bool success = false;
      debug_only(int old_macro_count = C->macro_count(););
      if (n->is_AbstractLock()) {
        success = eliminate_locking_node(n->as_AbstractLock());
      }
      assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
      progress = progress || success;
    }
  }
  // Next, attempt to eliminate allocations
2458
  _has_locks = false;
2459
  progress = true;
2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470
  while (progress) {
    progress = false;
    for (int i = C->macro_count(); i > 0; i--) {
      Node * n = C->macro_node(i-1);
      bool success = false;
      debug_only(int old_macro_count = C->macro_count(););
      switch (n->class_id()) {
      case Node::Class_Allocate:
      case Node::Class_AllocateArray:
        success = eliminate_allocate_node(n->as_Allocate());
        break;
2471 2472 2473
      case Node::Class_CallStaticJava:
        success = eliminate_boxing_node(n->as_CallStaticJava());
        break;
2474 2475
      case Node::Class_Lock:
      case Node::Class_Unlock:
2476
        assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
2477
        _has_locks = true;
2478 2479
        break;
      default:
2480 2481
        assert(n->Opcode() == Op_LoopLimit ||
               n->Opcode() == Op_Opaque1   ||
2482 2483
               n->Opcode() == Op_Opaque2   ||
               n->Opcode() == Op_Opaque3, "unknown node type in macro list");
2484 2485 2486 2487 2488
      }
      assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
      progress = progress || success;
    }
  }
2489 2490 2491 2492 2493 2494 2495 2496
}

//------------------------------expand_macro_nodes----------------------
//  Returns true if a failure occurred.
bool PhaseMacroExpand::expand_macro_nodes() {
  // Last attempt to eliminate macro nodes.
  eliminate_macro_nodes();

2497 2498 2499
  // Make sure expansion will not cause node limit to be exceeded.
  // Worst case is a macro node gets expanded into about 50 nodes.
  // Allow 50% more for optimization.
D
duke 已提交
2500 2501
  if (C->check_node_count(C->macro_count() * 75, "out of nodes before macro expansion" ) )
    return true;
2502

2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515
  // Eliminate Opaque and LoopLimit nodes. Do it after all loop optimizations.
  bool progress = true;
  while (progress) {
    progress = false;
    for (int i = C->macro_count(); i > 0; i--) {
      Node * n = C->macro_node(i-1);
      bool success = false;
      debug_only(int old_macro_count = C->macro_count(););
      if (n->Opcode() == Op_LoopLimit) {
        // Remove it from macro list and put on IGVN worklist to optimize.
        C->remove_macro_node(n);
        _igvn._worklist.push(n);
        success = true;
2516 2517 2518 2519 2520
      } else if (n->Opcode() == Op_CallStaticJava) {
        // Remove it from macro list and put on IGVN worklist to optimize.
        C->remove_macro_node(n);
        _igvn._worklist.push(n);
        success = true;
2521 2522 2523
      } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) {
        _igvn.replace_node(n, n->in(1));
        success = true;
2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547
#if INCLUDE_RTM_OPT
      } else if ((n->Opcode() == Op_Opaque3) && ((Opaque3Node*)n)->rtm_opt()) {
        assert(C->profile_rtm(), "should be used only in rtm deoptimization code");
        assert((n->outcnt() == 1) && n->unique_out()->is_Cmp(), "");
        Node* cmp = n->unique_out();
#ifdef ASSERT
        // Validate graph.
        assert((cmp->outcnt() == 1) && cmp->unique_out()->is_Bool(), "");
        BoolNode* bol = cmp->unique_out()->as_Bool();
        assert((bol->outcnt() == 1) && bol->unique_out()->is_If() &&
               (bol->_test._test == BoolTest::ne), "");
        IfNode* ifn = bol->unique_out()->as_If();
        assert((ifn->outcnt() == 2) &&
               ifn->proj_out(1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change), "");
#endif
        Node* repl = n->in(1);
        if (!_has_locks) {
          // Remove RTM state check if there are no locks in the code.
          // Replace input to compare the same value.
          repl = (cmp->in(1) == n) ? cmp->in(2) : cmp->in(1);
        }
        _igvn.replace_node(n, repl);
        success = true;
#endif
2548 2549 2550 2551 2552 2553
      }
      assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
      progress = progress || success;
    }
  }

D
duke 已提交
2554 2555 2556
  // expand "macro" nodes
  // nodes are removed from the macro list as they are processed
  while (C->macro_count() > 0) {
2557 2558
    int macro_count = C->macro_count();
    Node * n = C->macro_node(macro_count-1);
D
duke 已提交
2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580
    assert(n->is_macro(), "only macro nodes expected here");
    if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) {
      // node is unreachable, so don't try to expand it
      C->remove_macro_node(n);
      continue;
    }
    switch (n->class_id()) {
    case Node::Class_Allocate:
      expand_allocate(n->as_Allocate());
      break;
    case Node::Class_AllocateArray:
      expand_allocate_array(n->as_AllocateArray());
      break;
    case Node::Class_Lock:
      expand_lock_node(n->as_Lock());
      break;
    case Node::Class_Unlock:
      expand_unlock_node(n->as_Unlock());
      break;
    default:
      assert(false, "unknown node type in macro list");
    }
2581
    assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
D
duke 已提交
2582 2583
    if (C->failing())  return true;
  }
2584 2585

  _igvn.set_delay_transform(false);
D
duke 已提交
2586
  _igvn.optimize();
2587
  if (C->failing())  return true;
D
duke 已提交
2588 2589
  return false;
}