memnode.cpp 187.0 KB
Newer Older
D
duke 已提交
1
/*
2
 * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
D
duke 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
19 20 21
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
D
duke 已提交
22 23 24
 *
 */

25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
#include "precompiled.hpp"
#include "classfile/systemDictionary.hpp"
#include "compiler/compileLog.hpp"
#include "memory/allocation.inline.hpp"
#include "oops/objArrayKlass.hpp"
#include "opto/addnode.hpp"
#include "opto/cfgnode.hpp"
#include "opto/compile.hpp"
#include "opto/connode.hpp"
#include "opto/loopnode.hpp"
#include "opto/machnode.hpp"
#include "opto/matcher.hpp"
#include "opto/memnode.hpp"
#include "opto/mulnode.hpp"
#include "opto/phaseX.hpp"
#include "opto/regmask.hpp"

D
duke 已提交
42 43 44 45
// Portions of code courtesy of Clifford Click

// Optimization - Graph Style

46 47
static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st);

D
duke 已提交
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
//=============================================================================
uint MemNode::size_of() const { return sizeof(*this); }

const TypePtr *MemNode::adr_type() const {
  Node* adr = in(Address);
  const TypePtr* cross_check = NULL;
  DEBUG_ONLY(cross_check = _adr_type);
  return calculate_adr_type(adr->bottom_type(), cross_check);
}

#ifndef PRODUCT
void MemNode::dump_spec(outputStream *st) const {
  if (in(Address) == NULL)  return; // node is dead
#ifndef ASSERT
  // fake the missing field
  const TypePtr* _adr_type = NULL;
  if (in(Address) != NULL)
    _adr_type = in(Address)->bottom_type()->isa_ptr();
#endif
  dump_adr_type(this, _adr_type, st);

  Compile* C = Compile::current();
  if( C->alias_type(_adr_type)->is_volatile() )
    st->print(" Volatile!");
}

void MemNode::dump_adr_type(const Node* mem, const TypePtr* adr_type, outputStream *st) {
  st->print(" @");
  if (adr_type == NULL) {
    st->print("NULL");
  } else {
    adr_type->dump_on(st);
    Compile* C = Compile::current();
    Compile::AliasType* atp = NULL;
    if (C->have_alias_type(adr_type))  atp = C->alias_type(adr_type);
    if (atp == NULL)
      st->print(", idx=?\?;");
    else if (atp->index() == Compile::AliasIdxBot)
      st->print(", idx=Bot;");
    else if (atp->index() == Compile::AliasIdxTop)
      st->print(", idx=Top;");
    else if (atp->index() == Compile::AliasIdxRaw)
      st->print(", idx=Raw;");
    else {
      ciField* field = atp->field();
      if (field) {
        st->print(", name=");
        field->print_name_on(st);
      }
      st->print(", idx=%d;", atp->index());
    }
  }
}

extern void print_alias_types();

#endif

106 107 108 109 110 111 112
Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
  assert((t_oop != NULL), "sanity");
  bool is_instance = t_oop->is_known_instance_field();
  bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
                             (load != NULL) && load->is_Load() &&
                             (phase->is_IterGVN() != NULL);
  if (!(is_instance || is_boxed_value_load))
113
    return mchain;  // don't try to optimize non-instance types
114
  uint instance_id = t_oop->instance_id();
115
  Node *start_mem = phase->C->start()->proj_out(TypeFunc::Memory);
116 117 118 119
  Node *prev = NULL;
  Node *result = mchain;
  while (prev != result) {
    prev = result;
120
    if (result == start_mem)
T
twisti 已提交
121
      break;  // hit one of our sentinels
122 123 124
    // skip over a call which does not affect this memory slice
    if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
      Node *proj_in = result->in(0);
125
      if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
T
twisti 已提交
126
        break;  // hit one of our sentinels
127
      } else if (proj_in->is_Call()) {
128
        CallNode *call = proj_in->as_Call();
129
        if (!call->may_modify(t_oop, phase)) { // returns false for instances
130 131 132 133 134 135
          result = call->in(TypeFunc::Memory);
        }
      } else if (proj_in->is_Initialize()) {
        AllocateNode* alloc = proj_in->as_Initialize()->allocation();
        // Stop if this is the initialization for the object instance which
        // which contains this memory slice, otherwise skip over it.
136 137 138 139
        if ((alloc == NULL) || (alloc->_idx == instance_id)) {
          break;
        }
        if (is_instance) {
140
          result = proj_in->in(TypeFunc::Memory);
141 142 143 144 145 146
        } else if (is_boxed_value_load) {
          Node* klass = alloc->in(AllocateNode::KlassNode);
          const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
          if (tklass->klass_is_exact() && !tklass->klass()->equals(t_oop->klass())) {
            result = proj_in->in(TypeFunc::Memory); // not related allocation
          }
147 148 149
        }
      } else if (proj_in->is_MemBar()) {
        result = proj_in->in(TypeFunc::Memory);
150 151
      } else {
        assert(false, "unexpected projection");
152
      }
153
    } else if (result->is_ClearArray()) {
154
      if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
155 156 157 158 159
        // Can not bypass initialization of the instance
        // we are looking for.
        break;
      }
      // Otherwise skip it (the call updated 'result' value).
160
    } else if (result->is_MergeMem()) {
161
      result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, NULL, tty);
162 163 164 165 166
    }
  }
  return result;
}

167 168 169 170 171 172
Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
  const TypeOopPtr* t_oop = t_adr->isa_oopptr();
  if (t_oop == NULL)
    return mchain;  // don't try to optimize non-oop types
  Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
  bool is_instance = t_oop->is_known_instance_field();
173 174 175 176 177
  PhaseIterGVN *igvn = phase->is_IterGVN();
  if (is_instance && igvn != NULL  && result->is_Phi()) {
    PhiNode *mphi = result->as_Phi();
    assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
    const TypePtr *t = mphi->adr_type();
178
    if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ||
179
        t->isa_oopptr() && !t->is_oopptr()->is_known_instance() &&
180 181 182
        t->is_oopptr()->cast_to_exactness(true)
         ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
         ->is_oopptr()->cast_to_instance_id(t_oop->instance_id()) == t_oop) {
183 184 185 186 187 188 189 190 191
      // clone the Phi with our address type
      result = mphi->split_out_instance(t_adr, igvn);
    } else {
      assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
    }
  }
  return result;
}

192 193 194 195 196 197 198 199 200 201 202
static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
  uint alias_idx = phase->C->get_alias_index(tp);
  Node *mem = mmem;
#ifdef ASSERT
  {
    // Check that current type is consistent with the alias index used during graph construction
    assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
    bool consistent =  adr_check == NULL || adr_check->empty() ||
                       phase->C->must_alias(adr_check, alias_idx );
    // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
    if( !consistent && adr_check != NULL && !adr_check->empty() &&
203
               tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
        adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
        ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
          adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
          adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
      // don't assert if it is dead code.
      consistent = true;
    }
    if( !consistent ) {
      st->print("alias_idx==%d, adr_check==", alias_idx);
      if( adr_check == NULL ) {
        st->print("NULL");
      } else {
        adr_check->dump();
      }
      st->cr();
      print_alias_types();
      assert(consistent, "adr_check must match alias idx");
    }
  }
#endif
224
  // TypeOopPtr::NOTNULL+any is an OOP with unknown offset - generally
225 226
  // means an array I have not precisely typed yet.  Do not do any
  // alias stuff with it any time soon.
227
  const TypeOopPtr *toop = tp->isa_oopptr();
228
  if( tp->base() != Type::AnyPtr &&
229 230 231 232
      !(toop &&
        toop->klass() != NULL &&
        toop->klass()->is_java_lang_Object() &&
        toop->offset() == Type::OffsetBot) ) {
233 234 235 236
    // compress paths and change unreachable cycles to TOP
    // If not, we can update the input infinitely along a MergeMem cycle
    // Equivalent code in PhiNode::Ideal
    Node* m  = phase->transform(mmem);
T
twisti 已提交
237
    // If transformed to a MergeMem, get the desired slice
238 239 240 241 242 243 244
    // Otherwise the returned node represents memory for every slice
    mem = (m->is_MergeMem())? m->as_MergeMem()->memory_at(alias_idx) : m;
    // Update input if it is progress over what we have now
  }
  return mem;
}

D
duke 已提交
245 246 247 248 249 250 251 252
//--------------------------Ideal_common---------------------------------------
// Look for degenerate control and memory inputs.  Bypass MergeMem inputs.
// Unhook non-raw memories from complete (macro-expanded) initializations.
Node *MemNode::Ideal_common(PhaseGVN *phase, bool can_reshape) {
  // If our control input is a dead region, kill all below the region
  Node *ctl = in(MemNode::Control);
  if (ctl && remove_dead_region(phase, can_reshape))
    return this;
253 254
  ctl = in(MemNode::Control);
  // Don't bother trying to transform a dead node
255
  if (ctl && ctl->is_top())  return NodeSentinel;
D
duke 已提交
256

257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
  PhaseIterGVN *igvn = phase->is_IterGVN();
  // Wait if control on the worklist.
  if (ctl && can_reshape && igvn != NULL) {
    Node* bol = NULL;
    Node* cmp = NULL;
    if (ctl->in(0)->is_If()) {
      assert(ctl->is_IfTrue() || ctl->is_IfFalse(), "sanity");
      bol = ctl->in(0)->in(1);
      if (bol->is_Bool())
        cmp = ctl->in(0)->in(1)->in(1);
    }
    if (igvn->_worklist.member(ctl) ||
        (bol != NULL && igvn->_worklist.member(bol)) ||
        (cmp != NULL && igvn->_worklist.member(cmp)) ) {
      // This control path may be dead.
      // Delay this memory node transformation until the control is processed.
      phase->is_IterGVN()->_worklist.push(this);
      return NodeSentinel; // caller will return NULL
    }
  }
D
duke 已提交
277 278
  // Ignore if memory is dead, or self-loop
  Node *mem = in(MemNode::Memory);
279 280
  if (phase->type( mem ) == Type::TOP) return NodeSentinel; // caller will return NULL
  assert(mem != this, "dead loop in MemNode::Ideal");
D
duke 已提交
281

282 283 284 285 286 287 288
  if (can_reshape && igvn != NULL && igvn->_worklist.member(mem)) {
    // This memory slice may be dead.
    // Delay this mem node transformation until the memory is processed.
    phase->is_IterGVN()->_worklist.push(this);
    return NodeSentinel; // caller will return NULL
  }

D
duke 已提交
289
  Node *address = in(MemNode::Address);
290 291
  const Type *t_adr = phase->type(address);
  if (t_adr == Type::TOP)              return NodeSentinel; // caller will return NULL
D
duke 已提交
292

293
  if (can_reshape && igvn != NULL &&
294
      (igvn->_worklist.member(address) ||
295
       igvn->_worklist.size() > 0 && (t_adr != adr_type())) ) {
296 297 298 299 300 301
    // The address's base and type may change when the address is processed.
    // Delay this mem node transformation until the address is processed.
    phase->is_IterGVN()->_worklist.push(this);
    return NodeSentinel; // caller will return NULL
  }

302 303 304
  // Do NOT remove or optimize the next lines: ensure a new alias index
  // is allocated for an oop pointer type before Escape Analysis.
  // Note: C++ will not remove it since the call has side effect.
305
  if (t_adr->isa_oopptr()) {
306 307 308
    int alias_idx = phase->C->get_alias_index(t_adr->is_ptr());
  }

309 310 311 312
#ifdef ASSERT
  Node* base = NULL;
  if (address->is_AddP())
    base = address->in(AddPNode::Base);
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
  if (base != NULL && phase->type(base)->higher_equal(TypePtr::NULL_PTR) &&
      !t_adr->isa_rawptr()) {
    // Note: raw address has TOP base and top->higher_equal(TypePtr::NULL_PTR) is true.
    Compile* C = phase->C;
    tty->cr();
    tty->print_cr("===== NULL+offs not RAW address =====");
    if (C->is_dead_node(this->_idx))    tty->print_cr("'this' is dead");
    if ((ctl != NULL) && C->is_dead_node(ctl->_idx)) tty->print_cr("'ctl' is dead");
    if (C->is_dead_node(mem->_idx))     tty->print_cr("'mem' is dead");
    if (C->is_dead_node(address->_idx)) tty->print_cr("'address' is dead");
    if (C->is_dead_node(base->_idx))    tty->print_cr("'base' is dead");
    tty->cr();
    base->dump(1);
    tty->cr();
    this->dump(2);
    tty->print("this->adr_type():     "); adr_type()->dump(); tty->cr();
    tty->print("phase->type(address): "); t_adr->dump(); tty->cr();
    tty->print("phase->type(base):    "); phase->type(address)->dump(); tty->cr();
    tty->cr();
  }
333 334 335 336
  assert(base == NULL || t_adr->isa_rawptr() ||
        !phase->type(base)->higher_equal(TypePtr::NULL_PTR), "NULL+offs not RAW address?");
#endif

D
duke 已提交
337 338 339
  // Avoid independent memory operations
  Node* old_mem = mem;

340 341 342 343 344 345 346
  // The code which unhooks non-raw memories from complete (macro-expanded)
  // initializations was removed. After macro-expansion all stores catched
  // by Initialize node became raw stores and there is no information
  // which memory slices they modify. So it is unsafe to move any memory
  // operation above these stores. Also in most cases hooked non-raw memories
  // were already unhooked by using information from detect_ptr_independence()
  // and find_previous_store().
D
duke 已提交
347 348 349 350

  if (mem->is_MergeMem()) {
    MergeMemNode* mmem = mem->as_MergeMem();
    const TypePtr *tp = t_adr->is_ptr();
351 352

    mem = step_through_mergemem(phase, mmem, tp, adr_type(), tty);
D
duke 已提交
353 354 355 356
  }

  if (mem != old_mem) {
    set_req(MemNode::Memory, mem);
357 358 359
    if (can_reshape && old_mem->outcnt() == 0) {
        igvn->_worklist.push(old_mem);
    }
360
    if (phase->type( mem ) == Type::TOP) return NodeSentinel;
D
duke 已提交
361 362 363 364 365 366 367 368
    return this;
  }

  // let the subclass continue analyzing...
  return NULL;
}

// Helper function for proving some simple control dominations.
369
// Attempt to prove that all control inputs of 'dom' dominate 'sub'.
D
duke 已提交
370 371 372 373 374
// Already assumes that 'dom' is available at 'sub', and that 'sub'
// is not a constant (dominated by the method's StartNode).
// Used by MemNode::find_previous_store to prove that the
// control input of a memory operation predates (dominates)
// an allocation it wants to look past.
375 376 377 378
bool MemNode::all_controls_dominate(Node* dom, Node* sub) {
  if (dom == NULL || dom->is_top() || sub == NULL || sub->is_top())
    return false; // Conservative answer for dead code

379
  // Check 'dom'. Skip Proj and CatchProj nodes.
380 381 382 383
  dom = dom->find_exact_control(dom);
  if (dom == NULL || dom->is_top())
    return false; // Conservative answer for dead code

384 385 386 387 388 389
  if (dom == sub) {
    // For the case when, for example, 'sub' is Initialize and the original
    // 'dom' is Proj node of the 'sub'.
    return false;
  }

390
  if (dom->is_Con() || dom->is_Start() || dom->is_Root() || dom == sub)
391 392 393 394 395 396
    return true;

  // 'dom' dominates 'sub' if its control edge and control edges
  // of all its inputs dominate or equal to sub's control edge.

  // Currently 'sub' is either Allocate, Initialize or Start nodes.
397 398 399
  // Or Region for the check in LoadNode::Ideal();
  // 'sub' should have sub->in(0) != NULL.
  assert(sub->is_Allocate() || sub->is_Initialize() || sub->is_Start() ||
400
         sub->is_Region() || sub->is_Call(), "expecting only these nodes");
401 402

  // Get control edge of 'sub'.
403
  Node* orig_sub = sub;
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
  sub = sub->find_exact_control(sub->in(0));
  if (sub == NULL || sub->is_top())
    return false; // Conservative answer for dead code

  assert(sub->is_CFG(), "expecting control");

  if (sub == dom)
    return true;

  if (sub->is_Start() || sub->is_Root())
    return false;

  {
    // Check all control edges of 'dom'.

    ResourceMark rm;
    Arena* arena = Thread::current()->resource_area();
    Node_List nlist(arena);
    Unique_Node_List dom_list(arena);

    dom_list.push(dom);
    bool only_dominating_controls = false;

    for (uint next = 0; next < dom_list.size(); next++) {
      Node* n = dom_list.at(next);
429 430
      if (n == orig_sub)
        return false; // One of dom's inputs dominated by sub.
431 432 433 434 435 436
      if (!n->is_CFG() && n->pinned()) {
        // Check only own control edge for pinned non-control nodes.
        n = n->find_exact_control(n->in(0));
        if (n == NULL || n->is_top())
          return false; // Conservative answer for dead code
        assert(n->is_CFG(), "expecting control");
437 438
        dom_list.push(n);
      } else if (n->is_Con() || n->is_Start() || n->is_Root()) {
439 440 441 442 443 444 445 446 447
        only_dominating_controls = true;
      } else if (n->is_CFG()) {
        if (n->dominates(sub, nlist))
          only_dominating_controls = true;
        else
          return false;
      } else {
        // First, own control edge.
        Node* m = n->find_exact_control(n->in(0));
448 449 450 451 452
        if (m != NULL) {
          if (m->is_top())
            return false; // Conservative answer for dead code
          dom_list.push(m);
        }
453 454 455 456 457 458 459
        // Now, the rest of edges.
        uint cnt = n->req();
        for (uint i = 1; i < cnt; i++) {
          m = n->find_exact_control(n->in(i));
          if (m == NULL || m->is_top())
            continue;
          dom_list.push(m);
D
duke 已提交
460 461 462
        }
      }
    }
463
    return only_dominating_controls;
D
duke 已提交
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
  }
}

//---------------------detect_ptr_independence---------------------------------
// Used by MemNode::find_previous_store to prove that two base
// pointers are never equal.
// The pointers are accompanied by their associated allocations,
// if any, which have been previously discovered by the caller.
bool MemNode::detect_ptr_independence(Node* p1, AllocateNode* a1,
                                      Node* p2, AllocateNode* a2,
                                      PhaseTransform* phase) {
  // Attempt to prove that these two pointers cannot be aliased.
  // They may both manifestly be allocations, and they should differ.
  // Or, if they are not both allocations, they can be distinct constants.
  // Otherwise, one is an allocation and the other a pre-existing value.
  if (a1 == NULL && a2 == NULL) {           // neither an allocation
    return (p1 != p2) && p1->is_Con() && p2->is_Con();
  } else if (a1 != NULL && a2 != NULL) {    // both allocations
    return (a1 != a2);
  } else if (a1 != NULL) {                  // one allocation a1
    // (Note:  p2->is_Con implies p2->in(0)->is_Root, which dominates.)
485
    return all_controls_dominate(p2, a1);
D
duke 已提交
486
  } else { //(a2 != NULL)                   // one allocation a2
487
    return all_controls_dominate(p1, a2);
D
duke 已提交
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
  }
  return false;
}


// The logic for reordering loads and stores uses four steps:
// (a) Walk carefully past stores and initializations which we
//     can prove are independent of this load.
// (b) Observe that the next memory state makes an exact match
//     with self (load or store), and locate the relevant store.
// (c) Ensure that, if we were to wire self directly to the store,
//     the optimizer would fold it up somehow.
// (d) Do the rewiring, and return, depending on some other part of
//     the optimizer to fold up the load.
// This routine handles steps (a) and (b).  Steps (c) and (d) are
// specific to loads and stores, so they are handled by the callers.
// (Currently, only LoadNode::Ideal has steps (c), (d).  More later.)
//
Node* MemNode::find_previous_store(PhaseTransform* phase) {
  Node*         ctrl   = in(MemNode::Control);
  Node*         adr    = in(MemNode::Address);
  intptr_t      offset = 0;
  Node*         base   = AddPNode::Ideal_base_and_offset(adr, phase, offset);
  AllocateNode* alloc  = AllocateNode::Ideal_allocation(base, phase);

  if (offset == Type::OffsetBot)
    return NULL;            // cannot unalias unless there are precise offsets

516 517
  const TypeOopPtr *addr_t = adr->bottom_type()->isa_oopptr();

D
duke 已提交
518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572
  intptr_t size_in_bytes = memory_size();

  Node* mem = in(MemNode::Memory);   // start searching here...

  int cnt = 50;             // Cycle limiter
  for (;;) {                // While we can dance past unrelated stores...
    if (--cnt < 0)  break;  // Caught in cycle or a complicated dance?

    if (mem->is_Store()) {
      Node* st_adr = mem->in(MemNode::Address);
      intptr_t st_offset = 0;
      Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);
      if (st_base == NULL)
        break;              // inscrutable pointer
      if (st_offset != offset && st_offset != Type::OffsetBot) {
        const int MAX_STORE = BytesPerLong;
        if (st_offset >= offset + size_in_bytes ||
            st_offset <= offset - MAX_STORE ||
            st_offset <= offset - mem->as_Store()->memory_size()) {
          // Success:  The offsets are provably independent.
          // (You may ask, why not just test st_offset != offset and be done?
          // The answer is that stores of different sizes can co-exist
          // in the same sequence of RawMem effects.  We sometimes initialize
          // a whole 'tile' of array elements with a single jint or jlong.)
          mem = mem->in(MemNode::Memory);
          continue;           // (a) advance through independent store memory
        }
      }
      if (st_base != base &&
          detect_ptr_independence(base, alloc,
                                  st_base,
                                  AllocateNode::Ideal_allocation(st_base, phase),
                                  phase)) {
        // Success:  The bases are provably independent.
        mem = mem->in(MemNode::Memory);
        continue;           // (a) advance through independent store memory
      }

      // (b) At this point, if the bases or offsets do not agree, we lose,
      // since we have not managed to prove 'this' and 'mem' independent.
      if (st_base == base && st_offset == offset) {
        return mem;         // let caller handle steps (c), (d)
      }

    } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
      InitializeNode* st_init = mem->in(0)->as_Initialize();
      AllocateNode*  st_alloc = st_init->allocation();
      if (st_alloc == NULL)
        break;              // something degenerated
      bool known_identical = false;
      bool known_independent = false;
      if (alloc == st_alloc)
        known_identical = true;
      else if (alloc != NULL)
        known_independent = true;
573
      else if (all_controls_dominate(this, st_alloc))
D
duke 已提交
574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595
        known_independent = true;

      if (known_independent) {
        // The bases are provably independent: Either they are
        // manifestly distinct allocations, or else the control
        // of this load dominates the store's allocation.
        int alias_idx = phase->C->get_alias_index(adr_type());
        if (alias_idx == Compile::AliasIdxRaw) {
          mem = st_alloc->in(TypeFunc::Memory);
        } else {
          mem = st_init->memory(alias_idx);
        }
        continue;           // (a) advance through independent store memory
      }

      // (b) at this point, if we are not looking at a store initializing
      // the same allocation we are loading from, we lose.
      if (known_identical) {
        // From caller, can_see_stored_value will consult find_captured_store.
        return mem;         // let caller handle steps (c), (d)
      }

596
    } else if (addr_t != NULL && addr_t->is_known_instance_field()) {
597 598 599 600 601 602 603 604 605 606
      // Can't use optimize_simple_memory_chain() since it needs PhaseGVN.
      if (mem->is_Proj() && mem->in(0)->is_Call()) {
        CallNode *call = mem->in(0)->as_Call();
        if (!call->may_modify(addr_t, phase)) {
          mem = call->in(TypeFunc::Memory);
          continue;         // (a) advance through independent call memory
        }
      } else if (mem->is_Proj() && mem->in(0)->is_MemBar()) {
        mem = mem->in(0)->in(TypeFunc::Memory);
        continue;           // (a) advance through independent MemBar memory
607 608 609 610 611 612 613 614 615
      } else if (mem->is_ClearArray()) {
        if (ClearArrayNode::step_through(&mem, (uint)addr_t->instance_id(), phase)) {
          // (the call updated 'mem' value)
          continue;         // (a) advance through independent allocation memory
        } else {
          // Can not bypass initialization of the instance
          // we are looking for.
          return mem;
        }
616 617 618 619 620
      } else if (mem->is_MergeMem()) {
        int alias_idx = phase->C->get_alias_index(adr_type());
        mem = mem->as_MergeMem()->memory_at(alias_idx);
        continue;           // (a) advance through independent MergeMem memory
      }
D
duke 已提交
621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659
    }

    // Unless there is an explicit 'continue', we must bail out here,
    // because 'mem' is an inscrutable memory state (e.g., a call).
    break;
  }

  return NULL;              // bail out
}

//----------------------calculate_adr_type-------------------------------------
// Helper function.  Notices when the given type of address hits top or bottom.
// Also, asserts a cross-check of the type against the expected address type.
const TypePtr* MemNode::calculate_adr_type(const Type* t, const TypePtr* cross_check) {
  if (t == Type::TOP)  return NULL; // does not touch memory any more?
  #ifdef PRODUCT
  cross_check = NULL;
  #else
  if (!VerifyAliases || is_error_reported() || Node::in_dump())  cross_check = NULL;
  #endif
  const TypePtr* tp = t->isa_ptr();
  if (tp == NULL) {
    assert(cross_check == NULL || cross_check == TypePtr::BOTTOM, "expected memory type must be wide");
    return TypePtr::BOTTOM;           // touches lots of memory
  } else {
    #ifdef ASSERT
    // %%%% [phh] We don't check the alias index if cross_check is
    //            TypeRawPtr::BOTTOM.  Needs to be investigated.
    if (cross_check != NULL &&
        cross_check != TypePtr::BOTTOM &&
        cross_check != TypeRawPtr::BOTTOM) {
      // Recheck the alias index, to see if it has changed (due to a bug).
      Compile* C = Compile::current();
      assert(C->get_alias_index(cross_check) == C->get_alias_index(tp),
             "must stay in the original alias category");
      // The type of the address must be contained in the adr_type,
      // disregarding "null"-ness.
      // (We make an exception for TypeRawPtr::BOTTOM, which is a bit bucket.)
      const TypePtr* tp_notnull = tp->join(TypePtr::NOTNULL)->is_ptr();
660
      assert(cross_check->meet(tp_notnull) == cross_check->remove_speculative(),
D
duke 已提交
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
             "real address must not escape from expected memory type");
    }
    #endif
    return tp;
  }
}

//------------------------adr_phi_is_loop_invariant----------------------------
// A helper function for Ideal_DU_postCCP to check if a Phi in a counted
// loop is loop invariant. Make a quick traversal of Phi and associated
// CastPP nodes, looking to see if they are a closed group within the loop.
bool MemNode::adr_phi_is_loop_invariant(Node* adr_phi, Node* cast) {
  // The idea is that the phi-nest must boil down to only CastPP nodes
  // with the same data. This implies that any path into the loop already
  // includes such a CastPP, and so the original cast, whatever its input,
  // must be covered by an equivalent cast, with an earlier control input.
  ResourceMark rm;

  // The loop entry input of the phi should be the unique dominating
  // node for every Phi/CastPP in the loop.
  Unique_Node_List closure;
  closure.push(adr_phi->in(LoopNode::EntryControl));

  // Add the phi node and the cast to the worklist.
  Unique_Node_List worklist;
  worklist.push(adr_phi);
  if( cast != NULL ){
    if( !cast->is_ConstraintCast() ) return false;
    worklist.push(cast);
  }

  // Begin recursive walk of phi nodes.
  while( worklist.size() ){
    // Take a node off the worklist
    Node *n = worklist.pop();
    if( !closure.member(n) ){
      // Add it to the closure.
      closure.push(n);
      // Make a sanity check to ensure we don't waste too much time here.
      if( closure.size() > 20) return false;
      // This node is OK if:
      //  - it is a cast of an identical value
      //  - or it is a phi node (then we add its inputs to the worklist)
      // Otherwise, the node is not OK, and we presume the cast is not invariant
      if( n->is_ConstraintCast() ){
        worklist.push(n->in(1));
      } else if( n->is_Phi() ) {
        for( uint i = 1; i < n->req(); i++ ) {
          worklist.push(n->in(i));
        }
      } else {
        return false;
      }
    }
  }

  // Quit when the worklist is empty, and we've found no offending nodes.
  return true;
}

//------------------------------Ideal_DU_postCCP-------------------------------
// Find any cast-away of null-ness and keep its control.  Null cast-aways are
// going away in this pass and we need to make this memory op depend on the
// gating null check.
725 726 727
Node *MemNode::Ideal_DU_postCCP( PhaseCCP *ccp ) {
  return Ideal_common_DU_postCCP(ccp, this, in(MemNode::Address));
}
D
duke 已提交
728 729 730 731 732 733 734 735 736

// I tried to leave the CastPP's in.  This makes the graph more accurate in
// some sense; we get to keep around the knowledge that an oop is not-null
// after some test.  Alas, the CastPP's interfere with GVN (some values are
// the regular oop, some are the CastPP of the oop, all merge at Phi's which
// cannot collapse, etc).  This cost us 10% on SpecJVM, even when I removed
// some of the more trivial cases in the optimizer.  Removing more useless
// Phi's started allowing Loads to illegally float above null checks.  I gave
// up on this approach.  CNC 10/20/2000
737 738 739
// This static method may be called not from MemNode (EncodePNode calls it).
// Only the control edge of the node 'n' might be updated.
Node *MemNode::Ideal_common_DU_postCCP( PhaseCCP *ccp, Node* n, Node* adr ) {
D
duke 已提交
740 741 742 743
  Node *skipped_cast = NULL;
  // Need a null check?  Regular static accesses do not because they are
  // from constant addresses.  Array ops are gated by the range check (which
  // always includes a NULL check).  Just check field ops.
744
  if( n->in(MemNode::Control) == NULL ) {
D
duke 已提交
745 746 747 748 749 750 751 752
    // Scan upwards for the highest location we can place this memory op.
    while( true ) {
      switch( adr->Opcode() ) {

      case Op_AddP:             // No change to NULL-ness, so peek thru AddP's
        adr = adr->in(AddPNode::Base);
        continue;

753
      case Op_DecodeN:         // No change to NULL-ness, so peek thru
754
      case Op_DecodeNKlass:
755 756 757
        adr = adr->in(1);
        continue;

758
      case Op_EncodeP:
759
      case Op_EncodePKlass:
760 761 762 763 764 765 766 767 768 769 770 771 772 773 774
        // EncodeP node's control edge could be set by this method
        // when EncodeP node depends on CastPP node.
        //
        // Use its control edge for memory op because EncodeP may go away
        // later when it is folded with following or preceding DecodeN node.
        if (adr->in(0) == NULL) {
          // Keep looking for cast nodes.
          adr = adr->in(1);
          continue;
        }
        ccp->hash_delete(n);
        n->set_req(MemNode::Control, adr->in(0));
        ccp->hash_insert(n);
        return n;

D
duke 已提交
775 776 777 778 779 780 781 782 783 784 785 786
      case Op_CastPP:
        // If the CastPP is useless, just peek on through it.
        if( ccp->type(adr) == ccp->type(adr->in(1)) ) {
          // Remember the cast that we've peeked though. If we peek
          // through more than one, then we end up remembering the highest
          // one, that is, if in a loop, the one closest to the top.
          skipped_cast = adr;
          adr = adr->in(1);
          continue;
        }
        // CastPP is going away in this pass!  We need this memory op to be
        // control-dependent on the test that is guarding the CastPP.
787 788 789 790
        ccp->hash_delete(n);
        n->set_req(MemNode::Control, adr->in(0));
        ccp->hash_insert(n);
        return n;
D
duke 已提交
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820

      case Op_Phi:
        // Attempt to float above a Phi to some dominating point.
        if (adr->in(0) != NULL && adr->in(0)->is_CountedLoop()) {
          // If we've already peeked through a Cast (which could have set the
          // control), we can't float above a Phi, because the skipped Cast
          // may not be loop invariant.
          if (adr_phi_is_loop_invariant(adr, skipped_cast)) {
            adr = adr->in(1);
            continue;
          }
        }

        // Intentional fallthrough!

        // No obvious dominating point.  The mem op is pinned below the Phi
        // by the Phi itself.  If the Phi goes away (no true value is merged)
        // then the mem op can float, but not indefinitely.  It must be pinned
        // behind the controls leading to the Phi.
      case Op_CheckCastPP:
        // These usually stick around to change address type, however a
        // useless one can be elided and we still need to pick up a control edge
        if (adr->in(0) == NULL) {
          // This CheckCastPP node has NO control and is likely useless. But we
          // need check further up the ancestor chain for a control input to keep
          // the node in place. 4959717.
          skipped_cast = adr;
          adr = adr->in(1);
          continue;
        }
821 822 823 824
        ccp->hash_delete(n);
        n->set_req(MemNode::Control, adr->in(0));
        ccp->hash_insert(n);
        return n;
D
duke 已提交
825 826 827 828 829 830

        // List of "safe" opcodes; those that implicitly block the memory
        // op below any null check.
      case Op_CastX2P:          // no null checks on native pointers
      case Op_Parm:             // 'this' pointer is not null
      case Op_LoadP:            // Loading from within a klass
831
      case Op_LoadN:            // Loading from within a klass
D
duke 已提交
832
      case Op_LoadKlass:        // Loading from within a klass
833
      case Op_LoadNKlass:       // Loading from within a klass
D
duke 已提交
834
      case Op_ConP:             // Loading from a klass
835
      case Op_ConN:             // Loading from a klass
836
      case Op_ConNKlass:        // Loading from a klass
D
duke 已提交
837 838 839
      case Op_CreateEx:         // Sucking up the guts of an exception oop
      case Op_Con:              // Reading from TLS
      case Op_CMoveP:           // CMoveP is pinned
840
      case Op_CMoveN:           // CMoveN is pinned
D
duke 已提交
841 842 843 844 845 846 847 848
        break;                  // No progress

      case Op_Proj:             // Direct call to an allocation routine
      case Op_SCMemProj:        // Memory state from store conditional ops
#ifdef ASSERT
        {
          assert(adr->as_Proj()->_con == TypeFunc::Parms, "must be return value");
          const Node* call = adr->in(0);
849 850
          if (call->is_CallJava()) {
            const CallJavaNode* call_java = call->as_CallJava();
851 852 853 854
            const TypeTuple *r = call_java->tf()->range();
            assert(r->cnt() > TypeFunc::Parms, "must return value");
            const Type* ret_type = r->field_at(TypeFunc::Parms);
            assert(ret_type && ret_type->isa_ptr(), "must return pointer");
D
duke 已提交
855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883
            // We further presume that this is one of
            // new_instance_Java, new_array_Java, or
            // the like, but do not assert for this.
          } else if (call->is_Allocate()) {
            // similar case to new_instance_Java, etc.
          } else if (!call->is_CallLeaf()) {
            // Projections from fetch_oop (OSR) are allowed as well.
            ShouldNotReachHere();
          }
        }
#endif
        break;
      default:
        ShouldNotReachHere();
      }
      break;
    }
  }

  return  NULL;               // No progress
}


//=============================================================================
uint LoadNode::size_of() const { return sizeof(*this); }
uint LoadNode::cmp( const Node &n ) const
{ return !Type::cmp( _type, ((LoadNode&)n)._type ); }
const Type *LoadNode::bottom_type() const { return _type; }
uint LoadNode::ideal_reg() const {
884
  return _type->ideal_reg();
D
duke 已提交
885 886 887 888 889 890 891 892 893 894 895 896
}

#ifndef PRODUCT
void LoadNode::dump_spec(outputStream *st) const {
  MemNode::dump_spec(st);
  if( !Verbose && !WizardMode ) {
    // standard dump does this in Verbose and WizardMode
    st->print(" #"); _type->dump_on(st);
  }
}
#endif

897 898 899 900 901 902 903 904 905 906
#ifdef ASSERT
//----------------------------is_immutable_value-------------------------------
// Helper function to allow a raw load without control edge for some cases
bool LoadNode::is_immutable_value(Node* adr) {
  return (adr->is_AddP() && adr->in(AddPNode::Base)->is_top() &&
          adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
          (adr->in(AddPNode::Offset)->find_intptr_t_con(-1) ==
           in_bytes(JavaThread::osthread_offset())));
}
#endif
D
duke 已提交
907 908 909

//----------------------------LoadNode::make-----------------------------------
// Polymorphic factory method:
910
Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypePtr* adr_type, const Type *rt, BasicType bt, MemOrd mo) {
911 912
  Compile* C = gvn.C;

D
duke 已提交
913 914 915 916 917 918 919
  // sanity check the alias category against the created node type
  assert(!(adr_type->isa_oopptr() &&
           adr_type->offset() == oopDesc::klass_offset_in_bytes()),
         "use LoadKlassNode instead");
  assert(!(adr_type->isa_aryptr() &&
           adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
         "use LoadRangeNode instead");
920 921 922 923 924
  // Check control edge of raw loads
  assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
          // oop will be recorded in oop map if load crosses safepoint
          rt->isa_oopptr() || is_immutable_value(adr),
          "raw memory operations should have control edge");
D
duke 已提交
925
  switch (bt) {
926 927 928 929 930 931 932 933 934
  case T_BOOLEAN: return new (C) LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(),  mo);
  case T_BYTE:    return new (C) LoadBNode (ctl, mem, adr, adr_type, rt->is_int(),  mo);
  case T_INT:     return new (C) LoadINode (ctl, mem, adr, adr_type, rt->is_int(),  mo);
  case T_CHAR:    return new (C) LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(),  mo);
  case T_SHORT:   return new (C) LoadSNode (ctl, mem, adr, adr_type, rt->is_int(),  mo);
  case T_LONG:    return new (C) LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo);
  case T_FLOAT:   return new (C) LoadFNode (ctl, mem, adr, adr_type, rt,            mo);
  case T_DOUBLE:  return new (C) LoadDNode (ctl, mem, adr, adr_type, rt,            mo);
  case T_ADDRESS: return new (C) LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(),  mo);
935 936
  case T_OBJECT:
#ifdef _LP64
937
    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
938
      Node* load  = gvn.transform(new (C) LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo));
939
      return new (C) DecodeNNode(load, load->bottom_type()->make_ptr());
940 941
    } else
#endif
942
    {
943
      assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
944
      return new (C) LoadPNode(ctl, mem, adr, adr_type, rt->is_oopptr(), mo);
945
    }
D
duke 已提交
946 947 948 949 950
  }
  ShouldNotReachHere();
  return (LoadNode*)NULL;
}

951
LoadLNode* LoadLNode::make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo) {
D
duke 已提交
952
  bool require_atomic = true;
953
  return new (C) LoadLNode(ctl, mem, adr, adr_type, rt->is_long(), mo, require_atomic);
D
duke 已提交
954 955 956 957 958 959 960 961 962 963 964
}




//------------------------------hash-------------------------------------------
uint LoadNode::hash() const {
  // unroll addition of interesting fields
  return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
}

965 966 967 968 969 970 971 972 973 974 975 976 977
static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
  if ((atp != NULL) && (atp->index() >= Compile::AliasIdxRaw)) {
    bool non_volatile = (atp->field() != NULL) && !atp->field()->is_volatile();
    bool is_stable_ary = FoldStableValues &&
                         (tp != NULL) && (tp->isa_aryptr() != NULL) &&
                         tp->isa_aryptr()->is_stable();

    return (eliminate_boxing && non_volatile) || is_stable_ary;
  }

  return false;
}

D
duke 已提交
978 979 980 981 982 983 984 985 986
//---------------------------can_see_stored_value------------------------------
// This routine exists to make sure this set of tests is done the same
// everywhere.  We need to make a coordinated change: first LoadNode::Ideal
// will change the graph shape in a way which makes memory alive twice at the
// same time (uses the Oracle model of aliasing), then some
// LoadXNode::Identity will fold things back to the equivalence-class model
// of aliasing.
Node* MemNode::can_see_stored_value(Node* st, PhaseTransform* phase) const {
  Node* ld_adr = in(MemNode::Address);
987 988
  intptr_t ld_off = 0;
  AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
989
  const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
990 991
  Compile::AliasType* atp = (tp != NULL) ? phase->C->alias_type(tp) : NULL;
  // This is more general than load from boxing objects.
992
  if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
993
    uint alias_idx = atp->index();
994
    bool final = !atp->is_rewritable();
995 996 997 998 999 1000 1001 1002 1003 1004
    Node* result = NULL;
    Node* current = st;
    // Skip through chains of MemBarNodes checking the MergeMems for
    // new states for the slice of this load.  Stop once any other
    // kind of node is encountered.  Loads from final memory can skip
    // through any kind of MemBar but normal loads shouldn't skip
    // through MemBarAcquire since the could allow them to move out of
    // a synchronized region.
    while (current->is_Proj()) {
      int opc = current->in(0)->Opcode();
1005 1006 1007 1008 1009 1010 1011
      if ((final && (opc == Op_MemBarAcquire ||
                     opc == Op_MemBarAcquireLock ||
                     opc == Op_LoadFence)) ||
          opc == Op_MemBarRelease ||
          opc == Op_StoreFence ||
          opc == Op_MemBarReleaseLock ||
          opc == Op_MemBarCPUOrder) {
1012 1013 1014 1015 1016 1017
        Node* mem = current->in(0)->in(TypeFunc::Memory);
        if (mem->is_MergeMem()) {
          MergeMemNode* merge = mem->as_MergeMem();
          Node* new_st = merge->memory_at(alias_idx);
          if (new_st == merge->base_memory()) {
            // Keep searching
1018
            current = new_st;
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
            continue;
          }
          // Save the new memory state for the slice and fall through
          // to exit.
          result = new_st;
        }
      }
      break;
    }
    if (result != NULL) {
      st = result;
    }
  }

D
duke 已提交
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
  // Loop around twice in the case Load -> Initialize -> Store.
  // (See PhaseIterGVN::add_users_to_worklist, which knows about this case.)
  for (int trip = 0; trip <= 1; trip++) {

    if (st->is_Store()) {
      Node* st_adr = st->in(MemNode::Address);
      if (!phase->eqv(st_adr, ld_adr)) {
        // Try harder before giving up...  Match raw and non-raw pointers.
        intptr_t st_off = 0;
        AllocateNode* alloc = AllocateNode::Ideal_allocation(st_adr, phase, st_off);
        if (alloc == NULL)       return NULL;
1044
        if (alloc != ld_alloc)   return NULL;
D
duke 已提交
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
        if (ld_off != st_off)    return NULL;
        // At this point we have proven something like this setup:
        //  A = Allocate(...)
        //  L = LoadQ(,  AddP(CastPP(, A.Parm),, #Off))
        //  S = StoreQ(, AddP(,        A.Parm  , #Off), V)
        // (Actually, we haven't yet proven the Q's are the same.)
        // In other words, we are loading from a casted version of
        // the same pointer-and-offset that we stored to.
        // Thus, we are able to replace L by V.
      }
      // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
      if (store_Opcode() != st->Opcode())
        return NULL;
      return st->in(MemNode::ValueIn);
    }

    // A load from a freshly-created object always returns zero.
    // (This can happen after LoadNode::Ideal resets the load's memory input
    // to find_captured_store, which returned InitializeNode::zero_memory.)
    if (st->is_Proj() && st->in(0)->is_Allocate() &&
1065 1066
        (st->in(0) == ld_alloc) &&
        (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
D
duke 已提交
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
      // return a zero value for the load's basic type
      // (This is one of the few places where a generic PhaseTransform
      // can create new nodes.  Think of it as lazily manifesting
      // virtually pre-existing constants.)
      return phase->zerocon(memory_type());
    }

    // A load from an initialization barrier can match a captured store.
    if (st->is_Proj() && st->in(0)->is_Initialize()) {
      InitializeNode* init = st->in(0)->as_Initialize();
      AllocateNode* alloc = init->allocation();
1078
      if ((alloc != NULL) && (alloc == ld_alloc)) {
D
duke 已提交
1079
        // examine a captured store value
1080
        st = init->find_captured_store(ld_off, memory_size(), phase);
D
duke 已提交
1081 1082 1083 1084 1085
        if (st != NULL)
          continue;             // take one more trip around
      }
    }

1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
    // Load boxed value from result of valueOf() call is input parameter.
    if (this->is_Load() && ld_adr->is_AddP() &&
        (tp != NULL) && tp->is_ptr_to_boxed_value()) {
      intptr_t ignore = 0;
      Node* base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ignore);
      if (base != NULL && base->is_Proj() &&
          base->as_Proj()->_con == TypeFunc::Parms &&
          base->in(0)->is_CallStaticJava() &&
          base->in(0)->as_CallStaticJava()->is_boxing_method()) {
        return base->in(0)->in(TypeFunc::Parms);
      }
    }

D
duke 已提交
1099 1100 1101 1102 1103 1104
    break;
  }

  return NULL;
}

1105 1106
//----------------------is_instance_field_load_with_local_phi------------------
bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1107 1108 1109 1110 1111 1112 1113
  if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
      in(Address)->is_AddP() ) {
    const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
    // Only instances and boxed values.
    if( t_oop != NULL &&
        (t_oop->is_ptr_to_boxed_value() ||
         t_oop->is_known_instance_field()) &&
1114 1115 1116 1117 1118 1119 1120 1121
        t_oop->offset() != Type::OffsetBot &&
        t_oop->offset() != Type::OffsetTop) {
      return true;
    }
  }
  return false;
}

D
duke 已提交
1122 1123 1124 1125 1126
//------------------------------Identity---------------------------------------
// Loads are identity if previous store is to same address
Node *LoadNode::Identity( PhaseTransform *phase ) {
  // If the previous store-maker is the right kind of Store, and the store is
  // to the same address, then we are equal to the value stored.
1127
  Node* mem = in(Memory);
D
duke 已提交
1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
  Node* value = can_see_stored_value(mem, phase);
  if( value ) {
    // byte, short & char stores truncate naturally.
    // A load has to load the truncated value which requires
    // some sort of masking operation and that requires an
    // Ideal call instead of an Identity call.
    if (memory_size() < BytesPerInt) {
      // If the input to the store does not fit with the load's result type,
      // it must be truncated via an Ideal call.
      if (!phase->type(value)->higher_equal(phase->type(this)))
        return this;
    }
    // (This works even when value is a Con, but LoadNode::Value
    // usually runs first, producing the singleton type of the Con.)
    return value;
  }
1144 1145

  // Search for an existing data phi which was generated before for the same
T
twisti 已提交
1146
  // instance's field to avoid infinite generation of phis in a loop.
1147 1148
  Node *region = mem->in(0);
  if (is_instance_field_load_with_local_phi(region)) {
1149
    const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1150 1151
    int this_index  = phase->C->get_alias_index(addr_t);
    int this_offset = addr_t->offset();
1152 1153 1154 1155 1156 1157 1158 1159
    int this_iid    = addr_t->instance_id();
    if (!addr_t->is_known_instance() &&
         addr_t->is_ptr_to_boxed_value()) {
      // Use _idx of address base (could be Phi node) for boxed values.
      intptr_t   ignore = 0;
      Node*      base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
      this_iid = base->_idx;
    }
1160 1161 1162 1163
    const Type* this_type = bottom_type();
    for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
      Node* phi = region->fast_out(i);
      if (phi->is_Phi() && phi != mem &&
1164
          phi->as_Phi()->is_same_inst_field(this_type, this_iid, this_index, this_offset)) {
1165 1166 1167 1168 1169
        return phi;
      }
    }
  }

D
duke 已提交
1170 1171 1172
  return this;
}

1173 1174 1175 1176 1177 1178
// We're loading from an object which has autobox behaviour.
// If this object is result of a valueOf call we'll have a phi
// merging a newly allocated object and a load from the cache.
// We want to replace this load with the original incoming
// argument to the valueOf call.
Node* LoadNode::eliminate_autobox(PhaseGVN* phase) {
1179 1180 1181 1182 1183 1184 1185 1186
  assert(phase->C->eliminate_boxing(), "sanity");
  intptr_t ignore = 0;
  Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
  if ((base == NULL) || base->is_Phi()) {
    // Push the loads from the phi that comes from valueOf up
    // through it to allow elimination of the loads and the recovery
    // of the original value. It is done in split_through_phi().
    return NULL;
1187 1188
  } else if (base->is_Load() ||
             base->is_DecodeN() && base->in(1)->is_Load()) {
1189 1190 1191 1192 1193
    // Eliminate the load of boxed value for integer types from the cache
    // array by deriving the value from the index into the array.
    // Capture the offset of the load and then reverse the computation.

    // Get LoadN node which loads a boxing object from 'cache' array.
1194 1195 1196
    if (base->is_DecodeN()) {
      base = base->in(1);
    }
1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
    if (!base->in(Address)->is_AddP()) {
      return NULL; // Complex address
    }
    AddPNode* address = base->in(Address)->as_AddP();
    Node* cache_base = address->in(AddPNode::Base);
    if ((cache_base != NULL) && cache_base->is_DecodeN()) {
      // Get ConP node which is static 'cache' field.
      cache_base = cache_base->in(1);
    }
    if ((cache_base != NULL) && cache_base->is_Con()) {
      const TypeAryPtr* base_type = cache_base->bottom_type()->isa_aryptr();
      if ((base_type != NULL) && base_type->is_autobox_cache()) {
1209
        Node* elements[4];
1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261
        int shift = exact_log2(type2aelembytes(T_OBJECT));
        int count = address->unpack_offsets(elements, ARRAY_SIZE(elements));
        if ((count >  0) && elements[0]->is_Con() &&
            ((count == 1) ||
             (count == 2) && elements[1]->Opcode() == Op_LShiftX &&
                             elements[1]->in(2) == phase->intcon(shift))) {
          ciObjArray* array = base_type->const_oop()->as_obj_array();
          // Fetch the box object cache[0] at the base of the array and get its value
          ciInstance* box = array->obj_at(0)->as_instance();
          ciInstanceKlass* ik = box->klass()->as_instance_klass();
          assert(ik->is_box_klass(), "sanity");
          assert(ik->nof_nonstatic_fields() == 1, "change following code");
          if (ik->nof_nonstatic_fields() == 1) {
            // This should be true nonstatic_field_at requires calling
            // nof_nonstatic_fields so check it anyway
            ciConstant c = box->field_value(ik->nonstatic_field_at(0));
            BasicType bt = c.basic_type();
            // Only integer types have boxing cache.
            assert(bt == T_BOOLEAN || bt == T_CHAR  ||
                   bt == T_BYTE    || bt == T_SHORT ||
                   bt == T_INT     || bt == T_LONG, err_msg_res("wrong type = %s", type2name(bt)));
            jlong cache_low = (bt == T_LONG) ? c.as_long() : c.as_int();
            if (cache_low != (int)cache_low) {
              return NULL; // should not happen since cache is array indexed by value
            }
            jlong offset = arrayOopDesc::base_offset_in_bytes(T_OBJECT) - (cache_low << shift);
            if (offset != (int)offset) {
              return NULL; // should not happen since cache is array indexed by value
            }
           // Add up all the offsets making of the address of the load
            Node* result = elements[0];
            for (int i = 1; i < count; i++) {
              result = phase->transform(new (phase->C) AddXNode(result, elements[i]));
            }
            // Remove the constant offset from the address and then
            result = phase->transform(new (phase->C) AddXNode(result, phase->MakeConX(-(int)offset)));
            // remove the scaling of the offset to recover the original index.
            if (result->Opcode() == Op_LShiftX && result->in(2) == phase->intcon(shift)) {
              // Peel the shift off directly but wrap it in a dummy node
              // since Ideal can't return existing nodes
              result = new (phase->C) RShiftXNode(result->in(1), phase->intcon(0));
            } else if (result->is_Add() && result->in(2)->is_Con() &&
                       result->in(1)->Opcode() == Op_LShiftX &&
                       result->in(1)->in(2) == phase->intcon(shift)) {
              // We can't do general optimization: ((X<<Z) + Y) >> Z ==> X + (Y>>Z)
              // but for boxing cache access we know that X<<Z will not overflow
              // (there is range check) so we do this optimizatrion by hand here.
              Node* add_con = new (phase->C) RShiftXNode(result->in(2), phase->intcon(shift));
              result = new (phase->C) AddXNode(result->in(1)->in(1), phase->transform(add_con));
            } else {
              result = new (phase->C) RShiftXNode(result, phase->intcon(shift));
            }
1262
#ifdef _LP64
1263 1264 1265 1266 1267 1268 1269
            if (bt != T_LONG) {
              result = new (phase->C) ConvL2INode(phase->transform(result));
            }
#else
            if (bt == T_LONG) {
              result = new (phase->C) ConvI2LNode(phase->transform(result));
            }
1270
#endif
1271 1272
            return result;
          }
1273 1274 1275 1276 1277 1278 1279
        }
      }
    }
  }
  return NULL;
}

1280 1281
static bool stable_phi(PhiNode* phi, PhaseGVN *phase) {
  Node* region = phi->in(0);
1282
  if (region == NULL) {
1283
    return false; // Wait stable graph
1284
  }
1285
  uint cnt = phi->req();
1286 1287 1288
  for (uint i = 1; i < cnt; i++) {
    Node* rc = region->in(i);
    if (rc == NULL || phase->type(rc) == Type::TOP)
1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321
      return false; // Wait stable graph
    Node* in = phi->in(i);
    if (in == NULL || phase->type(in) == Type::TOP)
      return false; // Wait stable graph
  }
  return true;
}
//------------------------------split_through_phi------------------------------
// Split instance or boxed field load through Phi.
Node *LoadNode::split_through_phi(PhaseGVN *phase) {
  Node* mem     = in(Memory);
  Node* address = in(Address);
  const TypeOopPtr *t_oop = phase->type(address)->isa_oopptr();

  assert((t_oop != NULL) &&
         (t_oop->is_known_instance_field() ||
          t_oop->is_ptr_to_boxed_value()), "invalide conditions");

  Compile* C = phase->C;
  intptr_t ignore = 0;
  Node*    base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
  bool base_is_phi = (base != NULL) && base->is_Phi();
  bool load_boxed_values = t_oop->is_ptr_to_boxed_value() && C->aggressive_unboxing() &&
                           (base != NULL) && (base == address->in(AddPNode::Base)) &&
                           phase->type(base)->higher_equal(TypePtr::NOTNULL);

  if (!((mem->is_Phi() || base_is_phi) &&
        (load_boxed_values || t_oop->is_known_instance_field()))) {
    return NULL; // memory is not Phi
  }

  if (mem->is_Phi()) {
    if (!stable_phi(mem->as_Phi(), phase)) {
1322
      return NULL; // Wait stable graph
1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334
    }
    uint cnt = mem->req();
    // Check for loop invariant memory.
    if (cnt == 3) {
      for (uint i = 1; i < cnt; i++) {
        Node* in = mem->in(i);
        Node*  m = optimize_memory_chain(in, t_oop, this, phase);
        if (m == mem) {
          set_req(Memory, mem->in(cnt - i));
          return this; // made change
        }
      }
1335 1336
    }
  }
1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
  if (base_is_phi) {
    if (!stable_phi(base->as_Phi(), phase)) {
      return NULL; // Wait stable graph
    }
    uint cnt = base->req();
    // Check for loop invariant memory.
    if (cnt == 3) {
      for (uint i = 1; i < cnt; i++) {
        if (base->in(i) == base) {
          return NULL; // Wait stable graph
        }
1348 1349 1350
      }
    }
  }
1351 1352 1353

  bool load_boxed_phi = load_boxed_values && base_is_phi && (base->in(0) == mem->in(0));

1354
  // Split through Phi (see original code in loopopts.cpp).
1355
  assert(C->have_alias_type(t_oop), "instance should have alias type");
1356 1357 1358

  // Do nothing here if Identity will find a value
  // (to avoid infinite chain of value phis generation).
1359
  if (!phase->eqv(this, this->Identity(phase)))
1360 1361
    return NULL;

1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388
  // Select Region to split through.
  Node* region;
  if (!base_is_phi) {
    assert(mem->is_Phi(), "sanity");
    region = mem->in(0);
    // Skip if the region dominates some control edge of the address.
    if (!MemNode::all_controls_dominate(address, region))
      return NULL;
  } else if (!mem->is_Phi()) {
    assert(base_is_phi, "sanity");
    region = base->in(0);
    // Skip if the region dominates some control edge of the memory.
    if (!MemNode::all_controls_dominate(mem, region))
      return NULL;
  } else if (base->in(0) != mem->in(0)) {
    assert(base_is_phi && mem->is_Phi(), "sanity");
    if (MemNode::all_controls_dominate(mem, base->in(0))) {
      region = base->in(0);
    } else if (MemNode::all_controls_dominate(address, mem->in(0))) {
      region = mem->in(0);
    } else {
      return NULL; // complex graph
    }
  } else {
    assert(base->in(0) == mem->in(0), "sanity");
    region = mem->in(0);
  }
1389 1390

  const Type* this_type = this->bottom_type();
1391 1392 1393 1394 1395 1396 1397 1398 1399
  int this_index  = C->get_alias_index(t_oop);
  int this_offset = t_oop->offset();
  int this_iid    = t_oop->instance_id();
  if (!t_oop->is_known_instance() && load_boxed_values) {
    // Use _idx of address base for boxed values.
    this_iid = base->_idx;
  }
  PhaseIterGVN* igvn = phase->is_IterGVN();
  Node* phi = new (C) PhiNode(region, this_type, NULL, this_iid, this_index, this_offset);
1400
  for (uint i = 1; i < region->req(); i++) {
1401
    Node* x;
1402
    Node* the_clone = NULL;
1403 1404
    if (region->in(i) == C->top()) {
      x = C->top();      // Dead path?  Use a dead data op
1405 1406 1407 1408
    } else {
      x = this->clone();        // Else clone up the data op
      the_clone = x;            // Remember for possible deletion.
      // Alter data node to use pre-phi inputs
1409 1410
      if (this->in(0) == region) {
        x->set_req(0, region->in(i));
1411
      } else {
1412
        x->set_req(0, NULL);
1413
      }
1414 1415 1416 1417 1418 1419 1420 1421 1422 1423
      if (mem->is_Phi() && (mem->in(0) == region)) {
        x->set_req(Memory, mem->in(i)); // Use pre-Phi input for the clone.
      }
      if (address->is_Phi() && address->in(0) == region) {
        x->set_req(Address, address->in(i)); // Use pre-Phi input for the clone
      }
      if (base_is_phi && (base->in(0) == region)) {
        Node* base_x = base->in(i); // Clone address for loads from boxed objects.
        Node* adr_x = phase->transform(new (C) AddPNode(base_x,base_x,address->in(AddPNode::Offset)));
        x->set_req(Address, adr_x);
1424 1425 1426 1427 1428 1429 1430 1431
      }
    }
    // Check for a 'win' on some paths
    const Type *t = x->Value(igvn);

    bool singleton = t->singleton();

    // See comments in PhaseIdealLoop::split_thru_phi().
1432
    if (singleton && t == Type::TOP) {
1433 1434 1435
      singleton &= region->is_Loop() && (i != LoopNode::EntryControl);
    }

1436
    if (singleton) {
1437 1438 1439 1440 1441 1442 1443 1444
      x = igvn->makecon(t);
    } else {
      // We now call Identity to try to simplify the cloned node.
      // Note that some Identity methods call phase->type(this).
      // Make sure that the type array is big enough for
      // our new node, even though we may throw the node away.
      // (This tweaking with igvn only works because x is a new node.)
      igvn->set_type(x, t);
1445
      // If x is a TypeNode, capture any more-precise type permanently into Node
T
twisti 已提交
1446
      // otherwise it will be not updated during igvn->transform since
1447 1448
      // igvn->type(x) is set to x->Value() already.
      x->raise_bottom_type(t);
1449
      Node *y = x->Identity(igvn);
1450
      if (y != x) {
1451 1452
        x = y;
      } else {
1453
        y = igvn->hash_find_insert(x);
1454
        if (y) {
1455 1456 1457 1458 1459 1460 1461 1462 1463
          x = y;
        } else {
          // Else x is a new node we are keeping
          // We do not need register_new_node_with_optimizer
          // because set_type has already been called.
          igvn->_worklist.push(x);
        }
      }
    }
1464
    if (x != the_clone && the_clone != NULL) {
1465
      igvn->remove_dead_node(the_clone);
1466
    }
1467 1468
    phi->set_req(i, x);
  }
1469 1470 1471
  // Record Phi
  igvn->register_new_node_with_optimizer(phi);
  return phi;
1472
}
1473

D
duke 已提交
1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
//------------------------------Ideal------------------------------------------
// If the load is from Field memory and the pointer is non-null, we can
// zero out the control input.
// If the offset is constant and the base is an object allocation,
// try to hook me up to the exact initializing store.
Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  Node* p = MemNode::Ideal_common(phase, can_reshape);
  if (p)  return (p == NodeSentinel) ? NULL : p;

  Node* ctrl    = in(MemNode::Control);
  Node* address = in(MemNode::Address);

  // Skip up past a SafePoint control.  Cannot do this for Stores because
  // pointer stores & cardmarks must stay on the same side of a SafePoint.
  if( ctrl != NULL && ctrl->Opcode() == Op_SafePoint &&
      phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw ) {
    ctrl = ctrl->in(0);
    set_req(MemNode::Control,ctrl);
  }

1494 1495 1496 1497 1498 1499
  intptr_t ignore = 0;
  Node*    base   = AddPNode::Ideal_base_and_offset(address, phase, ignore);
  if (base != NULL
      && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
    // Check for useless control edge in some common special cases
    if (in(MemNode::Control) != NULL
D
duke 已提交
1500
        && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1501
        && all_controls_dominate(base, phase->C->start())) {
D
duke 已提交
1502 1503 1504
      // A method-invariant, non-null address (constant or 'this' argument).
      set_req(MemNode::Control, NULL);
    }
1505 1506
  }

1507 1508 1509
  Node* mem = in(MemNode::Memory);
  const TypePtr *addr_t = phase->type(address)->isa_ptr();

1510
  if (can_reshape && (addr_t != NULL)) {
1511
    // try to optimize our memory input
1512
    Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1513 1514
    if (opt_mem != mem) {
      set_req(MemNode::Memory, opt_mem);
1515
      if (phase->type( opt_mem ) == Type::TOP) return NULL;
1516 1517 1518
      return this;
    }
    const TypeOopPtr *t_oop = addr_t->isa_oopptr();
1519 1520 1521
    if ((t_oop != NULL) &&
        (t_oop->is_known_instance_field() ||
         t_oop->is_ptr_to_boxed_value())) {
1522 1523 1524 1525 1526 1527
      PhaseIterGVN *igvn = phase->is_IterGVN();
      if (igvn != NULL && igvn->_worklist.member(opt_mem)) {
        // Delay this transformation until memory Phi is processed.
        phase->is_IterGVN()->_worklist.push(this);
        return NULL;
      }
1528 1529 1530
      // Split instance field load through Phi.
      Node* result = split_through_phi(phase);
      if (result != NULL) return result;
1531 1532 1533 1534 1535

      if (t_oop->is_ptr_to_boxed_value()) {
        Node* result = eliminate_autobox(phase);
        if (result != NULL) return result;
      }
1536 1537 1538
    }
  }

D
duke 已提交
1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554
  // Check for prior store with a different base or offset; make Load
  // independent.  Skip through any number of them.  Bail out if the stores
  // are in an endless dead cycle and report no progress.  This is a key
  // transform for Reflection.  However, if after skipping through the Stores
  // we can't then fold up against a prior store do NOT do the transform as
  // this amounts to using the 'Oracle' model of aliasing.  It leaves the same
  // array memory alive twice: once for the hoisted Load and again after the
  // bypassed Store.  This situation only works if EVERYBODY who does
  // anti-dependence work knows how to bypass.  I.e. we need all
  // anti-dependence checks to ask the same Oracle.  Right now, that Oracle is
  // the alias index stuff.  So instead, peek through Stores and IFF we can
  // fold up, do so.
  Node* prev_mem = find_previous_store(phase);
  // Steps (a), (b):  Walk past independent stores to find an exact match.
  if (prev_mem != NULL && prev_mem != in(MemNode::Memory)) {
    // (c) See if we can fold up on the spot, but don't fold up here.
1555
    // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
D
duke 已提交
1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571
    // just return a prior value, which is done by Identity calls.
    if (can_see_stored_value(prev_mem, phase)) {
      // Make ready for step (d):
      set_req(MemNode::Memory, prev_mem);
      return this;
    }
  }

  return NULL;                  // No further progress
}

// Helper to recognize certain Klass fields which are invariant across
// some group of array types (e.g., int[] or all T[] where T < Object).
const Type*
LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
                                 ciKlass* klass) const {
1572
  if (tkls->offset() == in_bytes(Klass::modifier_flags_offset())) {
D
duke 已提交
1573 1574 1575 1576 1577
    // The field is Klass::_modifier_flags.  Return its (constant) value.
    // (Folds up the 2nd indirection in aClassConstant.getModifiers().)
    assert(this->Opcode() == Op_LoadI, "must load an int from _modifier_flags");
    return TypeInt::make(klass->modifier_flags());
  }
1578
  if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
D
duke 已提交
1579 1580 1581 1582 1583
    // The field is Klass::_access_flags.  Return its (constant) value.
    // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
    assert(this->Opcode() == Op_LoadI, "must load an int from _access_flags");
    return TypeInt::make(klass->access_flags());
  }
1584
  if (tkls->offset() == in_bytes(Klass::layout_helper_offset())) {
D
duke 已提交
1585 1586 1587 1588 1589 1590 1591 1592 1593
    // The field is Klass::_layout_helper.  Return its constant value if known.
    assert(this->Opcode() == Op_LoadI, "must load an int from _layout_helper");
    return TypeInt::make(klass->layout_helper());
  }

  // No match.
  return NULL;
}

1594 1595
// Try to constant-fold a stable array element.
static const Type* fold_stable_ary_elem(const TypeAryPtr* ary, int off, BasicType loadbt) {
1596
  assert(ary->const_oop(), "array should be constant");
1597 1598
  assert(ary->is_stable(), "array should be stable");

1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613
  // Decode the results of GraphKit::array_element_address.
  ciArray* aobj = ary->const_oop()->as_array();
  ciConstant con = aobj->element_value_by_offset(off);

  if (con.basic_type() != T_ILLEGAL && !con.is_null_or_zero()) {
    const Type* con_type = Type::make_from_constant(con);
    if (con_type != NULL) {
      if (con_type->isa_aryptr()) {
        // Join with the array element type, in case it is also stable.
        int dim = ary->stable_dimension();
        con_type = con_type->is_aryptr()->cast_to_stable(true, dim-1);
      }
      if (loadbt == T_NARROWOOP && con_type->isa_oopptr()) {
        con_type = con_type->make_narrowoop();
      }
1614
#ifndef PRODUCT
1615 1616 1617
      if (TraceIterativeGVN) {
        tty->print("FoldStableValues: array element [off=%d]: con_type=", off);
        con_type->dump(); tty->cr();
1618
      }
1619 1620
#endif //PRODUCT
      return con_type;
1621 1622 1623 1624 1625
    }
  }
  return NULL;
}

D
duke 已提交
1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636
//------------------------------Value-----------------------------------------
const Type *LoadNode::Value( PhaseTransform *phase ) const {
  // Either input is TOP ==> the result is TOP
  Node* mem = in(MemNode::Memory);
  const Type *t1 = phase->type(mem);
  if (t1 == Type::TOP)  return Type::TOP;
  Node* adr = in(MemNode::Address);
  const TypePtr* tp = phase->type(adr)->isa_ptr();
  if (tp == NULL || tp->empty())  return Type::TOP;
  int off = tp->offset();
  assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
1637
  Compile* C = phase->C;
D
duke 已提交
1638 1639

  // Try to guess loaded type from pointer type
1640 1641
  if (tp->isa_aryptr()) {
    const TypeAryPtr* ary = tp->is_aryptr();
1642
    const Type* t = ary->elem();
1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653

    // Determine whether the reference is beyond the header or not, by comparing
    // the offset against the offset of the start of the array's data.
    // Different array types begin at slightly different offsets (12 vs. 16).
    // We choose T_BYTE as an example base type that is least restrictive
    // as to alignment, which will therefore produce the smallest
    // possible base offset.
    const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
    const bool off_beyond_header = ((uint)off >= (uint)min_base_off);

    // Try to constant-fold a stable array element.
1654 1655 1656
    if (FoldStableValues && ary->is_stable() && ary->const_oop() != NULL) {
      // Make sure the reference is not into the header and the offset is constant
      if (off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
1657 1658 1659 1660 1661 1662 1663
        const Type* con_type = fold_stable_ary_elem(ary, off, memory_type());
        if (con_type != NULL) {
          return con_type;
        }
      }
    }

D
duke 已提交
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679
    // Don't do this for integer types. There is only potential profit if
    // the element type t is lower than _type; that is, for int types, if _type is
    // more restrictive than t.  This only happens here if one is short and the other
    // char (both 16 bits), and in those cases we've made an intentional decision
    // to use one kind of load over the other. See AndINode::Ideal and 4965907.
    // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
    //
    // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
    // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
    // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
    // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
    // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
    // In fact, that could have been the original type of p1, and p1 could have
    // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
    // expression (LShiftL quux 3) independently optimized to the constant 8.
    if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1680
        && (_type->isa_vect() == NULL)
1681
        && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
D
duke 已提交
1682 1683
      // t might actually be lower than _type, if _type is a unique
      // concrete subclass of abstract class t.
1684
      if (off_beyond_header) {  // is the offset beyond the header?
1685
        const Type* jt = t->join_speculative(_type);
D
duke 已提交
1686 1687 1688 1689 1690
        // In any case, do not allow the join, per se, to empty out the type.
        if (jt->empty() && !t->empty()) {
          // This can happen if a interface-typed array narrows to a class type.
          jt = _type;
        }
1691 1692
#ifdef ASSERT
        if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1693
          // The pointers in the autobox arrays are always non-null
1694
          Node* base = adr->in(AddPNode::Base);
1695 1696 1697 1698 1699 1700 1701 1702 1703
          if ((base != NULL) && base->is_DecodeN()) {
            // Get LoadN node which loads IntegerCache.cache field
            base = base->in(1);
          }
          if ((base != NULL) && base->is_Con()) {
            const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
            if ((base_type != NULL) && base_type->is_autobox_cache()) {
              // It could be narrow oop
              assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1704 1705 1706
            }
          }
        }
1707
#endif
D
duke 已提交
1708 1709 1710 1711
        return jt;
      }
    }
  } else if (tp->base() == Type::InstPtr) {
1712
    ciEnv* env = C->env();
1713 1714
    const TypeInstPtr* tinst = tp->is_instptr();
    ciKlass* klass = tinst->klass();
D
duke 已提交
1715 1716 1717 1718
    assert( off != Type::OffsetBot ||
            // arrays can be cast to Objects
            tp->is_oopptr()->klass()->is_java_lang_Object() ||
            // unsafe field access may not have a constant offset
1719
            C->has_unsafe_access(),
D
duke 已提交
1720 1721
            "Field accesses must be precise" );
    // For oop loads, we expect the _type to be precise
1722
    if (klass == env->String_klass() &&
1723
        adr->is_AddP() && off != Type::OffsetBot) {
1724
      // For constant Strings treat the final fields as compile time constants.
1725
      Node* base = adr->in(AddPNode::Base);
1726 1727
      const TypeOopPtr* t = phase->type(base)->isa_oopptr();
      if (t != NULL && t->singleton()) {
1728
        ciField* field = env->String_klass()->get_field_by_offset(off, false);
1729 1730 1731 1732 1733 1734 1735
        if (field != NULL && field->is_final()) {
          ciObject* string = t->const_oop();
          ciConstant constant = string->as_instance()->field_value(field);
          if (constant.basic_type() == T_INT) {
            return TypeInt::make(constant.as_int());
          } else if (constant.basic_type() == T_ARRAY) {
            if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1736
              return TypeNarrowOop::make_from_constant(constant.as_object(), true);
1737
            } else {
1738
              return TypeOopPtr::make_from_constant(constant.as_object(), true);
1739
            }
1740 1741 1742 1743
          }
        }
      }
    }
1744 1745 1746
    // Optimizations for constant objects
    ciObject* const_oop = tinst->const_oop();
    if (const_oop != NULL) {
1747 1748 1749 1750
      // For constant Boxed value treat the target field as a compile time constant.
      if (tinst->is_ptr_to_boxed_value()) {
        return tinst->get_const_boxed_value();
      } else
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773
      // For constant CallSites treat the target field as a compile time constant.
      if (const_oop->is_call_site()) {
        ciCallSite* call_site = const_oop->as_call_site();
        ciField* field = call_site->klass()->as_instance_klass()->get_field_by_offset(off, /*is_static=*/ false);
        if (field != NULL && field->is_call_site_target()) {
          ciMethodHandle* target = call_site->get_target();
          if (target != NULL) {  // just in case
            ciConstant constant(T_OBJECT, target);
            const Type* t;
            if (adr->bottom_type()->is_ptr_to_narrowoop()) {
              t = TypeNarrowOop::make_from_constant(constant.as_object(), true);
            } else {
              t = TypeOopPtr::make_from_constant(constant.as_object(), true);
            }
            // Add a dependence for invalidation of the optimization.
            if (!call_site->is_constant_call_site()) {
              C->dependencies()->assert_call_site_target_value(call_site, target);
            }
            return t;
          }
        }
      }
    }
D
duke 已提交
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791
  } else if (tp->base() == Type::KlassPtr) {
    assert( off != Type::OffsetBot ||
            // arrays can be cast to Objects
            tp->is_klassptr()->klass()->is_java_lang_Object() ||
            // also allow array-loading from the primary supertype
            // array during subtype checks
            Opcode() == Op_LoadKlass,
            "Field accesses must be precise" );
    // For klass/static loads, we expect the _type to be precise
  }

  const TypeKlassPtr *tkls = tp->isa_klassptr();
  if (tkls != NULL && !StressReflectiveCode) {
    ciKlass* klass = tkls->klass();
    if (klass->is_loaded() && tkls->klass_is_exact()) {
      // We are loading a field from a Klass metaobject whose identity
      // is known at compile time (the type is "exact" or "precise").
      // Check for fields we know are maintained as constants by the VM.
1792
      if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
D
duke 已提交
1793 1794 1795 1796 1797 1798
        // The field is Klass::_super_check_offset.  Return its (constant) value.
        // (Folds up type checking code.)
        assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
        return TypeInt::make(klass->super_check_offset());
      }
      // Compute index into primary_supers array
1799
      juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
D
duke 已提交
1800 1801 1802 1803 1804 1805 1806 1807 1808 1809
      // Check for overflowing; use unsigned compare to handle the negative case.
      if( depth < ciKlass::primary_super_limit() ) {
        // The field is an element of Klass::_primary_supers.  Return its (constant) value.
        // (Folds up type checking code.)
        assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
        ciKlass *ss = klass->super_of_depth(depth);
        return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
      }
      const Type* aift = load_array_final_field(tkls, klass);
      if (aift != NULL)  return aift;
1810
      if (tkls->offset() == in_bytes(ArrayKlass::component_mirror_offset())
D
duke 已提交
1811
          && klass->is_array_klass()) {
1812
        // The field is ArrayKlass::_component_mirror.  Return its (constant) value.
D
duke 已提交
1813 1814 1815 1816
        // (Folds up aClassConstant.getComponentType, common in Arrays.copyOf.)
        assert(Opcode() == Op_LoadP, "must load an oop from _component_mirror");
        return TypeInstPtr::make(klass->as_array_klass()->component_mirror());
      }
1817
      if (tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
D
duke 已提交
1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828
        // The field is Klass::_java_mirror.  Return its (constant) value.
        // (Folds up the 2nd indirection in anObjConstant.getClass().)
        assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
        return TypeInstPtr::make(klass->java_mirror());
      }
    }

    // We can still check if we are loading from the primary_supers array at a
    // shallow enough depth.  Even though the klass is not exact, entries less
    // than or equal to its super depth are correct.
    if (klass->is_loaded() ) {
1829
      ciType *inner = klass;
D
duke 已提交
1830 1831 1832 1833 1834
      while( inner->is_obj_array_klass() )
        inner = inner->as_obj_array_klass()->base_element_type();
      if( inner->is_instance_klass() &&
          !inner->as_instance_klass()->flags().is_interface() ) {
        // Compute index into primary_supers array
1835
        juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
D
duke 已提交
1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850
        // Check for overflowing; use unsigned compare to handle the negative case.
        if( depth < ciKlass::primary_super_limit() &&
            depth <= klass->super_depth() ) { // allow self-depth checks to handle self-check case
          // The field is an element of Klass::_primary_supers.  Return its (constant) value.
          // (Folds up type checking code.)
          assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
          ciKlass *ss = klass->super_of_depth(depth);
          return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
        }
      }
    }

    // If the type is enough to determine that the thing is not an array,
    // we can give the layout_helper a positive interval type.
    // This will help short-circuit some reflective code.
1851
    if (tkls->offset() == in_bytes(Klass::layout_helper_offset())
D
duke 已提交
1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
        && !klass->is_array_klass() // not directly typed as an array
        && !klass->is_interface()  // specifically not Serializable & Cloneable
        && !klass->is_java_lang_Object()   // not the supertype of all T[]
        ) {
      // Note:  When interfaces are reliable, we can narrow the interface
      // test to (klass != Serializable && klass != Cloneable).
      assert(Opcode() == Op_LoadI, "must load an int from _layout_helper");
      jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
      // The key property of this type is that it folds up tests
      // for array-ness, since it proves that the layout_helper is positive.
      // Thus, a generic value like the basic object layout helper works fine.
      return TypeInt::make(min_size, max_jint, Type::WidenMin);
    }
  }

  // If we are loading from a freshly-allocated object, produce a zero,
  // if the load is provably beyond the header of the object.
  // (Also allow a variable load from a fresh array to produce zero.)
1870 1871
  const TypeOopPtr *tinst = tp->isa_oopptr();
  bool is_instance = (tinst != NULL) && tinst->is_known_instance_field();
1872 1873
  bool is_boxed_value = (tinst != NULL) && tinst->is_ptr_to_boxed_value();
  if (ReduceFieldZeroing || is_instance || is_boxed_value) {
D
duke 已提交
1874
    Node* value = can_see_stored_value(mem,phase);
1875 1876
    if (value != NULL && value->is_Con()) {
      assert(value->bottom_type()->higher_equal(_type),"sanity");
D
duke 已提交
1877
      return value->bottom_type();
1878
    }
D
duke 已提交
1879 1880
  }

1881
  if (is_instance) {
1882 1883 1884 1885 1886 1887 1888 1889 1890
    // If we have an instance type and our memory input is the
    // programs's initial memory state, there is no matching store,
    // so just return a zero of the appropriate type
    Node *mem = in(MemNode::Memory);
    if (mem->is_Parm() && mem->in(0)->is_Start()) {
      assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
      return Type::get_zero_type(_type->basic_type());
    }
  }
D
duke 已提交
1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910
  return _type;
}

//------------------------------match_edge-------------------------------------
// Do we Match on this edge index or not?  Match only the address.
uint LoadNode::match_edge(uint idx) const {
  return idx == MemNode::Address;
}

//--------------------------LoadBNode::Ideal--------------------------------------
//
//  If the previous store is to the same address as this load,
//  and the value stored was larger than a byte, replace this load
//  with the value stored truncated to a byte.  If no truncation is
//  needed, the replacement is done in LoadNode::Identity().
//
Node *LoadBNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  Node* mem = in(MemNode::Memory);
  Node* value = can_see_stored_value(mem,phase);
  if( value && !phase->type(value)->higher_equal( _type ) ) {
1911 1912
    Node *result = phase->transform( new (phase->C) LShiftINode(value, phase->intcon(24)) );
    return new (phase->C) RShiftINode(result, phase->intcon(24));
D
duke 已提交
1913 1914 1915 1916 1917
  }
  // Identity call will handle the case where truncation is not needed.
  return LoadNode::Ideal(phase, can_reshape);
}

1918 1919 1920
const Type* LoadBNode::Value(PhaseTransform *phase) const {
  Node* mem = in(MemNode::Memory);
  Node* value = can_see_stored_value(mem,phase);
1921 1922
  if (value != NULL && value->is_Con() &&
      !value->bottom_type()->higher_equal(_type)) {
1923 1924 1925 1926 1927 1928 1929 1930 1931
    // If the input to the store does not fit with the load's result type,
    // it must be truncated. We can't delay until Ideal call since
    // a singleton Value is needed for split_thru_phi optimization.
    int con = value->get_int();
    return TypeInt::make((con << 24) >> 24);
  }
  return LoadNode::Value(phase);
}

1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942
//--------------------------LoadUBNode::Ideal-------------------------------------
//
//  If the previous store is to the same address as this load,
//  and the value stored was larger than a byte, replace this load
//  with the value stored truncated to a byte.  If no truncation is
//  needed, the replacement is done in LoadNode::Identity().
//
Node* LoadUBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
  Node* mem = in(MemNode::Memory);
  Node* value = can_see_stored_value(mem, phase);
  if (value && !phase->type(value)->higher_equal(_type))
1943
    return new (phase->C) AndINode(value, phase->intcon(0xFF));
1944 1945 1946 1947
  // Identity call will handle the case where truncation is not needed.
  return LoadNode::Ideal(phase, can_reshape);
}

1948 1949 1950
const Type* LoadUBNode::Value(PhaseTransform *phase) const {
  Node* mem = in(MemNode::Memory);
  Node* value = can_see_stored_value(mem,phase);
1951 1952
  if (value != NULL && value->is_Con() &&
      !value->bottom_type()->higher_equal(_type)) {
1953 1954 1955 1956 1957 1958 1959 1960 1961
    // If the input to the store does not fit with the load's result type,
    // it must be truncated. We can't delay until Ideal call since
    // a singleton Value is needed for split_thru_phi optimization.
    int con = value->get_int();
    return TypeInt::make(con & 0xFF);
  }
  return LoadNode::Value(phase);
}

1962
//--------------------------LoadUSNode::Ideal-------------------------------------
D
duke 已提交
1963 1964 1965 1966 1967 1968
//
//  If the previous store is to the same address as this load,
//  and the value stored was larger than a char, replace this load
//  with the value stored truncated to a char.  If no truncation is
//  needed, the replacement is done in LoadNode::Identity().
//
1969
Node *LoadUSNode::Ideal(PhaseGVN *phase, bool can_reshape) {
D
duke 已提交
1970 1971 1972
  Node* mem = in(MemNode::Memory);
  Node* value = can_see_stored_value(mem,phase);
  if( value && !phase->type(value)->higher_equal( _type ) )
1973
    return new (phase->C) AndINode(value,phase->intcon(0xFFFF));
D
duke 已提交
1974 1975 1976 1977
  // Identity call will handle the case where truncation is not needed.
  return LoadNode::Ideal(phase, can_reshape);
}

1978 1979 1980
const Type* LoadUSNode::Value(PhaseTransform *phase) const {
  Node* mem = in(MemNode::Memory);
  Node* value = can_see_stored_value(mem,phase);
1981 1982
  if (value != NULL && value->is_Con() &&
      !value->bottom_type()->higher_equal(_type)) {
1983 1984 1985 1986 1987 1988 1989 1990 1991
    // If the input to the store does not fit with the load's result type,
    // it must be truncated. We can't delay until Ideal call since
    // a singleton Value is needed for split_thru_phi optimization.
    int con = value->get_int();
    return TypeInt::make(con & 0xFFFF);
  }
  return LoadNode::Value(phase);
}

D
duke 已提交
1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002
//--------------------------LoadSNode::Ideal--------------------------------------
//
//  If the previous store is to the same address as this load,
//  and the value stored was larger than a short, replace this load
//  with the value stored truncated to a short.  If no truncation is
//  needed, the replacement is done in LoadNode::Identity().
//
Node *LoadSNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  Node* mem = in(MemNode::Memory);
  Node* value = can_see_stored_value(mem,phase);
  if( value && !phase->type(value)->higher_equal( _type ) ) {
2003 2004
    Node *result = phase->transform( new (phase->C) LShiftINode(value, phase->intcon(16)) );
    return new (phase->C) RShiftINode(result, phase->intcon(16));
D
duke 已提交
2005 2006 2007 2008 2009
  }
  // Identity call will handle the case where truncation is not needed.
  return LoadNode::Ideal(phase, can_reshape);
}

2010 2011 2012
const Type* LoadSNode::Value(PhaseTransform *phase) const {
  Node* mem = in(MemNode::Memory);
  Node* value = can_see_stored_value(mem,phase);
2013 2014
  if (value != NULL && value->is_Con() &&
      !value->bottom_type()->higher_equal(_type)) {
2015 2016 2017 2018 2019 2020 2021 2022 2023
    // If the input to the store does not fit with the load's result type,
    // it must be truncated. We can't delay until Ideal call since
    // a singleton Value is needed for split_thru_phi optimization.
    int con = value->get_int();
    return TypeInt::make((con << 16) >> 16);
  }
  return LoadNode::Value(phase);
}

D
duke 已提交
2024
//=============================================================================
2025 2026 2027 2028 2029 2030
//----------------------------LoadKlassNode::make------------------------------
// Polymorphic factory method:
Node *LoadKlassNode::make( PhaseGVN& gvn, Node *mem, Node *adr, const TypePtr* at, const TypeKlassPtr *tk ) {
  Compile* C = gvn.C;
  Node *ctl = NULL;
  // sanity check the alias category against the created node type
2031 2032
  const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
  assert(adr_type != NULL, "expecting TypeKlassPtr");
2033
#ifdef _LP64
2034
  if (adr_type->is_ptr_to_narrowklass()) {
2035
    assert(UseCompressedClassPointers, "no compressed klasses");
2036
    Node* load_klass = gvn.transform(new (C) LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2037
    return new (C) DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2038
  }
2039
#endif
2040
  assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2041
  return new (C) LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2042 2043
}

D
duke 已提交
2044 2045
//------------------------------Value------------------------------------------
const Type *LoadKlassNode::Value( PhaseTransform *phase ) const {
2046 2047 2048 2049
  return klass_value_common(phase);
}

const Type *LoadNode::klass_value_common( PhaseTransform *phase ) const {
D
duke 已提交
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074
  // Either input is TOP ==> the result is TOP
  const Type *t1 = phase->type( in(MemNode::Memory) );
  if (t1 == Type::TOP)  return Type::TOP;
  Node *adr = in(MemNode::Address);
  const Type *t2 = phase->type( adr );
  if (t2 == Type::TOP)  return Type::TOP;
  const TypePtr *tp = t2->is_ptr();
  if (TypePtr::above_centerline(tp->ptr()) ||
      tp->ptr() == TypePtr::Null)  return Type::TOP;

  // Return a more precise klass, if possible
  const TypeInstPtr *tinst = tp->isa_instptr();
  if (tinst != NULL) {
    ciInstanceKlass* ik = tinst->klass()->as_instance_klass();
    int offset = tinst->offset();
    if (ik == phase->C->env()->Class_klass()
        && (offset == java_lang_Class::klass_offset_in_bytes() ||
            offset == java_lang_Class::array_klass_offset_in_bytes())) {
      // We are loading a special hidden field from a Class mirror object,
      // the field which points to the VM's Klass metaobject.
      ciType* t = tinst->java_mirror_type();
      // java_mirror_type returns non-null for compile-time Class constants.
      if (t != NULL) {
        // constant oop => constant klass
        if (offset == java_lang_Class::array_klass_offset_in_bytes()) {
2075 2076 2077 2078 2079
          if (t->is_void()) {
            // We cannot create a void array.  Since void is a primitive type return null
            // klass.  Users of this result need to do a null check on the returned klass.
            return TypePtr::NULL_PTR;
          }
D
duke 已提交
2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158
          return TypeKlassPtr::make(ciArrayKlass::make(t));
        }
        if (!t->is_klass()) {
          // a primitive Class (e.g., int.class) has NULL for a klass field
          return TypePtr::NULL_PTR;
        }
        // (Folds up the 1st indirection in aClassConstant.getModifiers().)
        return TypeKlassPtr::make(t->as_klass());
      }
      // non-constant mirror, so we can't tell what's going on
    }
    if( !ik->is_loaded() )
      return _type;             // Bail out if not loaded
    if (offset == oopDesc::klass_offset_in_bytes()) {
      if (tinst->klass_is_exact()) {
        return TypeKlassPtr::make(ik);
      }
      // See if we can become precise: no subklasses and no interface
      // (Note:  We need to support verified interfaces.)
      if (!ik->is_interface() && !ik->has_subklass()) {
        //assert(!UseExactTypes, "this code should be useless with exact types");
        // Add a dependence; if any subclass added we need to recompile
        if (!ik->is_final()) {
          // %%% should use stronger assert_unique_concrete_subtype instead
          phase->C->dependencies()->assert_leaf_type(ik);
        }
        // Return precise klass
        return TypeKlassPtr::make(ik);
      }

      // Return root of possible klass
      return TypeKlassPtr::make(TypePtr::NotNull, ik, 0/*offset*/);
    }
  }

  // Check for loading klass from an array
  const TypeAryPtr *tary = tp->isa_aryptr();
  if( tary != NULL ) {
    ciKlass *tary_klass = tary->klass();
    if (tary_klass != NULL   // can be NULL when at BOTTOM or TOP
        && tary->offset() == oopDesc::klass_offset_in_bytes()) {
      if (tary->klass_is_exact()) {
        return TypeKlassPtr::make(tary_klass);
      }
      ciArrayKlass *ak = tary->klass()->as_array_klass();
      // If the klass is an object array, we defer the question to the
      // array component klass.
      if( ak->is_obj_array_klass() ) {
        assert( ak->is_loaded(), "" );
        ciKlass *base_k = ak->as_obj_array_klass()->base_element_klass();
        if( base_k->is_loaded() && base_k->is_instance_klass() ) {
          ciInstanceKlass* ik = base_k->as_instance_klass();
          // See if we can become precise: no subklasses and no interface
          if (!ik->is_interface() && !ik->has_subklass()) {
            //assert(!UseExactTypes, "this code should be useless with exact types");
            // Add a dependence; if any subclass added we need to recompile
            if (!ik->is_final()) {
              phase->C->dependencies()->assert_leaf_type(ik);
            }
            // Return precise array klass
            return TypeKlassPtr::make(ak);
          }
        }
        return TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
      } else {                  // Found a type-array?
        //assert(!UseExactTypes, "this code should be useless with exact types");
        assert( ak->is_type_array_klass(), "" );
        return TypeKlassPtr::make(ak); // These are always precise
      }
    }
  }

  // Check for loading klass from an array klass
  const TypeKlassPtr *tkls = tp->isa_klassptr();
  if (tkls != NULL && !StressReflectiveCode) {
    ciKlass* klass = tkls->klass();
    if( !klass->is_loaded() )
      return _type;             // Bail out if not loaded
    if( klass->is_obj_array_klass() &&
2159
        tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
D
duke 已提交
2160 2161 2162 2163 2164 2165 2166 2167 2168 2169
      ciKlass* elem = klass->as_obj_array_klass()->element_klass();
      // // Always returning precise element type is incorrect,
      // // e.g., element type could be object and array may contain strings
      // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);

      // The array's TypeKlassPtr was declared 'precise' or 'not precise'
      // according to the element type's subclassing.
      return TypeKlassPtr::make(tkls->ptr(), elem, 0/*offset*/);
    }
    if( klass->is_instance_klass() && tkls->klass_is_exact() &&
2170
        tkls->offset() == in_bytes(Klass::super_offset())) {
D
duke 已提交
2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185
      ciKlass* sup = klass->as_instance_klass()->super();
      // The field is Klass::_super.  Return its (constant) value.
      // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
      return sup ? TypeKlassPtr::make(sup) : TypePtr::NULL_PTR;
    }
  }

  // Bailout case
  return LoadNode::Value(phase);
}

//------------------------------Identity---------------------------------------
// To clean up reflective code, simplify k.java_mirror.as_klass to plain k.
// Also feed through the klass in Allocate(...klass...)._klass.
Node* LoadKlassNode::Identity( PhaseTransform *phase ) {
2186 2187 2188 2189
  return klass_identity_common(phase);
}

Node* LoadNode::klass_identity_common(PhaseTransform *phase ) {
D
duke 已提交
2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210
  Node* x = LoadNode::Identity(phase);
  if (x != this)  return x;

  // Take apart the address into an oop and and offset.
  // Return 'this' if we cannot.
  Node*    adr    = in(MemNode::Address);
  intptr_t offset = 0;
  Node*    base   = AddPNode::Ideal_base_and_offset(adr, phase, offset);
  if (base == NULL)     return this;
  const TypeOopPtr* toop = phase->type(adr)->isa_oopptr();
  if (toop == NULL)     return this;

  // We can fetch the klass directly through an AllocateNode.
  // This works even if the klass is not constant (clone or newArray).
  if (offset == oopDesc::klass_offset_in_bytes()) {
    Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
    if (allocated_klass != NULL) {
      return allocated_klass;
    }
  }

2211
  // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2212
  // Simplify ak.component_mirror.array_klass to plain ak, ak an ArrayKlass.
D
duke 已提交
2213 2214 2215 2216 2217 2218 2219
  // See inline_native_Class_query for occurrences of these patterns.
  // Java Example:  x.getClass().isAssignableFrom(y)
  // Java Example:  Array.newInstance(x.getClass().getComponentType(), n)
  //
  // This improves reflective code, often making the Class
  // mirror go completely dead.  (Current exception:  Class
  // mirrors may appear in debug info, but we could clean them out by
2220
  // introducing a new debug info operator for Klass*.java_mirror).
D
duke 已提交
2221 2222 2223 2224
  if (toop->isa_instptr() && toop->klass() == phase->C->env()->Class_klass()
      && (offset == java_lang_Class::klass_offset_in_bytes() ||
          offset == java_lang_Class::array_klass_offset_in_bytes())) {
    // We are loading a special hidden field from a Class mirror,
2225
    // the field which points to its Klass or ArrayKlass metaobject.
D
duke 已提交
2226 2227 2228 2229 2230 2231 2232 2233
    if (base->is_Load()) {
      Node* adr2 = base->in(MemNode::Address);
      const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
      if (tkls != NULL && !tkls->empty()
          && (tkls->klass()->is_instance_klass() ||
              tkls->klass()->is_array_klass())
          && adr2->is_AddP()
          ) {
2234
        int mirror_field = in_bytes(Klass::java_mirror_offset());
D
duke 已提交
2235
        if (offset == java_lang_Class::array_klass_offset_in_bytes()) {
2236
          mirror_field = in_bytes(ArrayKlass::component_mirror_offset());
D
duke 已提交
2237
        }
2238
        if (tkls->offset() == mirror_field) {
D
duke 已提交
2239 2240 2241 2242 2243 2244 2245 2246 2247
          return adr2->in(AddPNode::Base);
        }
      }
    }
  }

  return this;
}

2248 2249 2250 2251

//------------------------------Value------------------------------------------
const Type *LoadNKlassNode::Value( PhaseTransform *phase ) const {
  const Type *t = klass_value_common(phase);
2252 2253
  if (t == Type::TOP)
    return t;
2254

2255
  return t->make_narrowklass();
2256 2257 2258 2259 2260 2261 2262 2263 2264 2265
}

//------------------------------Identity---------------------------------------
// To clean up reflective code, simplify k.java_mirror.as_klass to narrow k.
// Also feed through the klass in Allocate(...klass...)._klass.
Node* LoadNKlassNode::Identity( PhaseTransform *phase ) {
  Node *x = klass_identity_common(phase);

  const Type *t = phase->type( x );
  if( t == Type::TOP ) return x;
2266 2267
  if( t->isa_narrowklass()) return x;
  assert (!t->isa_narrowoop(), "no narrow oop here");
2268

2269
  return phase->transform(new (phase->C) EncodePKlassNode(x, t->make_narrowklass()));
2270 2271
}

D
duke 已提交
2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286
//------------------------------Value-----------------------------------------
const Type *LoadRangeNode::Value( PhaseTransform *phase ) const {
  // Either input is TOP ==> the result is TOP
  const Type *t1 = phase->type( in(MemNode::Memory) );
  if( t1 == Type::TOP ) return Type::TOP;
  Node *adr = in(MemNode::Address);
  const Type *t2 = phase->type( adr );
  if( t2 == Type::TOP ) return Type::TOP;
  const TypePtr *tp = t2->is_ptr();
  if (TypePtr::above_centerline(tp->ptr()))  return Type::TOP;
  const TypeAryPtr *tap = tp->isa_aryptr();
  if( !tap ) return _type;
  return tap->size();
}

2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
//-------------------------------Ideal---------------------------------------
// Feed through the length in AllocateArray(...length...)._length.
Node *LoadRangeNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  Node* p = MemNode::Ideal_common(phase, can_reshape);
  if (p)  return (p == NodeSentinel) ? NULL : p;

  // Take apart the address into an oop and and offset.
  // Return 'this' if we cannot.
  Node*    adr    = in(MemNode::Address);
  intptr_t offset = 0;
  Node*    base   = AddPNode::Ideal_base_and_offset(adr, phase,  offset);
  if (base == NULL)     return NULL;
  const TypeAryPtr* tary = phase->type(adr)->isa_aryptr();
  if (tary == NULL)     return NULL;

  // We can fetch the length directly through an AllocateArrayNode.
  // This works even if the length is not constant (clone or newArray).
  if (offset == arrayOopDesc::length_offset_in_bytes()) {
    AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(base, phase);
    if (alloc != NULL) {
      Node* allocated_length = alloc->Ideal_length();
      Node* len = alloc->make_ideal_length(tary, phase);
      if (allocated_length != len) {
        // New CastII improves on this.
        return len;
      }
    }
  }

  return NULL;
}

D
duke 已提交
2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336
//------------------------------Identity---------------------------------------
// Feed through the length in AllocateArray(...length...)._length.
Node* LoadRangeNode::Identity( PhaseTransform *phase ) {
  Node* x = LoadINode::Identity(phase);
  if (x != this)  return x;

  // Take apart the address into an oop and and offset.
  // Return 'this' if we cannot.
  Node*    adr    = in(MemNode::Address);
  intptr_t offset = 0;
  Node*    base   = AddPNode::Ideal_base_and_offset(adr, phase, offset);
  if (base == NULL)     return this;
  const TypeAryPtr* tary = phase->type(adr)->isa_aryptr();
  if (tary == NULL)     return this;

  // We can fetch the length directly through an AllocateArrayNode.
  // This works even if the length is not constant (clone or newArray).
  if (offset == arrayOopDesc::length_offset_in_bytes()) {
2337 2338 2339 2340 2341 2342 2343 2344 2345
    AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(base, phase);
    if (alloc != NULL) {
      Node* allocated_length = alloc->Ideal_length();
      // Do not allow make_ideal_length to allocate a CastII node.
      Node* len = alloc->make_ideal_length(tary, phase, false);
      if (allocated_length == len) {
        // Return allocated_length only if it would not be improved by a CastII.
        return allocated_length;
      }
D
duke 已提交
2346 2347 2348 2349 2350 2351
    }
  }

  return this;

}
2352

D
duke 已提交
2353 2354 2355
//=============================================================================
//---------------------------StoreNode::make-----------------------------------
// Polymorphic factory method:
2356 2357
StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo) {
  assert((mo == unordered || mo == release), "unexpected");
2358
  Compile* C = gvn.C;
2359 2360
  assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
         ctl != NULL, "raw memory operations should have control edge");
2361

D
duke 已提交
2362 2363
  switch (bt) {
  case T_BOOLEAN:
2364 2365
  case T_BYTE:    return new (C) StoreBNode(ctl, mem, adr, adr_type, val, mo);
  case T_INT:     return new (C) StoreINode(ctl, mem, adr, adr_type, val, mo);
D
duke 已提交
2366
  case T_CHAR:
2367 2368 2369 2370
  case T_SHORT:   return new (C) StoreCNode(ctl, mem, adr, adr_type, val, mo);
  case T_LONG:    return new (C) StoreLNode(ctl, mem, adr, adr_type, val, mo);
  case T_FLOAT:   return new (C) StoreFNode(ctl, mem, adr, adr_type, val, mo);
  case T_DOUBLE:  return new (C) StoreDNode(ctl, mem, adr, adr_type, val, mo);
2371
  case T_METADATA:
D
duke 已提交
2372
  case T_ADDRESS:
2373 2374
  case T_OBJECT:
#ifdef _LP64
2375
    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2376
      val = gvn.transform(new (C) EncodePNode(val, val->bottom_type()->make_narrowoop()));
2377
      return new (C) StoreNNode(ctl, mem, adr, adr_type, val, mo);
2378
    } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2379
               (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2380 2381
                adr->bottom_type()->isa_rawptr())) {
      val = gvn.transform(new (C) EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2382
      return new (C) StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2383
    }
2384
#endif
2385
    {
2386
      return new (C) StorePNode(ctl, mem, adr, adr_type, val, mo);
2387
    }
D
duke 已提交
2388 2389 2390 2391 2392
  }
  ShouldNotReachHere();
  return (StoreNode*)NULL;
}

2393
StoreLNode* StoreLNode::make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, MemOrd mo) {
D
duke 已提交
2394
  bool require_atomic = true;
2395
  return new (C) StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic);
D
duke 已提交
2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423
}


//--------------------------bottom_type----------------------------------------
const Type *StoreNode::bottom_type() const {
  return Type::MEMORY;
}

//------------------------------hash-------------------------------------------
uint StoreNode::hash() const {
  // unroll addition of interesting fields
  //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);

  // Since they are not commoned, do not hash them:
  return NO_HASH;
}

//------------------------------Ideal------------------------------------------
// Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
// When a store immediately follows a relevant allocation/initialization,
// try to capture it into the initialization, or hoist it above.
Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  Node* p = MemNode::Ideal_common(phase, can_reshape);
  if (p)  return (p == NodeSentinel) ? NULL : p;

  Node* mem     = in(MemNode::Memory);
  Node* address = in(MemNode::Address);

2424 2425 2426 2427
  // Back-to-back stores to same address?  Fold em up.  Generally
  // unsafe if I have intervening uses...  Also disallowed for StoreCM
  // since they must follow each StoreP operation.  Redundant StoreCMs
  // are eliminated just before matching in final_graph_reshape.
K
kvn 已提交
2428
  if (mem->is_Store() && mem->in(MemNode::Address)->eqv_uncast(address) &&
2429
      mem->Opcode() != Op_StoreCM) {
D
duke 已提交
2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461
    // Looking at a dead closed cycle of memory?
    assert(mem != mem->in(MemNode::Memory), "dead loop in StoreNode::Ideal");

    assert(Opcode() == mem->Opcode() ||
           phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw,
           "no mismatched stores, except on raw memory");

    if (mem->outcnt() == 1 &&           // check for intervening uses
        mem->as_Store()->memory_size() <= this->memory_size()) {
      // If anybody other than 'this' uses 'mem', we cannot fold 'mem' away.
      // For example, 'mem' might be the final state at a conditional return.
      // Or, 'mem' might be used by some node which is live at the same time
      // 'this' is live, which might be unschedulable.  So, require exactly
      // ONE user, the 'this' store, until such time as we clone 'mem' for
      // each of 'mem's uses (thus making the exactly-1-user-rule hold true).
      if (can_reshape) {  // (%%% is this an anachronism?)
        set_req_X(MemNode::Memory, mem->in(MemNode::Memory),
                  phase->is_IterGVN());
      } else {
        // It's OK to do this in the parser, since DU info is always accurate,
        // and the parser always refers to nodes via SafePointNode maps.
        set_req(MemNode::Memory, mem->in(MemNode::Memory));
      }
      return this;
    }
  }

  // Capture an unaliased, unconditional, simple store into an initializer.
  // Or, if it is independent of the allocation, hoist it above the allocation.
  if (ReduceFieldZeroing && /*can_reshape &&*/
      mem->is_Proj() && mem->in(0)->is_Initialize()) {
    InitializeNode* init = mem->in(0)->as_Initialize();
2462
    intptr_t offset = init->can_capture_store(this, phase, can_reshape);
D
duke 已提交
2463
    if (offset > 0) {
2464
      Node* moved = init->capture_store(this, offset, phase, can_reshape);
D
duke 已提交
2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500
      // If the InitializeNode captured me, it made a raw copy of me,
      // and I need to disappear.
      if (moved != NULL) {
        // %%% hack to ensure that Ideal returns a new node:
        mem = MergeMemNode::make(phase->C, mem);
        return mem;             // fold me away
      }
    }
  }

  return NULL;                  // No further progress
}

//------------------------------Value-----------------------------------------
const Type *StoreNode::Value( PhaseTransform *phase ) const {
  // Either input is TOP ==> the result is TOP
  const Type *t1 = phase->type( in(MemNode::Memory) );
  if( t1 == Type::TOP ) return Type::TOP;
  const Type *t2 = phase->type( in(MemNode::Address) );
  if( t2 == Type::TOP ) return Type::TOP;
  const Type *t3 = phase->type( in(MemNode::ValueIn) );
  if( t3 == Type::TOP ) return Type::TOP;
  return Type::MEMORY;
}

//------------------------------Identity---------------------------------------
// Remove redundant stores:
//   Store(m, p, Load(m, p)) changes to m.
//   Store(, p, x) -> Store(m, p, x) changes to Store(m, p, x).
Node *StoreNode::Identity( PhaseTransform *phase ) {
  Node* mem = in(MemNode::Memory);
  Node* adr = in(MemNode::Address);
  Node* val = in(MemNode::ValueIn);

  // Load then Store?  Then the Store is useless
  if (val->is_Load() &&
K
kvn 已提交
2501 2502
      val->in(MemNode::Address)->eqv_uncast(adr) &&
      val->in(MemNode::Memory )->eqv_uncast(mem) &&
D
duke 已提交
2503 2504 2505 2506 2507 2508
      val->as_Load()->store_Opcode() == Opcode()) {
    return mem;
  }

  // Two stores in a row of the same value?
  if (mem->is_Store() &&
K
kvn 已提交
2509 2510
      mem->in(MemNode::Address)->eqv_uncast(adr) &&
      mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
D
duke 已提交
2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603
      mem->Opcode() == Opcode()) {
    return mem;
  }

  // Store of zero anywhere into a freshly-allocated object?
  // Then the store is useless.
  // (It must already have been captured by the InitializeNode.)
  if (ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
    // a newly allocated object is already all-zeroes everywhere
    if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
      return mem;
    }

    // the store may also apply to zero-bits in an earlier object
    Node* prev_mem = find_previous_store(phase);
    // Steps (a), (b):  Walk past independent stores to find an exact match.
    if (prev_mem != NULL) {
      Node* prev_val = can_see_stored_value(prev_mem, phase);
      if (prev_val != NULL && phase->eqv(prev_val, val)) {
        // prev_val and val might differ by a cast; it would be good
        // to keep the more informative of the two.
        return mem;
      }
    }
  }

  return this;
}

//------------------------------match_edge-------------------------------------
// Do we Match on this edge index or not?  Match only memory & value
uint StoreNode::match_edge(uint idx) const {
  return idx == MemNode::Address || idx == MemNode::ValueIn;
}

//------------------------------cmp--------------------------------------------
// Do not common stores up together.  They generally have to be split
// back up anyways, so do not bother.
uint StoreNode::cmp( const Node &n ) const {
  return (&n == this);          // Always fail except on self
}

//------------------------------Ideal_masked_input-----------------------------
// Check for a useless mask before a partial-word store
// (StoreB ... (AndI valIn conIa) )
// If (conIa & mask == mask) this simplifies to
// (StoreB ... (valIn) )
Node *StoreNode::Ideal_masked_input(PhaseGVN *phase, uint mask) {
  Node *val = in(MemNode::ValueIn);
  if( val->Opcode() == Op_AndI ) {
    const TypeInt *t = phase->type( val->in(2) )->isa_int();
    if( t && t->is_con() && (t->get_con() & mask) == mask ) {
      set_req(MemNode::ValueIn, val->in(1));
      return this;
    }
  }
  return NULL;
}


//------------------------------Ideal_sign_extended_input----------------------
// Check for useless sign-extension before a partial-word store
// (StoreB ... (RShiftI _ (LShiftI _ valIn conIL ) conIR) )
// If (conIL == conIR && conIR <= num_bits)  this simplifies to
// (StoreB ... (valIn) )
Node *StoreNode::Ideal_sign_extended_input(PhaseGVN *phase, int num_bits) {
  Node *val = in(MemNode::ValueIn);
  if( val->Opcode() == Op_RShiftI ) {
    const TypeInt *t = phase->type( val->in(2) )->isa_int();
    if( t && t->is_con() && (t->get_con() <= num_bits) ) {
      Node *shl = val->in(1);
      if( shl->Opcode() == Op_LShiftI ) {
        const TypeInt *t2 = phase->type( shl->in(2) )->isa_int();
        if( t2 && t2->is_con() && (t2->get_con() == t->get_con()) ) {
          set_req(MemNode::ValueIn, shl->in(1));
          return this;
        }
      }
    }
  }
  return NULL;
}

//------------------------------value_never_loaded-----------------------------------
// Determine whether there are any possible loads of the value stored.
// For simplicity, we actually check if there are any loads from the
// address stored to, not just for loads of the value stored by this node.
//
bool StoreNode::value_never_loaded( PhaseTransform *phase) const {
  Node *adr = in(Address);
  const TypeOopPtr *adr_oop = phase->type(adr)->isa_oopptr();
  if (adr_oop == NULL)
    return false;
2604
  if (!adr_oop->is_known_instance_field())
D
duke 已提交
2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660
    return false; // if not a distinct instance, there may be aliases of the address
  for (DUIterator_Fast imax, i = adr->fast_outs(imax); i < imax; i++) {
    Node *use = adr->fast_out(i);
    int opc = use->Opcode();
    if (use->is_Load() || use->is_LoadStore()) {
      return false;
    }
  }
  return true;
}

//=============================================================================
//------------------------------Ideal------------------------------------------
// If the store is from an AND mask that leaves the low bits untouched, then
// we can skip the AND operation.  If the store is from a sign-extension
// (a left shift, then right shift) we can skip both.
Node *StoreBNode::Ideal(PhaseGVN *phase, bool can_reshape){
  Node *progress = StoreNode::Ideal_masked_input(phase, 0xFF);
  if( progress != NULL ) return progress;

  progress = StoreNode::Ideal_sign_extended_input(phase, 24);
  if( progress != NULL ) return progress;

  // Finally check the default case
  return StoreNode::Ideal(phase, can_reshape);
}

//=============================================================================
//------------------------------Ideal------------------------------------------
// If the store is from an AND mask that leaves the low bits untouched, then
// we can skip the AND operation
Node *StoreCNode::Ideal(PhaseGVN *phase, bool can_reshape){
  Node *progress = StoreNode::Ideal_masked_input(phase, 0xFFFF);
  if( progress != NULL ) return progress;

  progress = StoreNode::Ideal_sign_extended_input(phase, 16);
  if( progress != NULL ) return progress;

  // Finally check the default case
  return StoreNode::Ideal(phase, can_reshape);
}

//=============================================================================
//------------------------------Identity---------------------------------------
Node *StoreCMNode::Identity( PhaseTransform *phase ) {
  // No need to card mark when storing a null ptr
  Node* my_store = in(MemNode::OopStore);
  if (my_store->is_Store()) {
    const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
    if( t1 == TypePtr::NULL_PTR ) {
      return in(MemNode::Memory);
    }
  }
  return this;
}

2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676
//=============================================================================
//------------------------------Ideal---------------------------------------
Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
  Node* progress = StoreNode::Ideal(phase, can_reshape);
  if (progress != NULL) return progress;

  Node* my_store = in(MemNode::OopStore);
  if (my_store->is_MergeMem()) {
    Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
    set_req(MemNode::OopStore, mem);
    return this;
  }

  return NULL;
}

D
duke 已提交
2677 2678
//------------------------------Value-----------------------------------------
const Type *StoreCMNode::Value( PhaseTransform *phase ) const {
K
kvn 已提交
2679 2680 2681 2682 2683 2684 2685
  // Either input is TOP ==> the result is TOP
  const Type *t = phase->type( in(MemNode::Memory) );
  if( t == Type::TOP ) return Type::TOP;
  t = phase->type( in(MemNode::Address) );
  if( t == Type::TOP ) return Type::TOP;
  t = phase->type( in(MemNode::ValueIn) );
  if( t == Type::TOP ) return Type::TOP;
D
duke 已提交
2686
  // If extra input is TOP ==> the result is TOP
K
kvn 已提交
2687 2688
  t = phase->type( in(MemNode::OopStore) );
  if( t == Type::TOP ) return Type::TOP;
D
duke 已提交
2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701

  return StoreNode::Value( phase );
}


//=============================================================================
//----------------------------------SCMemProjNode------------------------------
const Type * SCMemProjNode::Value( PhaseTransform *phase ) const
{
  return bottom_type();
}

//=============================================================================
2702 2703 2704 2705 2706 2707
//----------------------------------LoadStoreNode------------------------------
LoadStoreNode::LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required )
  : Node(required),
    _type(rt),
    _adr_type(at)
{
D
duke 已提交
2708 2709 2710 2711 2712
  init_req(MemNode::Control, c  );
  init_req(MemNode::Memory , mem);
  init_req(MemNode::Address, adr);
  init_req(MemNode::ValueIn, val);
  init_class_id(Class_LoadStore);
2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728
}

uint LoadStoreNode::ideal_reg() const {
  return _type->ideal_reg();
}

bool LoadStoreNode::result_not_used() const {
  for( DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++ ) {
    Node *x = fast_out(i);
    if (x->Opcode() == Op_SCMemProj) continue;
    return false;
  }
  return true;
}

uint LoadStoreNode::size_of() const { return sizeof(*this); }
D
duke 已提交
2729

2730 2731 2732 2733
//=============================================================================
//----------------------------------LoadStoreConditionalNode--------------------
LoadStoreConditionalNode::LoadStoreConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex ) : LoadStoreNode(c, mem, adr, val, NULL, TypeInt::BOOL, 5) {
  init_req(ExpectedIn, ex );
D
duke 已提交
2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752
}

//=============================================================================
//-------------------------------adr_type--------------------------------------
// Do we Match on this edge index or not?  Do not match memory
const TypePtr* ClearArrayNode::adr_type() const {
  Node *adr = in(3);
  return MemNode::calculate_adr_type(adr->bottom_type());
}

//------------------------------match_edge-------------------------------------
// Do we Match on this edge index or not?  Do not match memory
uint ClearArrayNode::match_edge(uint idx) const {
  return idx > 1;
}

//------------------------------Identity---------------------------------------
// Clearing a zero length array does nothing
Node *ClearArrayNode::Identity( PhaseTransform *phase ) {
2753
  return phase->type(in(2))->higher_equal(TypeX::ZERO)  ? in(1) : this;
D
duke 已提交
2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787
}

//------------------------------Idealize---------------------------------------
// Clearing a short array is faster with stores
Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape){
  const int unit = BytesPerLong;
  const TypeX* t = phase->type(in(2))->isa_intptr_t();
  if (!t)  return NULL;
  if (!t->is_con())  return NULL;
  intptr_t raw_count = t->get_con();
  intptr_t size = raw_count;
  if (!Matcher::init_array_count_is_in_bytes) size *= unit;
  // Clearing nothing uses the Identity call.
  // Negative clears are possible on dead ClearArrays
  // (see jck test stmt114.stmt11402.val).
  if (size <= 0 || size % unit != 0)  return NULL;
  intptr_t count = size / unit;
  // Length too long; use fast hardware clear
  if (size > Matcher::init_array_short_size)  return NULL;
  Node *mem = in(1);
  if( phase->type(mem)==Type::TOP ) return NULL;
  Node *adr = in(3);
  const Type* at = phase->type(adr);
  if( at==Type::TOP ) return NULL;
  const TypePtr* atp = at->isa_ptr();
  // adjust atp to be the correct array element address type
  if (atp == NULL)  atp = TypePtr::BOTTOM;
  else              atp = atp->add_offset(Type::OffsetBot);
  // Get base for derived pointer purposes
  if( adr->Opcode() != Op_AddP ) Unimplemented();
  Node *base = adr->in(1);

  Node *zero = phase->makecon(TypeLong::ZERO);
  Node *off  = phase->MakeConX(BytesPerLong);
2788
  mem = new (phase->C) StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
D
duke 已提交
2789 2790 2791
  count--;
  while( count-- ) {
    mem = phase->transform(mem);
2792
    adr = phase->transform(new (phase->C) AddPNode(base,adr,off));
2793
    mem = new (phase->C) StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
D
duke 已提交
2794 2795 2796 2797
  }
  return mem;
}

2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822
//----------------------------step_through----------------------------------
// Return allocation input memory edge if it is different instance
// or itself if it is the one we are looking for.
bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
  Node* n = *np;
  assert(n->is_ClearArray(), "sanity");
  intptr_t offset;
  AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
  // This method is called only before Allocate nodes are expanded during
  // macro nodes expansion. Before that ClearArray nodes are only generated
  // in LibraryCallKit::generate_arraycopy() which follows allocations.
  assert(alloc != NULL, "should have allocation");
  if (alloc->_idx == instance_id) {
    // Can not bypass initialization of the instance we are looking for.
    return false;
  }
  // Otherwise skip it.
  InitializeNode* init = alloc->initialization();
  if (init != NULL)
    *np = init->in(TypeFunc::Memory);
  else
    *np = alloc->in(TypeFunc::Memory);
  return true;
}

D
duke 已提交
2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833
//----------------------------clear_memory-------------------------------------
// Generate code to initialize object storage to zero.
Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
                                   intptr_t start_offset,
                                   Node* end_offset,
                                   PhaseGVN* phase) {
  Compile* C = phase->C;
  intptr_t offset = start_offset;

  int unit = BytesPerLong;
  if ((offset % unit) != 0) {
2834
    Node* adr = new (C) AddPNode(dest, dest, phase->MakeConX(offset));
D
duke 已提交
2835 2836
    adr = phase->transform(adr);
    const TypePtr* atp = TypeRawPtr::BOTTOM;
2837
    mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
D
duke 已提交
2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850
    mem = phase->transform(mem);
    offset += BytesPerInt;
  }
  assert((offset % unit) == 0, "");

  // Initialize the remaining stuff, if any, with a ClearArray.
  return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
}

Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
                                   Node* start_offset,
                                   Node* end_offset,
                                   PhaseGVN* phase) {
2851 2852 2853 2854 2855
  if (start_offset == end_offset) {
    // nothing to do
    return mem;
  }

D
duke 已提交
2856 2857 2858 2859 2860 2861 2862 2863
  Compile* C = phase->C;
  int unit = BytesPerLong;
  Node* zbase = start_offset;
  Node* zend  = end_offset;

  // Scale to the unit required by the CPU:
  if (!Matcher::init_array_count_is_in_bytes) {
    Node* shift = phase->intcon(exact_log2(unit));
2864 2865
    zbase = phase->transform( new(C) URShiftXNode(zbase, shift) );
    zend  = phase->transform( new(C) URShiftXNode(zend,  shift) );
D
duke 已提交
2866 2867 2868
  }

  // Bulk clear double-words
2869
  Node* zsize = phase->transform( new(C) SubXNode(zend, zbase) );
2870 2871
  Node* adr = phase->transform( new(C) AddPNode(dest, dest, start_offset) );
  mem = new (C) ClearArrayNode(ctl, mem, zsize, adr);
D
duke 已提交
2872 2873 2874 2875 2876 2877 2878
  return phase->transform(mem);
}

Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
                                   intptr_t start_offset,
                                   intptr_t end_offset,
                                   PhaseGVN* phase) {
2879 2880 2881 2882 2883
  if (start_offset == end_offset) {
    // nothing to do
    return mem;
  }

D
duke 已提交
2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894
  Compile* C = phase->C;
  assert((end_offset % BytesPerInt) == 0, "odd end offset");
  intptr_t done_offset = end_offset;
  if ((done_offset % BytesPerLong) != 0) {
    done_offset -= BytesPerInt;
  }
  if (done_offset > start_offset) {
    mem = clear_memory(ctl, mem, dest,
                       start_offset, phase->MakeConX(done_offset), phase);
  }
  if (done_offset < end_offset) { // emit the final 32-bit store
2895
    Node* adr = new (C) AddPNode(dest, dest, phase->MakeConX(done_offset));
D
duke 已提交
2896 2897
    adr = phase->transform(adr);
    const TypePtr* atp = TypeRawPtr::BOTTOM;
2898
    mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
D
duke 已提交
2899 2900 2901 2902 2903 2904 2905 2906
    mem = phase->transform(mem);
    done_offset += BytesPerInt;
  }
  assert(done_offset == end_offset, "");
  return mem;
}

//=============================================================================
2907 2908 2909
// Do not match memory edge.
uint StrIntrinsicNode::match_edge(uint idx) const {
  return idx == 2 || idx == 3;
D
duke 已提交
2910 2911 2912 2913 2914
}

//------------------------------Ideal------------------------------------------
// Return a node which is more "ideal" than the current node.  Strip out
// control copies
2915 2916
Node *StrIntrinsicNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  if (remove_dead_region(phase, can_reshape)) return this;
2917 2918
  // Don't bother trying to transform a dead node
  if (in(0) && in(0)->is_top())  return NULL;
2919

K
kvn 已提交
2920 2921 2922 2923 2924 2925 2926 2927 2928 2929
  if (can_reshape) {
    Node* mem = phase->transform(in(MemNode::Memory));
    // If transformed to a MergeMem, get the desired slice
    uint alias_idx = phase->C->get_alias_index(adr_type());
    mem = mem->is_MergeMem() ? mem->as_MergeMem()->memory_at(alias_idx) : mem;
    if (mem != in(MemNode::Memory)) {
      set_req(MemNode::Memory, mem);
      return this;
    }
  }
2930
  return NULL;
C
cfang 已提交
2931
}
D
duke 已提交
2932

2933 2934 2935 2936 2937 2938
//------------------------------Value------------------------------------------
const Type *StrIntrinsicNode::Value( PhaseTransform *phase ) const {
  if (in(0) && phase->type(in(0)) == Type::TOP) return Type::TOP;
  return bottom_type();
}

2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958
//=============================================================================
//------------------------------match_edge-------------------------------------
// Do not match memory edge
uint EncodeISOArrayNode::match_edge(uint idx) const {
  return idx == 2 || idx == 3; // EncodeISOArray src (Binary dst len)
}

//------------------------------Ideal------------------------------------------
// Return a node which is more "ideal" than the current node.  Strip out
// control copies
Node *EncodeISOArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  return remove_dead_region(phase, can_reshape) ? this : NULL;
}

//------------------------------Value------------------------------------------
const Type *EncodeISOArrayNode::Value(PhaseTransform *phase) const {
  if (in(0) && phase->type(in(0)) == Type::TOP) return Type::TOP;
  return bottom_type();
}

D
duke 已提交
2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981
//=============================================================================
MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
  : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
    _adr_type(C->get_adr_type(alias_idx))
{
  init_class_id(Class_MemBar);
  Node* top = C->top();
  init_req(TypeFunc::I_O,top);
  init_req(TypeFunc::FramePtr,top);
  init_req(TypeFunc::ReturnAdr,top);
  if (precedent != NULL)
    init_req(TypeFunc::Parms, precedent);
}

//------------------------------cmp--------------------------------------------
uint MemBarNode::hash() const { return NO_HASH; }
uint MemBarNode::cmp( const Node &n ) const {
  return (&n == this);          // Always fail except on self
}

//------------------------------make-------------------------------------------
MemBarNode* MemBarNode::make(Compile* C, int opcode, int atp, Node* pn) {
  switch (opcode) {
2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992
  case Op_MemBarAcquire:     return new(C) MemBarAcquireNode(C, atp, pn);
  case Op_LoadFence:         return new(C) LoadFenceNode(C, atp, pn);
  case Op_MemBarRelease:     return new(C) MemBarReleaseNode(C, atp, pn);
  case Op_StoreFence:        return new(C) StoreFenceNode(C, atp, pn);
  case Op_MemBarAcquireLock: return new(C) MemBarAcquireLockNode(C, atp, pn);
  case Op_MemBarReleaseLock: return new(C) MemBarReleaseLockNode(C, atp, pn);
  case Op_MemBarVolatile:    return new(C) MemBarVolatileNode(C, atp, pn);
  case Op_MemBarCPUOrder:    return new(C) MemBarCPUOrderNode(C, atp, pn);
  case Op_Initialize:        return new(C) InitializeNode(C, atp, pn);
  case Op_MemBarStoreStore:  return new(C) MemBarStoreStoreNode(C, atp, pn);
  default: ShouldNotReachHere(); return NULL;
D
duke 已提交
2993 2994 2995 2996 2997 2998 2999
  }
}

//------------------------------Ideal------------------------------------------
// Return a node which is more "ideal" than the current node.  Strip out
// control copies
Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3000
  if (remove_dead_region(phase, can_reshape)) return this;
3001
  // Don't bother trying to transform a dead node
3002 3003 3004
  if (in(0) && in(0)->is_top()) {
    return NULL;
  }
3005 3006

  // Eliminate volatile MemBars for scalar replaced objects.
3007 3008 3009 3010 3011 3012
  if (can_reshape && req() == (Precedent+1)) {
    bool eliminate = false;
    int opc = Opcode();
    if ((opc == Op_MemBarAcquire || opc == Op_MemBarVolatile)) {
      // Volatile field loads and stores.
      Node* my_mem = in(MemBarNode::Precedent);
3013 3014
      // The MembarAquire may keep an unused LoadNode alive through the Precedent edge
      if ((my_mem != NULL) && (opc == Op_MemBarAcquire) && (my_mem->outcnt() == 1)) {
A
adlertz 已提交
3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027
        // if the Precedent is a decodeN and its input (a Load) is used at more than one place,
        // replace this Precedent (decodeN) with the Load instead.
        if ((my_mem->Opcode() == Op_DecodeN) && (my_mem->in(1)->outcnt() > 1))  {
          Node* load_node = my_mem->in(1);
          set_req(MemBarNode::Precedent, load_node);
          phase->is_IterGVN()->_worklist.push(my_mem);
          my_mem = load_node;
        } else {
          assert(my_mem->unique_out() == this, "sanity");
          del_req(Precedent);
          phase->is_IterGVN()->_worklist.push(my_mem); // remove dead node later
          my_mem = NULL;
        }
3028
      }
3029 3030 3031 3032 3033 3034 3035 3036
      if (my_mem != NULL && my_mem->is_Mem()) {
        const TypeOopPtr* t_oop = my_mem->in(MemNode::Address)->bottom_type()->isa_oopptr();
        // Check for scalar replaced object reference.
        if( t_oop != NULL && t_oop->is_known_instance_field() &&
            t_oop->offset() != Type::OffsetBot &&
            t_oop->offset() != Type::OffsetTop) {
          eliminate = true;
        }
3037
      }
3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054
    } else if (opc == Op_MemBarRelease) {
      // Final field stores.
      Node* alloc = AllocateNode::Ideal_allocation(in(MemBarNode::Precedent), phase);
      if ((alloc != NULL) && alloc->is_Allocate() &&
          alloc->as_Allocate()->_is_non_escaping) {
        // The allocated object does not escape.
        eliminate = true;
      }
    }
    if (eliminate) {
      // Replace MemBar projections by its inputs.
      PhaseIterGVN* igvn = phase->is_IterGVN();
      igvn->replace_node(proj_out(TypeFunc::Memory), in(TypeFunc::Memory));
      igvn->replace_node(proj_out(TypeFunc::Control), in(TypeFunc::Control));
      // Must return either the original node (now dead) or a new node
      // (Do not return a top here, since that would break the uniqueness of top.)
      return new (phase->C) ConINode(TypeInt::ZERO);
3055 3056 3057
    }
  }
  return NULL;
D
duke 已提交
3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073
}

//------------------------------Value------------------------------------------
const Type *MemBarNode::Value( PhaseTransform *phase ) const {
  if( !in(0) ) return Type::TOP;
  if( phase->type(in(0)) == Type::TOP )
    return Type::TOP;
  return TypeTuple::MEMBAR;
}

//------------------------------match------------------------------------------
// Construct projections for memory.
Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
  switch (proj->_con) {
  case TypeFunc::Control:
  case TypeFunc::Memory:
3074
    return new (m->C) MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
D
duke 已提交
3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137
  }
  ShouldNotReachHere();
  return NULL;
}

//===========================InitializeNode====================================
// SUMMARY:
// This node acts as a memory barrier on raw memory, after some raw stores.
// The 'cooked' oop value feeds from the Initialize, not the Allocation.
// The Initialize can 'capture' suitably constrained stores as raw inits.
// It can coalesce related raw stores into larger units (called 'tiles').
// It can avoid zeroing new storage for memory units which have raw inits.
// At macro-expansion, it is marked 'complete', and does not optimize further.
//
// EXAMPLE:
// The object 'new short[2]' occupies 16 bytes in a 32-bit machine.
//   ctl = incoming control; mem* = incoming memory
// (Note:  A star * on a memory edge denotes I/O and other standard edges.)
// First allocate uninitialized memory and fill in the header:
//   alloc = (Allocate ctl mem* 16 #short[].klass ...)
//   ctl := alloc.Control; mem* := alloc.Memory*
//   rawmem = alloc.Memory; rawoop = alloc.RawAddress
// Then initialize to zero the non-header parts of the raw memory block:
//   init = (Initialize alloc.Control alloc.Memory* alloc.RawAddress)
//   ctl := init.Control; mem.SLICE(#short[*]) := init.Memory
// After the initialize node executes, the object is ready for service:
//   oop := (CheckCastPP init.Control alloc.RawAddress #short[])
// Suppose its body is immediately initialized as {1,2}:
//   store1 = (StoreC init.Control init.Memory (+ oop 12) 1)
//   store2 = (StoreC init.Control store1      (+ oop 14) 2)
//   mem.SLICE(#short[*]) := store2
//
// DETAILS:
// An InitializeNode collects and isolates object initialization after
// an AllocateNode and before the next possible safepoint.  As a
// memory barrier (MemBarNode), it keeps critical stores from drifting
// down past any safepoint or any publication of the allocation.
// Before this barrier, a newly-allocated object may have uninitialized bits.
// After this barrier, it may be treated as a real oop, and GC is allowed.
//
// The semantics of the InitializeNode include an implicit zeroing of
// the new object from object header to the end of the object.
// (The object header and end are determined by the AllocateNode.)
//
// Certain stores may be added as direct inputs to the InitializeNode.
// These stores must update raw memory, and they must be to addresses
// derived from the raw address produced by AllocateNode, and with
// a constant offset.  They must be ordered by increasing offset.
// The first one is at in(RawStores), the last at in(req()-1).
// Unlike most memory operations, they are not linked in a chain,
// but are displayed in parallel as users of the rawmem output of
// the allocation.
//
// (See comments in InitializeNode::capture_store, which continue
// the example given above.)
//
// When the associated Allocate is macro-expanded, the InitializeNode
// may be rewritten to optimize collected stores.  A ClearArrayNode
// may also be created at that point to represent any required zeroing.
// The InitializeNode is then marked 'complete', prohibiting further
// capturing of nearby memory operations.
//
// During macro-expansion, all captured initializations which store
T
twisti 已提交
3138
// constant values of 32 bits or smaller are coalesced (if advantageous)
D
duke 已提交
3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172
// into larger 'tiles' 32 or 64 bits.  This allows an object to be
// initialized in fewer memory operations.  Memory words which are
// covered by neither tiles nor non-constant stores are pre-zeroed
// by explicit stores of zero.  (The code shape happens to do all
// zeroing first, then all other stores, with both sequences occurring
// in order of ascending offsets.)
//
// Alternatively, code may be inserted between an AllocateNode and its
// InitializeNode, to perform arbitrary initialization of the new object.
// E.g., the object copying intrinsics insert complex data transfers here.
// The initialization must then be marked as 'complete' disable the
// built-in zeroing semantics and the collection of initializing stores.
//
// While an InitializeNode is incomplete, reads from the memory state
// produced by it are optimizable if they match the control edge and
// new oop address associated with the allocation/initialization.
// They return a stored value (if the offset matches) or else zero.
// A write to the memory state, if it matches control and address,
// and if it is to a constant offset, may be 'captured' by the
// InitializeNode.  It is cloned as a raw memory operation and rewired
// inside the initialization, to the raw oop produced by the allocation.
// Operations on addresses which are provably distinct (e.g., to
// other AllocateNodes) are allowed to bypass the initialization.
//
// The effect of all this is to consolidate object initialization
// (both arrays and non-arrays, both piecewise and bulk) into a
// single location, where it can be optimized as a unit.
//
// Only stores with an offset less than TrackedInitializationLimit words
// will be considered for capture by an InitializeNode.  This puts a
// reasonable limit on the complexity of optimized initializations.

//---------------------------InitializeNode------------------------------------
InitializeNode::InitializeNode(Compile* C, int adr_type, Node* rawoop)
3173
  : _is_complete(Incomplete), _does_not_escape(false),
D
duke 已提交
3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210
    MemBarNode(C, adr_type, rawoop)
{
  init_class_id(Class_Initialize);

  assert(adr_type == Compile::AliasIdxRaw, "only valid atp");
  assert(in(RawAddress) == rawoop, "proper init");
  // Note:  allocation() can be NULL, for secondary initialization barriers
}

// Since this node is not matched, it will be processed by the
// register allocator.  Declare that there are no constraints
// on the allocation of the RawAddress edge.
const RegMask &InitializeNode::in_RegMask(uint idx) const {
  // This edge should be set to top, by the set_complete.  But be conservative.
  if (idx == InitializeNode::RawAddress)
    return *(Compile::current()->matcher()->idealreg2spillmask[in(idx)->ideal_reg()]);
  return RegMask::Empty;
}

Node* InitializeNode::memory(uint alias_idx) {
  Node* mem = in(Memory);
  if (mem->is_MergeMem()) {
    return mem->as_MergeMem()->memory_at(alias_idx);
  } else {
    // incoming raw memory is not split
    return mem;
  }
}

bool InitializeNode::is_non_zero() {
  if (is_complete())  return false;
  remove_extra_zeroes();
  return (req() > RawStores);
}

void InitializeNode::set_complete(PhaseGVN* phase) {
  assert(!is_complete(), "caller responsibility");
3211
  _is_complete = Complete;
D
duke 已提交
3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266

  // After this node is complete, it contains a bunch of
  // raw-memory initializations.  There is no need for
  // it to have anything to do with non-raw memory effects.
  // Therefore, tell all non-raw users to re-optimize themselves,
  // after skipping the memory effects of this initialization.
  PhaseIterGVN* igvn = phase->is_IterGVN();
  if (igvn)  igvn->add_users_to_worklist(this);
}

// convenience function
// return false if the init contains any stores already
bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
  InitializeNode* init = initialization();
  if (init == NULL || init->is_complete())  return false;
  init->remove_extra_zeroes();
  // for now, if this allocation has already collected any inits, bail:
  if (init->is_non_zero())  return false;
  init->set_complete(phase);
  return true;
}

void InitializeNode::remove_extra_zeroes() {
  if (req() == RawStores)  return;
  Node* zmem = zero_memory();
  uint fill = RawStores;
  for (uint i = fill; i < req(); i++) {
    Node* n = in(i);
    if (n->is_top() || n == zmem)  continue;  // skip
    if (fill < i)  set_req(fill, n);          // compact
    ++fill;
  }
  // delete any empty spaces created:
  while (fill < req()) {
    del_req(fill);
  }
}

// Helper for remembering which stores go with which offsets.
intptr_t InitializeNode::get_store_offset(Node* st, PhaseTransform* phase) {
  if (!st->is_Store())  return -1;  // can happen to dead code via subsume_node
  intptr_t offset = -1;
  Node* base = AddPNode::Ideal_base_and_offset(st->in(MemNode::Address),
                                               phase, offset);
  if (base == NULL)     return -1;  // something is dead,
  if (offset < 0)       return -1;  //        dead, dead
  return offset;
}

// Helper for proving that an initialization expression is
// "simple enough" to be folded into an object initialization.
// Attempts to prove that a store's initial value 'n' can be captured
// within the initialization without creating a vicious cycle, such as:
//     { Foo p = new Foo(); p.next = p; }
// True for constants and parameters and small combinations thereof.
3267
bool InitializeNode::detect_init_independence(Node* n, int& count) {
D
duke 已提交
3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284
  if (n == NULL)      return true;   // (can this really happen?)
  if (n->is_Proj())   n = n->in(0);
  if (n == this)      return false;  // found a cycle
  if (n->is_Con())    return true;
  if (n->is_Start())  return true;   // params, etc., are OK
  if (n->is_Root())   return true;   // even better

  Node* ctl = n->in(0);
  if (ctl != NULL && !ctl->is_top()) {
    if (ctl->is_Proj())  ctl = ctl->in(0);
    if (ctl == this)  return false;

    // If we already know that the enclosing memory op is pinned right after
    // the init, then any control flow that the store has picked up
    // must have preceded the init, or else be equal to the init.
    // Even after loop optimizations (which might change control edges)
    // a store is never pinned *before* the availability of its inputs.
3285
    if (!MemNode::all_controls_dominate(n, this))
D
duke 已提交
3286 3287 3288 3289 3290 3291 3292 3293 3294 3295
      return false;                  // failed to prove a good control
  }

  // Check data edges for possible dependencies on 'this'.
  if ((count += 1) > 20)  return false;  // complexity limit
  for (uint i = 1; i < n->req(); i++) {
    Node* m = n->in(i);
    if (m == NULL || m == n || m->is_top())  continue;
    uint first_i = n->find_edge(m);
    if (i != first_i)  continue;  // process duplicate edge just once
3296
    if (!detect_init_independence(m, count)) {
D
duke 已提交
3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307
      return false;
    }
  }

  return true;
}

// Here are all the checks a Store must pass before it can be moved into
// an initialization.  Returns zero if a check fails.
// On success, returns the (constant) offset to which the store applies,
// within the initialized memory.
3308
intptr_t InitializeNode::can_capture_store(StoreNode* st, PhaseTransform* phase, bool can_reshape) {
D
duke 已提交
3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326
  const int FAIL = 0;
  if (st->req() != MemNode::ValueIn + 1)
    return FAIL;                // an inscrutable StoreNode (card mark?)
  Node* ctl = st->in(MemNode::Control);
  if (!(ctl != NULL && ctl->is_Proj() && ctl->in(0) == this))
    return FAIL;                // must be unconditional after the initialization
  Node* mem = st->in(MemNode::Memory);
  if (!(mem->is_Proj() && mem->in(0) == this))
    return FAIL;                // must not be preceded by other stores
  Node* adr = st->in(MemNode::Address);
  intptr_t offset;
  AllocateNode* alloc = AllocateNode::Ideal_allocation(adr, phase, offset);
  if (alloc == NULL)
    return FAIL;                // inscrutable address
  if (alloc != allocation())
    return FAIL;                // wrong allocation!  (store needs to float up)
  Node* val = st->in(MemNode::ValueIn);
  int complexity_count = 0;
3327
  if (!detect_init_independence(val, complexity_count))
D
duke 已提交
3328 3329
    return FAIL;                // stored value must be 'simple enough'

3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414
  // The Store can be captured only if nothing after the allocation
  // and before the Store is using the memory location that the store
  // overwrites.
  bool failed = false;
  // If is_complete_with_arraycopy() is true the shape of the graph is
  // well defined and is safe so no need for extra checks.
  if (!is_complete_with_arraycopy()) {
    // We are going to look at each use of the memory state following
    // the allocation to make sure nothing reads the memory that the
    // Store writes.
    const TypePtr* t_adr = phase->type(adr)->isa_ptr();
    int alias_idx = phase->C->get_alias_index(t_adr);
    ResourceMark rm;
    Unique_Node_List mems;
    mems.push(mem);
    Node* unique_merge = NULL;
    for (uint next = 0; next < mems.size(); ++next) {
      Node *m  = mems.at(next);
      for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) {
        Node *n = m->fast_out(j);
        if (n->outcnt() == 0) {
          continue;
        }
        if (n == st) {
          continue;
        } else if (n->in(0) != NULL && n->in(0) != ctl) {
          // If the control of this use is different from the control
          // of the Store which is right after the InitializeNode then
          // this node cannot be between the InitializeNode and the
          // Store.
          continue;
        } else if (n->is_MergeMem()) {
          if (n->as_MergeMem()->memory_at(alias_idx) == m) {
            // We can hit a MergeMemNode (that will likely go away
            // later) that is a direct use of the memory state
            // following the InitializeNode on the same slice as the
            // store node that we'd like to capture. We need to check
            // the uses of the MergeMemNode.
            mems.push(n);
          }
        } else if (n->is_Mem()) {
          Node* other_adr = n->in(MemNode::Address);
          if (other_adr == adr) {
            failed = true;
            break;
          } else {
            const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
            if (other_t_adr != NULL) {
              int other_alias_idx = phase->C->get_alias_index(other_t_adr);
              if (other_alias_idx == alias_idx) {
                // A load from the same memory slice as the store right
                // after the InitializeNode. We check the control of the
                // object/array that is loaded from. If it's the same as
                // the store control then we cannot capture the store.
                assert(!n->is_Store(), "2 stores to same slice on same control?");
                Node* base = other_adr;
                assert(base->is_AddP(), err_msg_res("should be addp but is %s", base->Name()));
                base = base->in(AddPNode::Base);
                if (base != NULL) {
                  base = base->uncast();
                  if (base->is_Proj() && base->in(0) == alloc) {
                    failed = true;
                    break;
                  }
                }
              }
            }
          }
        } else {
          failed = true;
          break;
        }
      }
    }
  }
  if (failed) {
    if (!can_reshape) {
      // We decided we couldn't capture the store during parsing. We
      // should try again during the next IGVN once the graph is
      // cleaner.
      phase->C->record_for_igvn(st);
    }
    return FAIL;
  }

D
duke 已提交
3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435
  return offset;                // success
}

// Find the captured store in(i) which corresponds to the range
// [start..start+size) in the initialized object.
// If there is one, return its index i.  If there isn't, return the
// negative of the index where it should be inserted.
// Return 0 if the queried range overlaps an initialization boundary
// or if dead code is encountered.
// If size_in_bytes is zero, do not bother with overlap checks.
int InitializeNode::captured_store_insertion_point(intptr_t start,
                                                   int size_in_bytes,
                                                   PhaseTransform* phase) {
  const int FAIL = 0, MAX_STORE = BytesPerLong;

  if (is_complete())
    return FAIL;                // arraycopy got here first; punt

  assert(allocation() != NULL, "must be present");

  // no negatives, no header fields:
3436
  if (start < (intptr_t) allocation()->minimum_header_size())  return FAIL;
D
duke 已提交
3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499

  // after a certain size, we bail out on tracking all the stores:
  intptr_t ti_limit = (TrackedInitializationLimit * HeapWordSize);
  if (start >= ti_limit)  return FAIL;

  for (uint i = InitializeNode::RawStores, limit = req(); ; ) {
    if (i >= limit)  return -(int)i; // not found; here is where to put it

    Node*    st     = in(i);
    intptr_t st_off = get_store_offset(st, phase);
    if (st_off < 0) {
      if (st != zero_memory()) {
        return FAIL;            // bail out if there is dead garbage
      }
    } else if (st_off > start) {
      // ...we are done, since stores are ordered
      if (st_off < start + size_in_bytes) {
        return FAIL;            // the next store overlaps
      }
      return -(int)i;           // not found; here is where to put it
    } else if (st_off < start) {
      if (size_in_bytes != 0 &&
          start < st_off + MAX_STORE &&
          start < st_off + st->as_Store()->memory_size()) {
        return FAIL;            // the previous store overlaps
      }
    } else {
      if (size_in_bytes != 0 &&
          st->as_Store()->memory_size() != size_in_bytes) {
        return FAIL;            // mismatched store size
      }
      return i;
    }

    ++i;
  }
}

// Look for a captured store which initializes at the offset 'start'
// with the given size.  If there is no such store, and no other
// initialization interferes, then return zero_memory (the memory
// projection of the AllocateNode).
Node* InitializeNode::find_captured_store(intptr_t start, int size_in_bytes,
                                          PhaseTransform* phase) {
  assert(stores_are_sane(phase), "");
  int i = captured_store_insertion_point(start, size_in_bytes, phase);
  if (i == 0) {
    return NULL;                // something is dead
  } else if (i < 0) {
    return zero_memory();       // just primordial zero bits here
  } else {
    Node* st = in(i);           // here is the store at this position
    assert(get_store_offset(st->as_Store(), phase) == start, "sanity");
    return st;
  }
}

// Create, as a raw pointer, an address within my new object at 'offset'.
Node* InitializeNode::make_raw_address(intptr_t offset,
                                       PhaseTransform* phase) {
  Node* addr = in(RawAddress);
  if (offset != 0) {
    Compile* C = phase->C;
3500
    addr = phase->transform( new (C) AddPNode(C->top(), addr,
D
duke 已提交
3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524
                                                 phase->MakeConX(offset)) );
  }
  return addr;
}

// Clone the given store, converting it into a raw store
// initializing a field or element of my new object.
// Caller is responsible for retiring the original store,
// with subsume_node or the like.
//
// From the example above InitializeNode::InitializeNode,
// here are the old stores to be captured:
//   store1 = (StoreC init.Control init.Memory (+ oop 12) 1)
//   store2 = (StoreC init.Control store1      (+ oop 14) 2)
//
// Here is the changed code; note the extra edges on init:
//   alloc = (Allocate ...)
//   rawoop = alloc.RawAddress
//   rawstore1 = (StoreC alloc.Control alloc.Memory (+ rawoop 12) 1)
//   rawstore2 = (StoreC alloc.Control alloc.Memory (+ rawoop 14) 2)
//   init = (Initialize alloc.Control alloc.Memory rawoop
//                      rawstore1 rawstore2)
//
Node* InitializeNode::capture_store(StoreNode* st, intptr_t start,
3525
                                    PhaseTransform* phase, bool can_reshape) {
D
duke 已提交
3526 3527 3528
  assert(stores_are_sane(phase), "");

  if (start < 0)  return NULL;
3529
  assert(can_capture_store(st, phase, can_reshape) == start, "sanity");
D
duke 已提交
3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772

  Compile* C = phase->C;
  int size_in_bytes = st->memory_size();
  int i = captured_store_insertion_point(start, size_in_bytes, phase);
  if (i == 0)  return NULL;     // bail out
  Node* prev_mem = NULL;        // raw memory for the captured store
  if (i > 0) {
    prev_mem = in(i);           // there is a pre-existing store under this one
    set_req(i, C->top());       // temporarily disconnect it
    // See StoreNode::Ideal 'st->outcnt() == 1' for the reason to disconnect.
  } else {
    i = -i;                     // no pre-existing store
    prev_mem = zero_memory();   // a slice of the newly allocated object
    if (i > InitializeNode::RawStores && in(i-1) == prev_mem)
      set_req(--i, C->top());   // reuse this edge; it has been folded away
    else
      ins_req(i, C->top());     // build a new edge
  }
  Node* new_st = st->clone();
  new_st->set_req(MemNode::Control, in(Control));
  new_st->set_req(MemNode::Memory,  prev_mem);
  new_st->set_req(MemNode::Address, make_raw_address(start, phase));
  new_st = phase->transform(new_st);

  // At this point, new_st might have swallowed a pre-existing store
  // at the same offset, or perhaps new_st might have disappeared,
  // if it redundantly stored the same value (or zero to fresh memory).

  // In any case, wire it in:
  set_req(i, new_st);

  // The caller may now kill the old guy.
  DEBUG_ONLY(Node* check_st = find_captured_store(start, size_in_bytes, phase));
  assert(check_st == new_st || check_st == NULL, "must be findable");
  assert(!is_complete(), "");
  return new_st;
}

static bool store_constant(jlong* tiles, int num_tiles,
                           intptr_t st_off, int st_size,
                           jlong con) {
  if ((st_off & (st_size-1)) != 0)
    return false;               // strange store offset (assume size==2**N)
  address addr = (address)tiles + st_off;
  assert(st_off >= 0 && addr+st_size <= (address)&tiles[num_tiles], "oob");
  switch (st_size) {
  case sizeof(jbyte):  *(jbyte*) addr = (jbyte) con; break;
  case sizeof(jchar):  *(jchar*) addr = (jchar) con; break;
  case sizeof(jint):   *(jint*)  addr = (jint)  con; break;
  case sizeof(jlong):  *(jlong*) addr = (jlong) con; break;
  default: return false;        // strange store size (detect size!=2**N here)
  }
  return true;                  // return success to caller
}

// Coalesce subword constants into int constants and possibly
// into long constants.  The goal, if the CPU permits,
// is to initialize the object with a small number of 64-bit tiles.
// Also, convert floating-point constants to bit patterns.
// Non-constants are not relevant to this pass.
//
// In terms of the running example on InitializeNode::InitializeNode
// and InitializeNode::capture_store, here is the transformation
// of rawstore1 and rawstore2 into rawstore12:
//   alloc = (Allocate ...)
//   rawoop = alloc.RawAddress
//   tile12 = 0x00010002
//   rawstore12 = (StoreI alloc.Control alloc.Memory (+ rawoop 12) tile12)
//   init = (Initialize alloc.Control alloc.Memory rawoop rawstore12)
//
void
InitializeNode::coalesce_subword_stores(intptr_t header_size,
                                        Node* size_in_bytes,
                                        PhaseGVN* phase) {
  Compile* C = phase->C;

  assert(stores_are_sane(phase), "");
  // Note:  After this pass, they are not completely sane,
  // since there may be some overlaps.

  int old_subword = 0, old_long = 0, new_int = 0, new_long = 0;

  intptr_t ti_limit = (TrackedInitializationLimit * HeapWordSize);
  intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, ti_limit);
  size_limit = MIN2(size_limit, ti_limit);
  size_limit = align_size_up(size_limit, BytesPerLong);
  int num_tiles = size_limit / BytesPerLong;

  // allocate space for the tile map:
  const int small_len = DEBUG_ONLY(true ? 3 :) 30; // keep stack frames small
  jlong  tiles_buf[small_len];
  Node*  nodes_buf[small_len];
  jlong  inits_buf[small_len];
  jlong* tiles = ((num_tiles <= small_len) ? &tiles_buf[0]
                  : NEW_RESOURCE_ARRAY(jlong, num_tiles));
  Node** nodes = ((num_tiles <= small_len) ? &nodes_buf[0]
                  : NEW_RESOURCE_ARRAY(Node*, num_tiles));
  jlong* inits = ((num_tiles <= small_len) ? &inits_buf[0]
                  : NEW_RESOURCE_ARRAY(jlong, num_tiles));
  // tiles: exact bitwise model of all primitive constants
  // nodes: last constant-storing node subsumed into the tiles model
  // inits: which bytes (in each tile) are touched by any initializations

  //// Pass A: Fill in the tile model with any relevant stores.

  Copy::zero_to_bytes(tiles, sizeof(tiles[0]) * num_tiles);
  Copy::zero_to_bytes(nodes, sizeof(nodes[0]) * num_tiles);
  Copy::zero_to_bytes(inits, sizeof(inits[0]) * num_tiles);
  Node* zmem = zero_memory(); // initially zero memory state
  for (uint i = InitializeNode::RawStores, limit = req(); i < limit; i++) {
    Node* st = in(i);
    intptr_t st_off = get_store_offset(st, phase);

    // Figure out the store's offset and constant value:
    if (st_off < header_size)             continue; //skip (ignore header)
    if (st->in(MemNode::Memory) != zmem)  continue; //skip (odd store chain)
    int st_size = st->as_Store()->memory_size();
    if (st_off + st_size > size_limit)    break;

    // Record which bytes are touched, whether by constant or not.
    if (!store_constant(inits, num_tiles, st_off, st_size, (jlong) -1))
      continue;                 // skip (strange store size)

    const Type* val = phase->type(st->in(MemNode::ValueIn));
    if (!val->singleton())                continue; //skip (non-con store)
    BasicType type = val->basic_type();

    jlong con = 0;
    switch (type) {
    case T_INT:    con = val->is_int()->get_con();  break;
    case T_LONG:   con = val->is_long()->get_con(); break;
    case T_FLOAT:  con = jint_cast(val->getf());    break;
    case T_DOUBLE: con = jlong_cast(val->getd());   break;
    default:                              continue; //skip (odd store type)
    }

    if (type == T_LONG && Matcher::isSimpleConstant64(con) &&
        st->Opcode() == Op_StoreL) {
      continue;                 // This StoreL is already optimal.
    }

    // Store down the constant.
    store_constant(tiles, num_tiles, st_off, st_size, con);

    intptr_t j = st_off >> LogBytesPerLong;

    if (type == T_INT && st_size == BytesPerInt
        && (st_off & BytesPerInt) == BytesPerInt) {
      jlong lcon = tiles[j];
      if (!Matcher::isSimpleConstant64(lcon) &&
          st->Opcode() == Op_StoreI) {
        // This StoreI is already optimal by itself.
        jint* intcon = (jint*) &tiles[j];
        intcon[1] = 0;  // undo the store_constant()

        // If the previous store is also optimal by itself, back up and
        // undo the action of the previous loop iteration... if we can.
        // But if we can't, just let the previous half take care of itself.
        st = nodes[j];
        st_off -= BytesPerInt;
        con = intcon[0];
        if (con != 0 && st != NULL && st->Opcode() == Op_StoreI) {
          assert(st_off >= header_size, "still ignoring header");
          assert(get_store_offset(st, phase) == st_off, "must be");
          assert(in(i-1) == zmem, "must be");
          DEBUG_ONLY(const Type* tcon = phase->type(st->in(MemNode::ValueIn)));
          assert(con == tcon->is_int()->get_con(), "must be");
          // Undo the effects of the previous loop trip, which swallowed st:
          intcon[0] = 0;        // undo store_constant()
          set_req(i-1, st);     // undo set_req(i, zmem)
          nodes[j] = NULL;      // undo nodes[j] = st
          --old_subword;        // undo ++old_subword
        }
        continue;               // This StoreI is already optimal.
      }
    }

    // This store is not needed.
    set_req(i, zmem);
    nodes[j] = st;              // record for the moment
    if (st_size < BytesPerLong) // something has changed
          ++old_subword;        // includes int/float, but who's counting...
    else  ++old_long;
  }

  if ((old_subword + old_long) == 0)
    return;                     // nothing more to do

  //// Pass B: Convert any non-zero tiles into optimal constant stores.
  // Be sure to insert them before overlapping non-constant stores.
  // (E.g., byte[] x = { 1,2,y,4 }  =>  x[int 0] = 0x01020004, x[2]=y.)
  for (int j = 0; j < num_tiles; j++) {
    jlong con  = tiles[j];
    jlong init = inits[j];
    if (con == 0)  continue;
    jint con0,  con1;           // split the constant, address-wise
    jint init0, init1;          // split the init map, address-wise
    { union { jlong con; jint intcon[2]; } u;
      u.con = con;
      con0  = u.intcon[0];
      con1  = u.intcon[1];
      u.con = init;
      init0 = u.intcon[0];
      init1 = u.intcon[1];
    }

    Node* old = nodes[j];
    assert(old != NULL, "need the prior store");
    intptr_t offset = (j * BytesPerLong);

    bool split = !Matcher::isSimpleConstant64(con);

    if (offset < header_size) {
      assert(offset + BytesPerInt >= header_size, "second int counts");
      assert(*(jint*)&tiles[j] == 0, "junk in header");
      split = true;             // only the second word counts
      // Example:  int a[] = { 42 ... }
    } else if (con0 == 0 && init0 == -1) {
      split = true;             // first word is covered by full inits
      // Example:  int a[] = { ... foo(), 42 ... }
    } else if (con1 == 0 && init1 == -1) {
      split = true;             // second word is covered by full inits
      // Example:  int a[] = { ... 42, foo() ... }
    }

    // Here's a case where init0 is neither 0 nor -1:
    //   byte a[] = { ... 0,0,foo(),0,  0,0,0,42 ... }
    // Assuming big-endian memory, init0, init1 are 0x0000FF00, 0x000000FF.
    // In this case the tile is not split; it is (jlong)42.
    // The big tile is stored down, and then the foo() value is inserted.
    // (If there were foo(),foo() instead of foo(),0, init0 would be -1.)

    Node* ctl = old->in(MemNode::Control);
    Node* adr = make_raw_address(offset, phase);
    const TypePtr* atp = TypeRawPtr::BOTTOM;

    // One or two coalesced stores to plop down.
    Node*    st[2];
    intptr_t off[2];
    int  nst = 0;
    if (!split) {
      ++new_long;
      off[nst] = offset;
3773
      st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp,
3774
                                  phase->longcon(con), T_LONG, MemNode::unordered);
D
duke 已提交
3775 3776 3777 3778 3779
    } else {
      // Omit either if it is a zero.
      if (con0 != 0) {
        ++new_int;
        off[nst]  = offset;
3780
        st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp,
3781
                                    phase->intcon(con0), T_INT, MemNode::unordered);
D
duke 已提交
3782 3783 3784 3785 3786 3787
      }
      if (con1 != 0) {
        ++new_int;
        offset += BytesPerInt;
        adr = make_raw_address(offset, phase);
        off[nst]  = offset;
3788
        st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp,
3789
                                    phase->intcon(con1), T_INT, MemNode::unordered);
D
duke 已提交
3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895
      }
    }

    // Insert second store first, then the first before the second.
    // Insert each one just before any overlapping non-constant stores.
    while (nst > 0) {
      Node* st1 = st[--nst];
      C->copy_node_notes_to(st1, old);
      st1 = phase->transform(st1);
      offset = off[nst];
      assert(offset >= header_size, "do not smash header");
      int ins_idx = captured_store_insertion_point(offset, /*size:*/0, phase);
      guarantee(ins_idx != 0, "must re-insert constant store");
      if (ins_idx < 0)  ins_idx = -ins_idx;  // never overlap
      if (ins_idx > InitializeNode::RawStores && in(ins_idx-1) == zmem)
        set_req(--ins_idx, st1);
      else
        ins_req(ins_idx, st1);
    }
  }

  if (PrintCompilation && WizardMode)
    tty->print_cr("Changed %d/%d subword/long constants into %d/%d int/long",
                  old_subword, old_long, new_int, new_long);
  if (C->log() != NULL)
    C->log()->elem("comment that='%d/%d subword/long to %d/%d int/long'",
                   old_subword, old_long, new_int, new_long);

  // Clean up any remaining occurrences of zmem:
  remove_extra_zeroes();
}

// Explore forward from in(start) to find the first fully initialized
// word, and return its offset.  Skip groups of subword stores which
// together initialize full words.  If in(start) is itself part of a
// fully initialized word, return the offset of in(start).  If there
// are no following full-word stores, or if something is fishy, return
// a negative value.
intptr_t InitializeNode::find_next_fullword_store(uint start, PhaseGVN* phase) {
  int       int_map = 0;
  intptr_t  int_map_off = 0;
  const int FULL_MAP = right_n_bits(BytesPerInt);  // the int_map we hope for

  for (uint i = start, limit = req(); i < limit; i++) {
    Node* st = in(i);

    intptr_t st_off = get_store_offset(st, phase);
    if (st_off < 0)  break;  // return conservative answer

    int st_size = st->as_Store()->memory_size();
    if (st_size >= BytesPerInt && (st_off % BytesPerInt) == 0) {
      return st_off;            // we found a complete word init
    }

    // update the map:

    intptr_t this_int_off = align_size_down(st_off, BytesPerInt);
    if (this_int_off != int_map_off) {
      // reset the map:
      int_map = 0;
      int_map_off = this_int_off;
    }

    int subword_off = st_off - this_int_off;
    int_map |= right_n_bits(st_size) << subword_off;
    if ((int_map & FULL_MAP) == FULL_MAP) {
      return this_int_off;      // we found a complete word init
    }

    // Did this store hit or cross the word boundary?
    intptr_t next_int_off = align_size_down(st_off + st_size, BytesPerInt);
    if (next_int_off == this_int_off + BytesPerInt) {
      // We passed the current int, without fully initializing it.
      int_map_off = next_int_off;
      int_map >>= BytesPerInt;
    } else if (next_int_off > this_int_off + BytesPerInt) {
      // We passed the current and next int.
      return this_int_off + BytesPerInt;
    }
  }

  return -1;
}


// Called when the associated AllocateNode is expanded into CFG.
// At this point, we may perform additional optimizations.
// Linearize the stores by ascending offset, to make memory
// activity as coherent as possible.
Node* InitializeNode::complete_stores(Node* rawctl, Node* rawmem, Node* rawptr,
                                      intptr_t header_size,
                                      Node* size_in_bytes,
                                      PhaseGVN* phase) {
  assert(!is_complete(), "not already complete");
  assert(stores_are_sane(phase), "");
  assert(allocation() != NULL, "must be present");

  remove_extra_zeroes();

  if (ReduceFieldZeroing || ReduceBulkZeroing)
    // reduce instruction count for common initialization patterns
    coalesce_subword_stores(header_size, size_in_bytes, phase);

  Node* zmem = zero_memory();   // initially zero memory state
  Node* inits = zmem;           // accumulating a linearized chain of inits
  #ifdef ASSERT
3896 3897 3898 3899
  intptr_t first_offset = allocation()->minimum_header_size();
  intptr_t last_init_off = first_offset;  // previous init offset
  intptr_t last_init_end = first_offset;  // previous init offset+size
  intptr_t last_tile_end = first_offset;  // previous tile offset+size
D
duke 已提交
3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013
  #endif
  intptr_t zeroes_done = header_size;

  bool do_zeroing = true;       // we might give up if inits are very sparse
  int  big_init_gaps = 0;       // how many large gaps have we seen?

  if (ZeroTLAB)  do_zeroing = false;
  if (!ReduceFieldZeroing && !ReduceBulkZeroing)  do_zeroing = false;

  for (uint i = InitializeNode::RawStores, limit = req(); i < limit; i++) {
    Node* st = in(i);
    intptr_t st_off = get_store_offset(st, phase);
    if (st_off < 0)
      break;                    // unknown junk in the inits
    if (st->in(MemNode::Memory) != zmem)
      break;                    // complicated store chains somehow in list

    int st_size = st->as_Store()->memory_size();
    intptr_t next_init_off = st_off + st_size;

    if (do_zeroing && zeroes_done < next_init_off) {
      // See if this store needs a zero before it or under it.
      intptr_t zeroes_needed = st_off;

      if (st_size < BytesPerInt) {
        // Look for subword stores which only partially initialize words.
        // If we find some, we must lay down some word-level zeroes first,
        // underneath the subword stores.
        //
        // Examples:
        //   byte[] a = { p,q,r,s }  =>  a[0]=p,a[1]=q,a[2]=r,a[3]=s
        //   byte[] a = { x,y,0,0 }  =>  a[0..3] = 0, a[0]=x,a[1]=y
        //   byte[] a = { 0,0,z,0 }  =>  a[0..3] = 0, a[2]=z
        //
        // Note:  coalesce_subword_stores may have already done this,
        // if it was prompted by constant non-zero subword initializers.
        // But this case can still arise with non-constant stores.

        intptr_t next_full_store = find_next_fullword_store(i, phase);

        // In the examples above:
        //   in(i)          p   q   r   s     x   y     z
        //   st_off        12  13  14  15    12  13    14
        //   st_size        1   1   1   1     1   1     1
        //   next_full_s.  12  16  16  16    16  16    16
        //   z's_done      12  16  16  16    12  16    12
        //   z's_needed    12  16  16  16    16  16    16
        //   zsize          0   0   0   0     4   0     4
        if (next_full_store < 0) {
          // Conservative tack:  Zero to end of current word.
          zeroes_needed = align_size_up(zeroes_needed, BytesPerInt);
        } else {
          // Zero to beginning of next fully initialized word.
          // Or, don't zero at all, if we are already in that word.
          assert(next_full_store >= zeroes_needed, "must go forward");
          assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
          zeroes_needed = next_full_store;
        }
      }

      if (zeroes_needed > zeroes_done) {
        intptr_t zsize = zeroes_needed - zeroes_done;
        // Do some incremental zeroing on rawmem, in parallel with inits.
        zeroes_done = align_size_down(zeroes_done, BytesPerInt);
        rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
                                              zeroes_done, zeroes_needed,
                                              phase);
        zeroes_done = zeroes_needed;
        if (zsize > Matcher::init_array_short_size && ++big_init_gaps > 2)
          do_zeroing = false;   // leave the hole, next time
      }
    }

    // Collect the store and move on:
    st->set_req(MemNode::Memory, inits);
    inits = st;                 // put it on the linearized chain
    set_req(i, zmem);           // unhook from previous position

    if (zeroes_done == st_off)
      zeroes_done = next_init_off;

    assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");

    #ifdef ASSERT
    // Various order invariants.  Weaker than stores_are_sane because
    // a large constant tile can be filled in by smaller non-constant stores.
    assert(st_off >= last_init_off, "inits do not reverse");
    last_init_off = st_off;
    const Type* val = NULL;
    if (st_size >= BytesPerInt &&
        (val = phase->type(st->in(MemNode::ValueIn)))->singleton() &&
        (int)val->basic_type() < (int)T_OBJECT) {
      assert(st_off >= last_tile_end, "tiles do not overlap");
      assert(st_off >= last_init_end, "tiles do not overwrite inits");
      last_tile_end = MAX2(last_tile_end, next_init_off);
    } else {
      intptr_t st_tile_end = align_size_up(next_init_off, BytesPerLong);
      assert(st_tile_end >= last_tile_end, "inits stay with tiles");
      assert(st_off      >= last_init_end, "inits do not overlap");
      last_init_end = next_init_off;  // it's a non-tile
    }
    #endif //ASSERT
  }

  remove_extra_zeroes();        // clear out all the zmems left over
  add_req(inits);

  if (!ZeroTLAB) {
    // If anything remains to be zeroed, zero it all now.
    zeroes_done = align_size_down(zeroes_done, BytesPerInt);
    // if it is the last unused 4 bytes of an instance, forget about it
    intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
    if (zeroes_done + BytesPerLong >= size_limit) {
      assert(allocation() != NULL, "");
4014 4015 4016 4017 4018 4019
      if (allocation()->Opcode() == Op_Allocate) {
        Node* klass_node = allocation()->in(AllocateNode::KlassNode);
        ciKlass* k = phase->type(klass_node)->is_klassptr()->klass();
        if (zeroes_done == k->layout_helper())
          zeroes_done = size_limit;
      }
D
duke 已提交
4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035
    }
    if (zeroes_done < size_limit) {
      rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
                                            zeroes_done, size_in_bytes, phase);
    }
  }

  set_complete(phase);
  return rawmem;
}


#ifdef ASSERT
bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
  if (is_complete())
    return true;                // stores could be anything at this point
4036 4037
  assert(allocation() != NULL, "must be present");
  intptr_t last_off = allocation()->minimum_header_size();
D
duke 已提交
4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188
  for (uint i = InitializeNode::RawStores; i < req(); i++) {
    Node* st = in(i);
    intptr_t st_off = get_store_offset(st, phase);
    if (st_off < 0)  continue;  // ignore dead garbage
    if (last_off > st_off) {
      tty->print_cr("*** bad store offset at %d: %d > %d", i, last_off, st_off);
      this->dump(2);
      assert(false, "ascending store offsets");
      return false;
    }
    last_off = st_off + st->as_Store()->memory_size();
  }
  return true;
}
#endif //ASSERT




//============================MergeMemNode=====================================
//
// SEMANTICS OF MEMORY MERGES:  A MergeMem is a memory state assembled from several
// contributing store or call operations.  Each contributor provides the memory
// state for a particular "alias type" (see Compile::alias_type).  For example,
// if a MergeMem has an input X for alias category #6, then any memory reference
// to alias category #6 may use X as its memory state input, as an exact equivalent
// to using the MergeMem as a whole.
//   Load<6>( MergeMem(<6>: X, ...), p ) <==> Load<6>(X,p)
//
// (Here, the <N> notation gives the index of the relevant adr_type.)
//
// In one special case (and more cases in the future), alias categories overlap.
// The special alias category "Bot" (Compile::AliasIdxBot) includes all memory
// states.  Therefore, if a MergeMem has only one contributing input W for Bot,
// it is exactly equivalent to that state W:
//   MergeMem(<Bot>: W) <==> W
//
// Usually, the merge has more than one input.  In that case, where inputs
// overlap (i.e., one is Bot), the narrower alias type determines the memory
// state for that type, and the wider alias type (Bot) fills in everywhere else:
//   Load<5>( MergeMem(<Bot>: W, <6>: X), p ) <==> Load<5>(W,p)
//   Load<6>( MergeMem(<Bot>: W, <6>: X), p ) <==> Load<6>(X,p)
//
// A merge can take a "wide" memory state as one of its narrow inputs.
// This simply means that the merge observes out only the relevant parts of
// the wide input.  That is, wide memory states arriving at narrow merge inputs
// are implicitly "filtered" or "sliced" as necessary.  (This is rare.)
//
// These rules imply that MergeMem nodes may cascade (via their <Bot> links),
// and that memory slices "leak through":
//   MergeMem(<Bot>: MergeMem(<Bot>: W, <7>: Y)) <==> MergeMem(<Bot>: W, <7>: Y)
//
// But, in such a cascade, repeated memory slices can "block the leak":
//   MergeMem(<Bot>: MergeMem(<Bot>: W, <7>: Y), <7>: Y') <==> MergeMem(<Bot>: W, <7>: Y')
//
// In the last example, Y is not part of the combined memory state of the
// outermost MergeMem.  The system must, of course, prevent unschedulable
// memory states from arising, so you can be sure that the state Y is somehow
// a precursor to state Y'.
//
//
// REPRESENTATION OF MEMORY MERGES: The indexes used to address the Node::in array
// of each MergeMemNode array are exactly the numerical alias indexes, including
// but not limited to AliasIdxTop, AliasIdxBot, and AliasIdxRaw.  The functions
// Compile::alias_type (and kin) produce and manage these indexes.
//
// By convention, the value of in(AliasIdxTop) (i.e., in(1)) is always the top node.
// (Note that this provides quick access to the top node inside MergeMem methods,
// without the need to reach out via TLS to Compile::current.)
//
// As a consequence of what was just described, a MergeMem that represents a full
// memory state has an edge in(AliasIdxBot) which is a "wide" memory state,
// containing all alias categories.
//
// MergeMem nodes never (?) have control inputs, so in(0) is NULL.
//
// All other edges in(N) (including in(AliasIdxRaw), which is in(3)) are either
// a memory state for the alias type <N>, or else the top node, meaning that
// there is no particular input for that alias type.  Note that the length of
// a MergeMem is variable, and may be extended at any time to accommodate new
// memory states at larger alias indexes.  When merges grow, they are of course
// filled with "top" in the unused in() positions.
//
// This use of top is named "empty_memory()", or "empty_mem" (no-memory) as a variable.
// (Top was chosen because it works smoothly with passes like GCM.)
//
// For convenience, we hardwire the alias index for TypeRawPtr::BOTTOM.  (It is
// the type of random VM bits like TLS references.)  Since it is always the
// first non-Bot memory slice, some low-level loops use it to initialize an
// index variable:  for (i = AliasIdxRaw; i < req(); i++).
//
//
// ACCESSORS:  There is a special accessor MergeMemNode::base_memory which returns
// the distinguished "wide" state.  The accessor MergeMemNode::memory_at(N) returns
// the memory state for alias type <N>, or (if there is no particular slice at <N>,
// it returns the base memory.  To prevent bugs, memory_at does not accept <Top>
// or <Bot> indexes.  The iterator MergeMemStream provides robust iteration over
// MergeMem nodes or pairs of such nodes, ensuring that the non-top edges are visited.
//
// %%%% We may get rid of base_memory as a separate accessor at some point; it isn't
// really that different from the other memory inputs.  An abbreviation called
// "bot_memory()" for "memory_at(AliasIdxBot)" would keep code tidy.
//
//
// PARTIAL MEMORY STATES:  During optimization, MergeMem nodes may arise that represent
// partial memory states.  When a Phi splits through a MergeMem, the copy of the Phi
// that "emerges though" the base memory will be marked as excluding the alias types
// of the other (narrow-memory) copies which "emerged through" the narrow edges:
//
//   Phi<Bot>(U, MergeMem(<Bot>: W, <8>: Y))
//     ==Ideal=>  MergeMem(<Bot>: Phi<Bot-8>(U, W), Phi<8>(U, Y))
//
// This strange "subtraction" effect is necessary to ensure IGVN convergence.
// (It is currently unimplemented.)  As you can see, the resulting merge is
// actually a disjoint union of memory states, rather than an overlay.
//

//------------------------------MergeMemNode-----------------------------------
Node* MergeMemNode::make_empty_memory() {
  Node* empty_memory = (Node*) Compile::current()->top();
  assert(empty_memory->is_top(), "correct sentinel identity");
  return empty_memory;
}

MergeMemNode::MergeMemNode(Node *new_base) : Node(1+Compile::AliasIdxRaw) {
  init_class_id(Class_MergeMem);
  // all inputs are nullified in Node::Node(int)
  // set_input(0, NULL);  // no control input

  // Initialize the edges uniformly to top, for starters.
  Node* empty_mem = make_empty_memory();
  for (uint i = Compile::AliasIdxTop; i < req(); i++) {
    init_req(i,empty_mem);
  }
  assert(empty_memory() == empty_mem, "");

  if( new_base != NULL && new_base->is_MergeMem() ) {
    MergeMemNode* mdef = new_base->as_MergeMem();
    assert(mdef->empty_memory() == empty_mem, "consistent sentinels");
    for (MergeMemStream mms(this, mdef); mms.next_non_empty2(); ) {
      mms.set_memory(mms.memory2());
    }
    assert(base_memory() == mdef->base_memory(), "");
  } else {
    set_base_memory(new_base);
  }
}

// Make a new, untransformed MergeMem with the same base as 'mem'.
// If mem is itself a MergeMem, populate the result with the same edges.
MergeMemNode* MergeMemNode::make(Compile* C, Node* mem) {
4189
  return new(C) MergeMemNode(mem);
D
duke 已提交
4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308
}

//------------------------------cmp--------------------------------------------
uint MergeMemNode::hash() const { return NO_HASH; }
uint MergeMemNode::cmp( const Node &n ) const {
  return (&n == this);          // Always fail except on self
}

//------------------------------Identity---------------------------------------
Node* MergeMemNode::Identity(PhaseTransform *phase) {
  // Identity if this merge point does not record any interesting memory
  // disambiguations.
  Node* base_mem = base_memory();
  Node* empty_mem = empty_memory();
  if (base_mem != empty_mem) {  // Memory path is not dead?
    for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
      Node* mem = in(i);
      if (mem != empty_mem && mem != base_mem) {
        return this;            // Many memory splits; no change
      }
    }
  }
  return base_mem;              // No memory splits; ID on the one true input
}

//------------------------------Ideal------------------------------------------
// This method is invoked recursively on chains of MergeMem nodes
Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) {
  // Remove chain'd MergeMems
  //
  // This is delicate, because the each "in(i)" (i >= Raw) is interpreted
  // relative to the "in(Bot)".  Since we are patching both at the same time,
  // we have to be careful to read each "in(i)" relative to the old "in(Bot)",
  // but rewrite each "in(i)" relative to the new "in(Bot)".
  Node *progress = NULL;


  Node* old_base = base_memory();
  Node* empty_mem = empty_memory();
  if (old_base == empty_mem)
    return NULL; // Dead memory path.

  MergeMemNode* old_mbase;
  if (old_base != NULL && old_base->is_MergeMem())
    old_mbase = old_base->as_MergeMem();
  else
    old_mbase = NULL;
  Node* new_base = old_base;

  // simplify stacked MergeMems in base memory
  if (old_mbase)  new_base = old_mbase->base_memory();

  // the base memory might contribute new slices beyond my req()
  if (old_mbase)  grow_to_match(old_mbase);

  // Look carefully at the base node if it is a phi.
  PhiNode* phi_base;
  if (new_base != NULL && new_base->is_Phi())
    phi_base = new_base->as_Phi();
  else
    phi_base = NULL;

  Node*    phi_reg = NULL;
  uint     phi_len = (uint)-1;
  if (phi_base != NULL && !phi_base->is_copy()) {
    // do not examine phi if degraded to a copy
    phi_reg = phi_base->region();
    phi_len = phi_base->req();
    // see if the phi is unfinished
    for (uint i = 1; i < phi_len; i++) {
      if (phi_base->in(i) == NULL) {
        // incomplete phi; do not look at it yet!
        phi_reg = NULL;
        phi_len = (uint)-1;
        break;
      }
    }
  }

  // Note:  We do not call verify_sparse on entry, because inputs
  // can normalize to the base_memory via subsume_node or similar
  // mechanisms.  This method repairs that damage.

  assert(!old_mbase || old_mbase->is_empty_memory(empty_mem), "consistent sentinels");

  // Look at each slice.
  for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
    Node* old_in = in(i);
    // calculate the old memory value
    Node* old_mem = old_in;
    if (old_mem == empty_mem)  old_mem = old_base;
    assert(old_mem == memory_at(i), "");

    // maybe update (reslice) the old memory value

    // simplify stacked MergeMems
    Node* new_mem = old_mem;
    MergeMemNode* old_mmem;
    if (old_mem != NULL && old_mem->is_MergeMem())
      old_mmem = old_mem->as_MergeMem();
    else
      old_mmem = NULL;
    if (old_mmem == this) {
      // This can happen if loops break up and safepoints disappear.
      // A merge of BotPtr (default) with a RawPtr memory derived from a
      // safepoint can be rewritten to a merge of the same BotPtr with
      // the BotPtr phi coming into the loop.  If that phi disappears
      // also, we can end up with a self-loop of the mergemem.
      // In general, if loops degenerate and memory effects disappear,
      // a mergemem can be left looking at itself.  This simply means
      // that the mergemem's default should be used, since there is
      // no longer any apparent effect on this slice.
      // Note: If a memory slice is a MergeMem cycle, it is unreachable
      //       from start.  Update the input to TOP.
      new_mem = (new_base == this || new_base == empty_mem)? empty_mem : new_base;
    }
    else if (old_mmem != NULL) {
      new_mem = old_mmem->memory_at(i);
    }
T
twisti 已提交
4309
    // else preceding memory was not a MergeMem
D
duke 已提交
4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391

    // replace equivalent phis (unfortunately, they do not GVN together)
    if (new_mem != NULL && new_mem != new_base &&
        new_mem->req() == phi_len && new_mem->in(0) == phi_reg) {
      if (new_mem->is_Phi()) {
        PhiNode* phi_mem = new_mem->as_Phi();
        for (uint i = 1; i < phi_len; i++) {
          if (phi_base->in(i) != phi_mem->in(i)) {
            phi_mem = NULL;
            break;
          }
        }
        if (phi_mem != NULL) {
          // equivalent phi nodes; revert to the def
          new_mem = new_base;
        }
      }
    }

    // maybe store down a new value
    Node* new_in = new_mem;
    if (new_in == new_base)  new_in = empty_mem;

    if (new_in != old_in) {
      // Warning:  Do not combine this "if" with the previous "if"
      // A memory slice might have be be rewritten even if it is semantically
      // unchanged, if the base_memory value has changed.
      set_req(i, new_in);
      progress = this;          // Report progress
    }
  }

  if (new_base != old_base) {
    set_req(Compile::AliasIdxBot, new_base);
    // Don't use set_base_memory(new_base), because we need to update du.
    assert(base_memory() == new_base, "");
    progress = this;
  }

  if( base_memory() == this ) {
    // a self cycle indicates this memory path is dead
    set_req(Compile::AliasIdxBot, empty_mem);
  }

  // Resolve external cycles by calling Ideal on a MergeMem base_memory
  // Recursion must occur after the self cycle check above
  if( base_memory()->is_MergeMem() ) {
    MergeMemNode *new_mbase = base_memory()->as_MergeMem();
    Node *m = phase->transform(new_mbase);  // Rollup any cycles
    if( m != NULL && (m->is_top() ||
        m->is_MergeMem() && m->as_MergeMem()->base_memory() == empty_mem) ) {
      // propagate rollup of dead cycle to self
      set_req(Compile::AliasIdxBot, empty_mem);
    }
  }

  if( base_memory() == empty_mem ) {
    progress = this;
    // Cut inputs during Parse phase only.
    // During Optimize phase a dead MergeMem node will be subsumed by Top.
    if( !can_reshape ) {
      for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
        if( in(i) != empty_mem ) { set_req(i, empty_mem); }
      }
    }
  }

  if( !progress && base_memory()->is_Phi() && can_reshape ) {
    // Check if PhiNode::Ideal's "Split phis through memory merges"
    // transform should be attempted. Look for this->phi->this cycle.
    uint merge_width = req();
    if (merge_width > Compile::AliasIdxRaw) {
      PhiNode* phi = base_memory()->as_Phi();
      for( uint i = 1; i < phi->req(); ++i ) {// For all paths in
        if (phi->in(i) == this) {
          phase->is_IterGVN()->_worklist.push(phi);
          break;
        }
      }
    }
  }

4392
  assert(progress || verify_sparse(), "please, no dups of base");
D
duke 已提交
4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473
  return progress;
}

//-------------------------set_base_memory-------------------------------------
void MergeMemNode::set_base_memory(Node *new_base) {
  Node* empty_mem = empty_memory();
  set_req(Compile::AliasIdxBot, new_base);
  assert(memory_at(req()) == new_base, "must set default memory");
  // Clear out other occurrences of new_base:
  if (new_base != empty_mem) {
    for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
      if (in(i) == new_base)  set_req(i, empty_mem);
    }
  }
}

//------------------------------out_RegMask------------------------------------
const RegMask &MergeMemNode::out_RegMask() const {
  return RegMask::Empty;
}

//------------------------------dump_spec--------------------------------------
#ifndef PRODUCT
void MergeMemNode::dump_spec(outputStream *st) const {
  st->print(" {");
  Node* base_mem = base_memory();
  for( uint i = Compile::AliasIdxRaw; i < req(); i++ ) {
    Node* mem = memory_at(i);
    if (mem == base_mem) { st->print(" -"); continue; }
    st->print( " N%d:", mem->_idx );
    Compile::current()->get_adr_type(i)->dump_on(st);
  }
  st->print(" }");
}
#endif // !PRODUCT


#ifdef ASSERT
static bool might_be_same(Node* a, Node* b) {
  if (a == b)  return true;
  if (!(a->is_Phi() || b->is_Phi()))  return false;
  // phis shift around during optimization
  return true;  // pretty stupid...
}

// verify a narrow slice (either incoming or outgoing)
static void verify_memory_slice(const MergeMemNode* m, int alias_idx, Node* n) {
  if (!VerifyAliases)       return;  // don't bother to verify unless requested
  if (is_error_reported())  return;  // muzzle asserts when debugging an error
  if (Node::in_dump())      return;  // muzzle asserts when printing
  assert(alias_idx >= Compile::AliasIdxRaw, "must not disturb base_memory or sentinel");
  assert(n != NULL, "");
  // Elide intervening MergeMem's
  while (n->is_MergeMem()) {
    n = n->as_MergeMem()->memory_at(alias_idx);
  }
  Compile* C = Compile::current();
  const TypePtr* n_adr_type = n->adr_type();
  if (n == m->empty_memory()) {
    // Implicit copy of base_memory()
  } else if (n_adr_type != TypePtr::BOTTOM) {
    assert(n_adr_type != NULL, "new memory must have a well-defined adr_type");
    assert(C->must_alias(n_adr_type, alias_idx), "new memory must match selected slice");
  } else {
    // A few places like make_runtime_call "know" that VM calls are narrow,
    // and can be used to update only the VM bits stored as TypeRawPtr::BOTTOM.
    bool expected_wide_mem = false;
    if (n == m->base_memory()) {
      expected_wide_mem = true;
    } else if (alias_idx == Compile::AliasIdxRaw ||
               n == m->memory_at(Compile::AliasIdxRaw)) {
      expected_wide_mem = true;
    } else if (!C->alias_type(alias_idx)->is_rewritable()) {
      // memory can "leak through" calls on channels that
      // are write-once.  Allow this also.
      expected_wide_mem = true;
    }
    assert(expected_wide_mem, "expected narrow slice replacement");
  }
}
#else // !ASSERT
4474
#define verify_memory_slice(m,i,n) (void)(0)  // PRODUCT version is no-op
D
duke 已提交
4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491
#endif


//-----------------------------memory_at---------------------------------------
Node* MergeMemNode::memory_at(uint alias_idx) const {
  assert(alias_idx >= Compile::AliasIdxRaw ||
         alias_idx == Compile::AliasIdxBot && Compile::current()->AliasLevel() == 0,
         "must avoid base_memory and AliasIdxTop");

  // Otherwise, it is a narrow slice.
  Node* n = alias_idx < req() ? in(alias_idx) : empty_memory();
  Compile *C = Compile::current();
  if (is_empty_memory(n)) {
    // the array is sparse; empty slots are the "top" node
    n = base_memory();
    assert(Node::in_dump()
           || n == NULL || n->bottom_type() == Type::TOP
4492
           || n->adr_type() == NULL // address is TOP
D
duke 已提交
4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593
           || n->adr_type() == TypePtr::BOTTOM
           || n->adr_type() == TypeRawPtr::BOTTOM
           || Compile::current()->AliasLevel() == 0,
           "must be a wide memory");
    // AliasLevel == 0 if we are organizing the memory states manually.
    // See verify_memory_slice for comments on TypeRawPtr::BOTTOM.
  } else {
    // make sure the stored slice is sane
    #ifdef ASSERT
    if (is_error_reported() || Node::in_dump()) {
    } else if (might_be_same(n, base_memory())) {
      // Give it a pass:  It is a mostly harmless repetition of the base.
      // This can arise normally from node subsumption during optimization.
    } else {
      verify_memory_slice(this, alias_idx, n);
    }
    #endif
  }
  return n;
}

//---------------------------set_memory_at-------------------------------------
void MergeMemNode::set_memory_at(uint alias_idx, Node *n) {
  verify_memory_slice(this, alias_idx, n);
  Node* empty_mem = empty_memory();
  if (n == base_memory())  n = empty_mem;  // collapse default
  uint need_req = alias_idx+1;
  if (req() < need_req) {
    if (n == empty_mem)  return;  // already the default, so do not grow me
    // grow the sparse array
    do {
      add_req(empty_mem);
    } while (req() < need_req);
  }
  set_req( alias_idx, n );
}



//--------------------------iteration_setup------------------------------------
void MergeMemNode::iteration_setup(const MergeMemNode* other) {
  if (other != NULL) {
    grow_to_match(other);
    // invariant:  the finite support of mm2 is within mm->req()
    #ifdef ASSERT
    for (uint i = req(); i < other->req(); i++) {
      assert(other->is_empty_memory(other->in(i)), "slice left uncovered");
    }
    #endif
  }
  // Replace spurious copies of base_memory by top.
  Node* base_mem = base_memory();
  if (base_mem != NULL && !base_mem->is_top()) {
    for (uint i = Compile::AliasIdxBot+1, imax = req(); i < imax; i++) {
      if (in(i) == base_mem)
        set_req(i, empty_memory());
    }
  }
}

//---------------------------grow_to_match-------------------------------------
void MergeMemNode::grow_to_match(const MergeMemNode* other) {
  Node* empty_mem = empty_memory();
  assert(other->is_empty_memory(empty_mem), "consistent sentinels");
  // look for the finite support of the other memory
  for (uint i = other->req(); --i >= req(); ) {
    if (other->in(i) != empty_mem) {
      uint new_len = i+1;
      while (req() < new_len)  add_req(empty_mem);
      break;
    }
  }
}

//---------------------------verify_sparse-------------------------------------
#ifndef PRODUCT
bool MergeMemNode::verify_sparse() const {
  assert(is_empty_memory(make_empty_memory()), "sane sentinel");
  Node* base_mem = base_memory();
  // The following can happen in degenerate cases, since empty==top.
  if (is_empty_memory(base_mem))  return true;
  for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
    assert(in(i) != NULL, "sane slice");
    if (in(i) == base_mem)  return false;  // should have been the sentinel value!
  }
  return true;
}

bool MergeMemStream::match_memory(Node* mem, const MergeMemNode* mm, int idx) {
  Node* n;
  n = mm->in(idx);
  if (mem == n)  return true;  // might be empty_memory()
  n = (idx == Compile::AliasIdxBot)? mm->base_memory(): mm->memory_at(idx);
  if (mem == n)  return true;
  while (n->is_Phi() && (n = n->as_Phi()->is_copy()) != NULL) {
    if (mem == n)  return true;
    if (n == NULL)  break;
  }
  return false;
}
#endif // !PRODUCT