metaspace.cpp 136.3 KB
Newer Older
1
/*
2
 * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved.
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 *
 */
#include "precompiled.hpp"
#include "gc_interface/collectedHeap.hpp"
26
#include "memory/allocation.hpp"
27
#include "memory/binaryTreeDictionary.hpp"
28
#include "memory/freeList.hpp"
29 30 31
#include "memory/collectorPolicy.hpp"
#include "memory/filemap.hpp"
#include "memory/freeList.hpp"
32
#include "memory/gcLocker.hpp"
33
#include "memory/metachunk.hpp"
34
#include "memory/metaspace.hpp"
35
#include "memory/metaspaceGCThresholdUpdater.hpp"
36
#include "memory/metaspaceShared.hpp"
37
#include "memory/metaspaceTracer.hpp"
38 39
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
40
#include "runtime/atomic.inline.hpp"
41
#include "runtime/globals.hpp"
42
#include "runtime/init.hpp"
43
#include "runtime/java.hpp"
44
#include "runtime/mutex.hpp"
45
#include "runtime/orderAccess.inline.hpp"
46
#include "services/memTracker.hpp"
47
#include "services/memoryService.hpp"
48 49 50
#include "utilities/copy.hpp"
#include "utilities/debug.hpp"

51 52
PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC

53 54
typedef BinaryTreeDictionary<Metablock, FreeList<Metablock> > BlockTreeDictionary;
typedef BinaryTreeDictionary<Metachunk, FreeList<Metachunk> > ChunkTreeDictionary;
55 56

// Set this constant to enable slow integrity checking of the free chunk lists
57 58
const bool metaspace_slow_verify = false;

59
size_t const allocation_from_dictionary_limit = 4 * K;
60 61 62

MetaWord* last_allocated = 0;

63
size_t Metaspace::_compressed_class_space_size;
64
const MetaspaceTracer* Metaspace::_tracer = NULL;
65

66 67
// Used in declarations in SpaceManager and ChunkManager
enum ChunkIndex {
68 69 70 71 72 73 74 75 76 77 78 79 80 81
  ZeroIndex = 0,
  SpecializedIndex = ZeroIndex,
  SmallIndex = SpecializedIndex + 1,
  MediumIndex = SmallIndex + 1,
  HumongousIndex = MediumIndex + 1,
  NumberOfFreeLists = 3,
  NumberOfInUseLists = 4
};

enum ChunkSizes {    // in words.
  ClassSpecializedChunk = 128,
  SpecializedChunk = 128,
  ClassSmallChunk = 256,
  SmallChunk = 512,
82
  ClassMediumChunk = 4 * K,
83
  MediumChunk = 8 * K
84 85 86
};

static ChunkIndex next_chunk_index(ChunkIndex i) {
87
  assert(i < NumberOfInUseLists, "Out of bound");
88 89 90
  return (ChunkIndex) (i+1);
}

91
volatile intptr_t MetaspaceGC::_capacity_until_GC = 0;
92 93 94
uint MetaspaceGC::_shrink_factor = 0;
bool MetaspaceGC::_should_concurrent_collect = false;

95
typedef class FreeList<Metachunk> ChunkList;
96 97

// Manages the global free lists of chunks.
98
class ChunkManager : public CHeapObj<mtInternal> {
99
  friend class TestVirtualSpaceNodeTest;
100 101

  // Free list of chunks of different sizes.
102
  //   SpecializedChunk
103 104 105
  //   SmallChunk
  //   MediumChunk
  //   HumongousChunk
106 107 108 109
  ChunkList _free_chunks[NumberOfFreeLists];

  //   HumongousChunk
  ChunkTreeDictionary _humongous_dictionary;
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129

  // ChunkManager in all lists of this type
  size_t _free_chunks_total;
  size_t _free_chunks_count;

  void dec_free_chunks_total(size_t v) {
    assert(_free_chunks_count > 0 &&
             _free_chunks_total > 0,
             "About to go negative");
    Atomic::add_ptr(-1, &_free_chunks_count);
    jlong minus_v = (jlong) - (jlong) v;
    Atomic::add_ptr(minus_v, &_free_chunks_total);
  }

  // Debug support

  size_t sum_free_chunks();
  size_t sum_free_chunks_count();

  void locked_verify_free_chunks_total();
130 131 132 133 134
  void slow_locked_verify_free_chunks_total() {
    if (metaspace_slow_verify) {
      locked_verify_free_chunks_total();
    }
  }
135
  void locked_verify_free_chunks_count();
136 137 138 139 140
  void slow_locked_verify_free_chunks_count() {
    if (metaspace_slow_verify) {
      locked_verify_free_chunks_count();
    }
  }
141 142 143 144
  void verify_free_chunks_count();

 public:

145 146 147 148 149 150
  ChunkManager(size_t specialized_size, size_t small_size, size_t medium_size)
      : _free_chunks_total(0), _free_chunks_count(0) {
    _free_chunks[SpecializedIndex].set_size(specialized_size);
    _free_chunks[SmallIndex].set_size(small_size);
    _free_chunks[MediumIndex].set_size(medium_size);
  }
151 152 153 154

  // add or delete (return) a chunk to the global freelist.
  Metachunk* chunk_freelist_allocate(size_t word_size);

155 156 157 158
  // Map a size to a list index assuming that there are lists
  // for special, small, medium, and humongous chunks.
  static ChunkIndex list_index(size_t size);

159 160 161 162
  // Remove the chunk from its freelist.  It is
  // expected to be on one of the _free_chunks[] lists.
  void remove_chunk(Metachunk* chunk);

163 164 165 166
  // Add the simple linked list of chunks to the freelist of chunks
  // of type index.
  void return_chunks(ChunkIndex index, Metachunk* chunks);

167
  // Total of the space in the free chunks list
E
ehelin 已提交
168 169
  size_t free_chunks_total_words();
  size_t free_chunks_total_bytes();
170 171 172 173 174 175 176 177

  // Number of chunks in the free chunks list
  size_t free_chunks_count();

  void inc_free_chunks_total(size_t v, size_t count = 1) {
    Atomic::add_ptr(count, &_free_chunks_count);
    Atomic::add_ptr(v, &_free_chunks_total);
  }
178 179 180
  ChunkTreeDictionary* humongous_dictionary() {
    return &_humongous_dictionary;
  }
181 182 183 184 185 186

  ChunkList* free_chunks(ChunkIndex index);

  // Returns the list for the given chunk word size.
  ChunkList* find_free_chunks_list(size_t word_size);

187
  // Remove from a list by size.  Selects list based on size of chunk.
188 189
  Metachunk* free_chunks_get(size_t chunk_word_size);

190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
#define index_bounds_check(index)                                         \
  assert(index == SpecializedIndex ||                                     \
         index == SmallIndex ||                                           \
         index == MediumIndex ||                                          \
         index == HumongousIndex, err_msg("Bad index: %d", (int) index))

  size_t num_free_chunks(ChunkIndex index) const {
    index_bounds_check(index);

    if (index == HumongousIndex) {
      return _humongous_dictionary.total_free_blocks();
    }

    ssize_t count = _free_chunks[index].count();
    return count == -1 ? 0 : (size_t) count;
  }

  size_t size_free_chunks_in_bytes(ChunkIndex index) const {
    index_bounds_check(index);

    size_t word_size = 0;
    if (index == HumongousIndex) {
      word_size = _humongous_dictionary.total_size();
    } else {
      const size_t size_per_chunk_in_words = _free_chunks[index].size();
      word_size = size_per_chunk_in_words * num_free_chunks(index);
    }

    return word_size * BytesPerWord;
  }

  MetaspaceChunkFreeListSummary chunk_free_list_summary() const {
    return MetaspaceChunkFreeListSummary(num_free_chunks(SpecializedIndex),
                                         num_free_chunks(SmallIndex),
                                         num_free_chunks(MediumIndex),
                                         num_free_chunks(HumongousIndex),
                                         size_free_chunks_in_bytes(SpecializedIndex),
                                         size_free_chunks_in_bytes(SmallIndex),
                                         size_free_chunks_in_bytes(MediumIndex),
                                         size_free_chunks_in_bytes(HumongousIndex));
  }

232 233
  // Debug support
  void verify();
234 235 236 237 238
  void slow_verify() {
    if (metaspace_slow_verify) {
      verify();
    }
  }
239
  void locked_verify();
240 241 242 243 244
  void slow_locked_verify() {
    if (metaspace_slow_verify) {
      locked_verify();
    }
  }
245 246 247 248
  void verify_free_chunks_total();

  void locked_print_free_chunks(outputStream* st);
  void locked_print_sum_free_chunks(outputStream* st);
249

250
  void print_on(outputStream* st) const;
251 252 253 254 255
};

// Used to manage the free list of Metablocks (a block corresponds
// to the allocation of a quantum of metadata).
class BlockFreelist VALUE_OBJ_CLASS_SPEC {
256
  BlockTreeDictionary* _dictionary;
257

258 259 260 261
  // Only allocate and split from freelist if the size of the allocation
  // is at least 1/4th the size of the available block.
  const static int WasteMultiplier = 4;

262
  // Accessors
263
  BlockTreeDictionary* dictionary() const { return _dictionary; }
264 265 266 267 268 269

 public:
  BlockFreelist();
  ~BlockFreelist();

  // Get and return a block to the free list
270 271
  MetaWord* get_block(size_t word_size);
  void return_block(MetaWord* p, size_t word_size);
272

273 274
  size_t total_size() {
  if (dictionary() == NULL) {
275
    return 0;
276 277
  } else {
    return dictionary()->total_size();
278
  }
279
}
280 281 282 283

  void print_on(outputStream* st) const;
};

284
// A VirtualSpaceList node.
285 286 287 288 289 290 291 292 293 294 295
class VirtualSpaceNode : public CHeapObj<mtClass> {
  friend class VirtualSpaceList;

  // Link to next VirtualSpaceNode
  VirtualSpaceNode* _next;

  // total in the VirtualSpace
  MemRegion _reserved;
  ReservedSpace _rs;
  VirtualSpace _virtual_space;
  MetaWord* _top;
296 297
  // count of chunks contained in this VirtualSpace
  uintx _container_count;
298 299 300 301 302

  // Convenience functions to access the _virtual_space
  char* low()  const { return virtual_space()->low(); }
  char* high() const { return virtual_space()->high(); }

303 304 305 306
  // The first Metachunk will be allocated at the bottom of the
  // VirtualSpace
  Metachunk* first_chunk() { return (Metachunk*) bottom(); }

307 308
  // Committed but unused space in the virtual space
  size_t free_words_in_vs() const;
309 310 311
 public:

  VirtualSpaceNode(size_t byte_size);
312
  VirtualSpaceNode(ReservedSpace rs) : _top(NULL), _next(NULL), _rs(rs), _container_count(0) {}
313 314
  ~VirtualSpaceNode();

315 316 317 318
  // Convenience functions for logical bottom and end
  MetaWord* bottom() const { return (MetaWord*) _virtual_space.low(); }
  MetaWord* end() const { return (MetaWord*) _virtual_space.high(); }

319 320
  bool contains(const void* ptr) { return ptr >= low() && ptr < high(); }

321 322 323
  size_t reserved_words() const  { return _virtual_space.reserved_size() / BytesPerWord; }
  size_t committed_words() const { return _virtual_space.actual_committed_size() / BytesPerWord; }

324 325
  bool is_pre_committed() const { return _virtual_space.special(); }

326 327 328 329 330 331 332 333 334 335 336 337
  // address of next available space in _virtual_space;
  // Accessors
  VirtualSpaceNode* next() { return _next; }
  void set_next(VirtualSpaceNode* v) { _next = v; }

  void set_reserved(MemRegion const v) { _reserved = v; }
  void set_top(MetaWord* v) { _top = v; }

  // Accessors
  MemRegion* reserved() { return &_reserved; }
  VirtualSpace* virtual_space() const { return (VirtualSpace*) &_virtual_space; }

338
  // Returns true if "word_size" is available in the VirtualSpace
339
  bool is_available(size_t word_size) { return word_size <= pointer_delta(end(), _top, sizeof(MetaWord)); }
340 341 342 343

  MetaWord* top() const { return _top; }
  void inc_top(size_t word_size) { _top += word_size; }

344
  uintx container_count() { return _container_count; }
345
  void inc_container_count();
346 347
  void dec_container_count();
#ifdef ASSERT
348
  uint container_count_slow();
349 350 351
  void verify_container_count();
#endif

352 353 354 355 356 357 358 359 360 361 362 363 364 365
  // used and capacity in this single entry in the list
  size_t used_words_in_vs() const;
  size_t capacity_words_in_vs() const;

  bool initialize();

  // get space from the virtual space
  Metachunk* take_from_committed(size_t chunk_word_size);

  // Allocate a chunk from the virtual space and return it.
  Metachunk* get_chunk_vs(size_t chunk_word_size);

  // Expands/shrinks the committed space in a virtual space.  Delegates
  // to Virtualspace
366
  bool expand_by(size_t min_words, size_t preferred_words);
367

368 369 370 371
  // In preparation for deleting this node, remove all the chunks
  // in the node from any freelist.
  void purge(ChunkManager* chunk_manager);

372 373 374 375 376 377 378
  // If an allocation doesn't fit in the current node a new node is created.
  // Allocate chunks out of the remaining committed space in this node
  // to avoid wasting that memory.
  // This always adds up because all the chunk sizes are multiples of
  // the smallest chunk size.
  void retire(ChunkManager* chunk_manager);

379
#ifdef ASSERT
380 381
  // Debug support
  void mangle();
382
#endif
383 384 385 386

  void print_on(outputStream* st) const;
};

387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411
#define assert_is_ptr_aligned(ptr, alignment) \
  assert(is_ptr_aligned(ptr, alignment),      \
    err_msg(PTR_FORMAT " is not aligned to "  \
      SIZE_FORMAT, ptr, alignment))

#define assert_is_size_aligned(size, alignment) \
  assert(is_size_aligned(size, alignment),      \
    err_msg(SIZE_FORMAT " is not aligned to "   \
       SIZE_FORMAT, size, alignment))


// Decide if large pages should be committed when the memory is reserved.
static bool should_commit_large_pages_when_reserving(size_t bytes) {
  if (UseLargePages && UseLargePagesInMetaspace && !os::can_commit_large_page_memory()) {
    size_t words = bytes / BytesPerWord;
    bool is_class = false; // We never reserve large pages for the class space.
    if (MetaspaceGC::can_expand(words, is_class) &&
        MetaspaceGC::allowed_expansion() >= words) {
      return true;
    }
  }

  return false;
}

412
  // byte_size is the size of the associated virtualspace.
413 414
VirtualSpaceNode::VirtualSpaceNode(size_t bytes) : _top(NULL), _next(NULL), _rs(), _container_count(0) {
  assert_is_size_aligned(bytes, Metaspace::reserve_alignment());
415

416
#if INCLUDE_CDS
417 418 419
  // This allocates memory with mmap.  For DumpSharedspaces, try to reserve
  // configurable address, generally at the top of the Java heap so other
  // memory addresses don't conflict.
420
  if (DumpSharedSpaces) {
421 422 423 424
    bool large_pages = false; // No large pages when dumping the CDS archive.
    char* shared_base = (char*)align_ptr_up((char*)SharedBaseAddress, Metaspace::reserve_alignment());

    _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages, shared_base, 0);
425
    if (_rs.is_reserved()) {
426
      assert(shared_base == 0 || _rs.base() == shared_base, "should match");
427
    } else {
428
      // Get a mmap region anywhere if the SharedBaseAddress fails.
429
      _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages);
430 431
    }
    MetaspaceShared::set_shared_rs(&_rs);
432 433 434
  } else
#endif
  {
435 436 437
    bool large_pages = should_commit_large_pages_when_reserving(bytes);

    _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages);
438 439
  }

440 441 442 443 444 445 446 447
  if (_rs.is_reserved()) {
    assert(_rs.base() != NULL, "Catch if we get a NULL address");
    assert(_rs.size() != 0, "Catch if we get a 0 size");
    assert_is_ptr_aligned(_rs.base(), Metaspace::reserve_alignment());
    assert_is_size_aligned(_rs.size(), Metaspace::reserve_alignment());

    MemTracker::record_virtual_memory_type((address)_rs.base(), mtClass);
  }
448 449
}

450 451 452 453
void VirtualSpaceNode::purge(ChunkManager* chunk_manager) {
  Metachunk* chunk = first_chunk();
  Metachunk* invalid_chunk = (Metachunk*) top();
  while (chunk < invalid_chunk ) {
454 455 456 457 458 459 460
    assert(chunk->is_tagged_free(), "Should be tagged free");
    MetaWord* next = ((MetaWord*)chunk) + chunk->word_size();
    chunk_manager->remove_chunk(chunk);
    assert(chunk->next() == NULL &&
           chunk->prev() == NULL,
           "Was not removed from its list");
    chunk = (Metachunk*) next;
461 462 463 464 465 466 467 468 469 470 471 472 473
  }
}

#ifdef ASSERT
uint VirtualSpaceNode::container_count_slow() {
  uint count = 0;
  Metachunk* chunk = first_chunk();
  Metachunk* invalid_chunk = (Metachunk*) top();
  while (chunk < invalid_chunk ) {
    MetaWord* next = ((MetaWord*)chunk) + chunk->word_size();
    // Don't count the chunks on the free lists.  Those are
    // still part of the VirtualSpaceNode but not currently
    // counted.
474
    if (!chunk->is_tagged_free()) {
475 476 477 478 479 480 481 482
      count++;
    }
    chunk = (Metachunk*) next;
  }
  return count;
}
#endif

483 484 485 486 487 488 489 490 491 492 493 494 495
// List of VirtualSpaces for metadata allocation.
class VirtualSpaceList : public CHeapObj<mtClass> {
  friend class VirtualSpaceNode;

  enum VirtualSpaceSizes {
    VirtualSpaceSize = 256 * K
  };

  // Head of the list
  VirtualSpaceNode* _virtual_space_list;
  // virtual space currently being used for allocations
  VirtualSpaceNode* _current_virtual_space;

496
  // Is this VirtualSpaceList used for the compressed class space
497 498
  bool _is_class;

499 500 501 502 503
  // Sum of reserved and committed memory in the virtual spaces
  size_t _reserved_words;
  size_t _committed_words;

  // Number of virtual spaces
504 505 506 507 508 509 510 511 512 513 514 515 516
  size_t _virtual_space_count;

  ~VirtualSpaceList();

  VirtualSpaceNode* virtual_space_list() const { return _virtual_space_list; }

  void set_virtual_space_list(VirtualSpaceNode* v) {
    _virtual_space_list = v;
  }
  void set_current_virtual_space(VirtualSpaceNode* v) {
    _current_virtual_space = v;
  }

517
  void link_vs(VirtualSpaceNode* new_entry);
518 519 520 521

  // Get another virtual space and add it to the list.  This
  // is typically prompted by a failed attempt to allocate a chunk
  // and is typically followed by the allocation of a chunk.
522
  bool create_new_virtual_space(size_t vs_word_size);
523

524 525 526 527
  // Chunk up the unused committed space in the current
  // virtual space and add the chunks to the free list.
  void retire_current_virtual_space();

528 529 530 531
 public:
  VirtualSpaceList(size_t word_size);
  VirtualSpaceList(ReservedSpace rs);

532 533
  size_t free_bytes();

534 535 536 537
  Metachunk* get_new_chunk(size_t word_size,
                           size_t grow_chunks_by_words,
                           size_t medium_chunk_bunch);

538 539 540
  bool expand_node_by(VirtualSpaceNode* node,
                      size_t min_words,
                      size_t preferred_words);
541

542 543
  bool expand_by(size_t min_words,
                 size_t preferred_words);
544 545 546 547 548 549 550

  VirtualSpaceNode* current_virtual_space() {
    return _current_virtual_space;
  }

  bool is_class() const { return _is_class; }

551
  bool initialization_succeeded() { return _virtual_space_list != NULL; }
552

553 554 555 556
  size_t reserved_words()  { return _reserved_words; }
  size_t reserved_bytes()  { return reserved_words() * BytesPerWord; }
  size_t committed_words() { return _committed_words; }
  size_t committed_bytes() { return committed_words() * BytesPerWord; }
557

558 559 560 561
  void inc_reserved_words(size_t v);
  void dec_reserved_words(size_t v);
  void inc_committed_words(size_t v);
  void dec_committed_words(size_t v);
562 563 564
  void inc_virtual_space_count();
  void dec_virtual_space_count();

565 566
  bool contains(const void* ptr);

567
  // Unlink empty VirtualSpaceNodes and free it.
568
  void purge(ChunkManager* chunk_manager);
569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611

  void print_on(outputStream* st) const;

  class VirtualSpaceListIterator : public StackObj {
    VirtualSpaceNode* _virtual_spaces;
   public:
    VirtualSpaceListIterator(VirtualSpaceNode* virtual_spaces) :
      _virtual_spaces(virtual_spaces) {}

    bool repeat() {
      return _virtual_spaces != NULL;
    }

    VirtualSpaceNode* get_next() {
      VirtualSpaceNode* result = _virtual_spaces;
      if (_virtual_spaces != NULL) {
        _virtual_spaces = _virtual_spaces->next();
      }
      return result;
    }
  };
};

class Metadebug : AllStatic {
  // Debugging support for Metaspaces
  static int _allocation_fail_alot_count;

 public:

  static void init_allocation_fail_alot_count();
#ifdef ASSERT
  static bool test_metadata_failure();
#endif
};

int Metadebug::_allocation_fail_alot_count = 0;

//  SpaceManager - used by Metaspace to handle allocations
class SpaceManager : public CHeapObj<mtClass> {
  friend class Metaspace;
  friend class Metadebug;

 private:
612

613
  // protects allocations
614 615
  Mutex* const _lock;

616 617 618
  // Type of metadata allocated.
  Metaspace::MetadataType _mdtype;

619 620 621
  // List of chunks in use by this SpaceManager.  Allocations
  // are done from the current chunk.  The list is used for deallocating
  // chunks when the SpaceManager is freed.
622
  Metachunk* _chunks_in_use[NumberOfInUseLists];
623 624 625 626 627 628 629
  Metachunk* _current_chunk;

  // Number of small chunks to allocate to a manager
  // If class space manager, small chunks are unlimited
  static uint const _small_chunk_limit;

  // Sum of all space in allocated chunks
630 631 632 633 634
  size_t _allocated_blocks_words;

  // Sum of all allocated chunks
  size_t _allocated_chunks_words;
  size_t _allocated_chunks_count;
635 636 637 638 639 640 641 642 643 644 645 646

  // Free lists of blocks are per SpaceManager since they
  // are assumed to be in chunks in use by the SpaceManager
  // and all chunks in use by a SpaceManager are freed when
  // the class loader using the SpaceManager is collected.
  BlockFreelist _block_freelists;

  // protects virtualspace and chunk expansions
  static const char*  _expand_lock_name;
  static const int    _expand_lock_rank;
  static Mutex* const _expand_lock;

647
 private:
648 649
  // Accessors
  Metachunk* chunks_in_use(ChunkIndex index) const { return _chunks_in_use[index]; }
650 651 652
  void set_chunks_in_use(ChunkIndex index, Metachunk* v) {
    _chunks_in_use[index] = v;
  }
653 654 655 656 657

  BlockFreelist* block_freelists() const {
    return (BlockFreelist*) &_block_freelists;
  }

658
  Metaspace::MetadataType mdtype() { return _mdtype; }
659 660 661

  VirtualSpaceList* vs_list()   const { return Metaspace::get_space_list(_mdtype); }
  ChunkManager* chunk_manager() const { return Metaspace::get_chunk_manager(_mdtype); }
662 663 664 665 666 667 668 669 670 671

  Metachunk* current_chunk() const { return _current_chunk; }
  void set_current_chunk(Metachunk* v) {
    _current_chunk = v;
  }

  Metachunk* find_current_chunk(size_t word_size);

  // Add chunk to the list of chunks in use
  void add_chunk(Metachunk* v, bool make_current);
672
  void retire_current_chunk();
673 674 675

  Mutex* lock() const { return _lock; }

676 677 678 679 680
  const char* chunk_size_name(ChunkIndex index) const;

 protected:
  void initialize();

681
 public:
682
  SpaceManager(Metaspace::MetadataType mdtype,
683
               Mutex* lock);
684 685
  ~SpaceManager();

686 687
  enum ChunkMultiples {
    MediumChunkMultiple = 4
688 689
  };

690 691
  bool is_class() { return _mdtype == Metaspace::ClassType; }

692
  // Accessors
693 694 695 696 697 698
  size_t specialized_chunk_size() { return (size_t) is_class() ? ClassSpecializedChunk : SpecializedChunk; }
  size_t small_chunk_size()       { return (size_t) is_class() ? ClassSmallChunk : SmallChunk; }
  size_t medium_chunk_size()      { return (size_t) is_class() ? ClassMediumChunk : MediumChunk; }
  size_t medium_chunk_bunch()     { return medium_chunk_size() * MediumChunkMultiple; }

  size_t smallest_chunk_size()  { return specialized_chunk_size(); }
699

700 701 702 703 704
  size_t allocated_blocks_words() const { return _allocated_blocks_words; }
  size_t allocated_blocks_bytes() const { return _allocated_blocks_words * BytesPerWord; }
  size_t allocated_chunks_words() const { return _allocated_chunks_words; }
  size_t allocated_chunks_count() const { return _allocated_chunks_count; }

705
  bool is_humongous(size_t word_size) { return word_size > medium_chunk_size(); }
706 707 708

  static Mutex* expand_lock() { return _expand_lock; }

709 710 711 712 713 714 715 716 717 718 719 720
  // Increment the per Metaspace and global running sums for Metachunks
  // by the given size.  This is used when a Metachunk to added to
  // the in-use list.
  void inc_size_metrics(size_t words);
  // Increment the per Metaspace and global running sums Metablocks by the given
  // size.  This is used when a Metablock is allocated.
  void inc_used_metrics(size_t words);
  // Delete the portion of the running sums for this SpaceManager. That is,
  // the globals running sums for the Metachunks and Metablocks are
  // decremented for all the Metachunks in-use by this SpaceManager.
  void dec_total_from_size_metrics();

721 722 723 724 725
  // Set the sizes for the initial chunks.
  void get_initial_chunk_sizes(Metaspace::MetaspaceType type,
                               size_t* chunk_word_size,
                               size_t* class_chunk_word_size);

726 727 728 729 730 731 732 733 734
  size_t sum_capacity_in_chunks_in_use() const;
  size_t sum_used_in_chunks_in_use() const;
  size_t sum_free_in_chunks_in_use() const;
  size_t sum_waste_in_chunks_in_use() const;
  size_t sum_waste_in_chunks_in_use(ChunkIndex index ) const;

  size_t sum_count_in_chunks_in_use();
  size_t sum_count_in_chunks_in_use(ChunkIndex i);

735 736
  Metachunk* get_new_chunk(size_t word_size, size_t grow_chunks_by_words);

737 738 739 740 741
  // Block allocation and deallocation.
  // Allocates a block from the current chunk
  MetaWord* allocate(size_t word_size);

  // Helper for allocations
742
  MetaWord* allocate_work(size_t word_size);
743 744

  // Returns a block to the per manager freelist
745
  void deallocate(MetaWord* p, size_t word_size);
746 747 748 749 750 751 752 753

  // Based on the allocation size and a minimum chunk size,
  // returned chunk size (for expanding space for chunk allocation).
  size_t calc_chunk_size(size_t allocation_word_size);

  // Called when an allocation from the current chunk fails.
  // Gets a new chunk (may require getting a new virtual space),
  // and allocates from that chunk.
754
  MetaWord* grow_and_allocate(size_t word_size);
755

756 757 758
  // Notify memory usage to MemoryService.
  void track_metaspace_memory_usage();

759 760 761 762 763 764 765
  // debugging support.

  void dump(outputStream* const out) const;
  void print_on(outputStream* st) const;
  void locked_print_chunks_in_use_on(outputStream* st) const;

  void verify();
766
  void verify_chunk_size(Metachunk* chunk);
767
  NOT_PRODUCT(void mangle_freed_chunks();)
768
#ifdef ASSERT
769
  void verify_allocated_blocks_words();
770
#endif
771 772 773 774

  size_t get_raw_word_size(size_t word_size) {
    size_t byte_size = word_size * BytesPerWord;

775 776 777
    size_t raw_bytes_size = MAX2(byte_size, sizeof(Metablock));
    raw_bytes_size = align_size_up(raw_bytes_size, Metachunk::object_alignment());

778 779 780 781 782
    size_t raw_word_size = raw_bytes_size / BytesPerWord;
    assert(raw_word_size * BytesPerWord == raw_bytes_size, "Size problem");

    return raw_word_size;
  }
783 784 785 786 787 788 789 790 791 792 793 794
};

uint const SpaceManager::_small_chunk_limit = 4;

const char* SpaceManager::_expand_lock_name =
  "SpaceManager chunk allocation lock";
const int SpaceManager::_expand_lock_rank = Monitor::leaf - 1;
Mutex* const SpaceManager::_expand_lock =
  new Mutex(SpaceManager::_expand_lock_rank,
            SpaceManager::_expand_lock_name,
            Mutex::_allow_vm_block_flag);

795 796 797 798 799
void VirtualSpaceNode::inc_container_count() {
  assert_lock_strong(SpaceManager::expand_lock());
  _container_count++;
  assert(_container_count == container_count_slow(),
         err_msg("Inconsistency in countainer_count _container_count " SIZE_FORMAT
800
                 " container_count_slow() " SIZE_FORMAT,
801 802 803 804 805 806 807 808 809 810 811 812
                 _container_count, container_count_slow()));
}

void VirtualSpaceNode::dec_container_count() {
  assert_lock_strong(SpaceManager::expand_lock());
  _container_count--;
}

#ifdef ASSERT
void VirtualSpaceNode::verify_container_count() {
  assert(_container_count == container_count_slow(),
    err_msg("Inconsistency in countainer_count _container_count " SIZE_FORMAT
813
            " container_count_slow() " SIZE_FORMAT, _container_count, container_count_slow()));
814 815 816
}
#endif

817 818 819 820 821 822 823 824 825 826 827 828 829
// BlockFreelist methods

BlockFreelist::BlockFreelist() : _dictionary(NULL) {}

BlockFreelist::~BlockFreelist() {
  if (_dictionary != NULL) {
    if (Verbose && TraceMetadataChunkAllocation) {
      _dictionary->print_free_lists(gclog_or_tty);
    }
    delete _dictionary;
  }
}

830
void BlockFreelist::return_block(MetaWord* p, size_t word_size) {
831
  Metablock* free_chunk = ::new (p) Metablock(word_size);
832
  if (dictionary() == NULL) {
833
   _dictionary = new BlockTreeDictionary();
834
  }
835
  dictionary()->return_chunk(free_chunk);
836 837
}

838
MetaWord* BlockFreelist::get_block(size_t word_size) {
839 840 841 842
  if (dictionary() == NULL) {
    return NULL;
  }

843
  if (word_size < TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
844
    // Dark matter.  Too small for dictionary.
845 846 847
    return NULL;
  }

848
  Metablock* free_block =
849
    dictionary()->get_chunk(word_size, FreeBlockDictionary<Metablock>::atLeast);
850 851 852 853
  if (free_block == NULL) {
    return NULL;
  }

854 855 856 857 858 859 860 861 862
  const size_t block_size = free_block->size();
  if (block_size > WasteMultiplier * word_size) {
    return_block((MetaWord*)free_block, block_size);
    return NULL;
  }

  MetaWord* new_block = (MetaWord*)free_block;
  assert(block_size >= word_size, "Incorrect size of block from freelist");
  const size_t unused = block_size - word_size;
863
  if (unused >= TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
864 865 866 867
    return_block(new_block + word_size, unused);
  }

  return new_block;
868 869 870 871 872 873 874 875 876 877 878 879 880
}

void BlockFreelist::print_on(outputStream* st) const {
  if (dictionary() == NULL) {
    return;
  }
  dictionary()->print_free_lists(st);
}

// VirtualSpaceNode methods

VirtualSpaceNode::~VirtualSpaceNode() {
  _rs.release();
881 882 883 884
#ifdef ASSERT
  size_t word_size = sizeof(*this) / BytesPerWord;
  Copy::fill_to_words((HeapWord*) this, word_size, 0xf1f1f1f1);
#endif
885 886 887 888 889 890 891 892 893 894 895
}

size_t VirtualSpaceNode::used_words_in_vs() const {
  return pointer_delta(top(), bottom(), sizeof(MetaWord));
}

// Space committed in the VirtualSpace
size_t VirtualSpaceNode::capacity_words_in_vs() const {
  return pointer_delta(end(), bottom(), sizeof(MetaWord));
}

896 897 898
size_t VirtualSpaceNode::free_words_in_vs() const {
  return pointer_delta(end(), top(), sizeof(MetaWord));
}
899 900 901 902 903 904 905 906 907

// Allocates the chunk from the virtual space only.
// This interface is also used internally for debugging.  Not all
// chunks removed here are necessarily used for allocation.
Metachunk* VirtualSpaceNode::take_from_committed(size_t chunk_word_size) {
  // Bottom of the new chunk
  MetaWord* chunk_limit = top();
  assert(chunk_limit != NULL, "Not safe to call this method");

908 909 910 911 912 913
  // The virtual spaces are always expanded by the
  // commit granularity to enforce the following condition.
  // Without this the is_available check will not work correctly.
  assert(_virtual_space.committed_size() == _virtual_space.actual_committed_size(),
      "The committed memory doesn't match the expanded memory.");

914 915
  if (!is_available(chunk_word_size)) {
    if (TraceMetadataChunkAllocation) {
916
      gclog_or_tty->print("VirtualSpaceNode::take_from_committed() not available %d words ", chunk_word_size);
917
      // Dump some information about the virtual space that is nearly full
918
      print_on(gclog_or_tty);
919 920 921 922 923 924 925
    }
    return NULL;
  }

  // Take the space  (bump top on the current virtual space).
  inc_top(chunk_word_size);

926 927
  // Initialize the chunk
  Metachunk* result = ::new (chunk_limit) Metachunk(chunk_word_size, this);
928 929 930 931 932
  return result;
}


// Expand the virtual space (commit more of the reserved space)
933 934 935 936 937 938 939 940
bool VirtualSpaceNode::expand_by(size_t min_words, size_t preferred_words) {
  size_t min_bytes = min_words * BytesPerWord;
  size_t preferred_bytes = preferred_words * BytesPerWord;

  size_t uncommitted = virtual_space()->reserved_size() - virtual_space()->actual_committed_size();

  if (uncommitted < min_bytes) {
    return false;
941
  }
942 943 944 945 946 947

  size_t commit = MIN2(preferred_bytes, uncommitted);
  bool result = virtual_space()->expand_by(commit, false);

  assert(result, "Failed to commit memory");

948 949 950 951 952
  return result;
}

Metachunk* VirtualSpaceNode::get_chunk_vs(size_t chunk_word_size) {
  assert_lock_strong(SpaceManager::expand_lock());
953 954 955 956 957
  Metachunk* result = take_from_committed(chunk_word_size);
  if (result != NULL) {
    inc_container_count();
  }
  return result;
958 959 960 961 962 963 964 965
}

bool VirtualSpaceNode::initialize() {

  if (!_rs.is_reserved()) {
    return false;
  }

966 967 968 969 970 971 972 973 974 975 976 977 978
  // These are necessary restriction to make sure that the virtual space always
  // grows in steps of Metaspace::commit_alignment(). If both base and size are
  // aligned only the middle alignment of the VirtualSpace is used.
  assert_is_ptr_aligned(_rs.base(), Metaspace::commit_alignment());
  assert_is_size_aligned(_rs.size(), Metaspace::commit_alignment());

  // ReservedSpaces marked as special will have the entire memory
  // pre-committed. Setting a committed size will make sure that
  // committed_size and actual_committed_size agrees.
  size_t pre_committed_size = _rs.special() ? _rs.size() : 0;

  bool result = virtual_space()->initialize_with_granularity(_rs, pre_committed_size,
                                            Metaspace::commit_alignment());
979
  if (result) {
980 981 982
    assert(virtual_space()->committed_size() == virtual_space()->actual_committed_size(),
        "Checking that the pre-committed memory was registered by the VirtualSpace");

983 984 985 986
    set_top((MetaWord*)virtual_space()->low());
    set_reserved(MemRegion((HeapWord*)_rs.base(),
                 (HeapWord*)(_rs.base() + _rs.size())));

987 988 989 990 991 992 993 994
    assert(reserved()->start() == (HeapWord*) _rs.base(),
      err_msg("Reserved start was not set properly " PTR_FORMAT
        " != " PTR_FORMAT, reserved()->start(), _rs.base()));
    assert(reserved()->word_size() == _rs.size() / BytesPerWord,
      err_msg("Reserved size was not set properly " SIZE_FORMAT
        " != " SIZE_FORMAT, reserved()->word_size(),
        _rs.size() / BytesPerWord));
  }
995 996 997 998 999 1000 1001 1002 1003 1004 1005

  return result;
}

void VirtualSpaceNode::print_on(outputStream* st) const {
  size_t used = used_words_in_vs();
  size_t capacity = capacity_words_in_vs();
  VirtualSpace* vs = virtual_space();
  st->print_cr("   space @ " PTR_FORMAT " " SIZE_FORMAT "K, %3d%% used "
           "[" PTR_FORMAT ", " PTR_FORMAT ", "
           PTR_FORMAT ", " PTR_FORMAT ")",
1006 1007
           vs, capacity / K,
           capacity == 0 ? 0 : used * 100 / capacity,
1008 1009 1010 1011
           bottom(), top(), end(),
           vs->high_boundary());
}

1012
#ifdef ASSERT
1013 1014 1015 1016
void VirtualSpaceNode::mangle() {
  size_t word_size = capacity_words_in_vs();
  Copy::fill_to_words((HeapWord*) low(), word_size, 0xf1f1f1f1);
}
1017
#endif // ASSERT
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029

// VirtualSpaceList methods
// Space allocated from the VirtualSpace

VirtualSpaceList::~VirtualSpaceList() {
  VirtualSpaceListIterator iter(virtual_space_list());
  while (iter.repeat()) {
    VirtualSpaceNode* vsl = iter.get_next();
    delete vsl;
  }
}

1030
void VirtualSpaceList::inc_reserved_words(size_t v) {
1031
  assert_lock_strong(SpaceManager::expand_lock());
1032
  _reserved_words = _reserved_words + v;
1033
}
1034
void VirtualSpaceList::dec_reserved_words(size_t v) {
1035
  assert_lock_strong(SpaceManager::expand_lock());
1036 1037 1038
  _reserved_words = _reserved_words - v;
}

1039 1040 1041 1042 1043 1044
#define assert_committed_below_limit()                             \
  assert(MetaspaceAux::committed_bytes() <= MaxMetaspaceSize,      \
      err_msg("Too much committed memory. Committed: " SIZE_FORMAT \
              " limit (MaxMetaspaceSize): " SIZE_FORMAT,           \
          MetaspaceAux::committed_bytes(), MaxMetaspaceSize));

1045 1046 1047
void VirtualSpaceList::inc_committed_words(size_t v) {
  assert_lock_strong(SpaceManager::expand_lock());
  _committed_words = _committed_words + v;
1048 1049

  assert_committed_below_limit();
1050 1051 1052 1053
}
void VirtualSpaceList::dec_committed_words(size_t v) {
  assert_lock_strong(SpaceManager::expand_lock());
  _committed_words = _committed_words - v;
1054 1055

  assert_committed_below_limit();
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
}

void VirtualSpaceList::inc_virtual_space_count() {
  assert_lock_strong(SpaceManager::expand_lock());
  _virtual_space_count++;
}
void VirtualSpaceList::dec_virtual_space_count() {
  assert_lock_strong(SpaceManager::expand_lock());
  _virtual_space_count--;
}

void ChunkManager::remove_chunk(Metachunk* chunk) {
  size_t word_size = chunk->word_size();
  ChunkIndex index = list_index(word_size);
  if (index != HumongousIndex) {
    free_chunks(index)->remove_chunk(chunk);
  } else {
    humongous_dictionary()->remove_chunk(chunk);
  }

  // Chunk is being removed from the chunks free list.
1077
  dec_free_chunks_total(chunk->word_size());
1078 1079 1080 1081 1082
}

// Walk the list of VirtualSpaceNodes and delete
// nodes with a 0 container_count.  Remove Metachunks in
// the node from their respective freelists.
1083
void VirtualSpaceList::purge(ChunkManager* chunk_manager) {
1084
  assert(SafepointSynchronize::is_at_safepoint(), "must be called at safepoint for contains to work");
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
  assert_lock_strong(SpaceManager::expand_lock());
  // Don't use a VirtualSpaceListIterator because this
  // list is being changed and a straightforward use of an iterator is not safe.
  VirtualSpaceNode* purged_vsl = NULL;
  VirtualSpaceNode* prev_vsl = virtual_space_list();
  VirtualSpaceNode* next_vsl = prev_vsl;
  while (next_vsl != NULL) {
    VirtualSpaceNode* vsl = next_vsl;
    next_vsl = vsl->next();
    // Don't free the current virtual space since it will likely
    // be needed soon.
    if (vsl->container_count() == 0 && vsl != current_virtual_space()) {
      // Unlink it from the list
      if (prev_vsl == vsl) {
1099 1100
        // This is the case of the current node being the first node.
        assert(vsl == virtual_space_list(), "Expected to be the first node");
1101 1102 1103 1104 1105
        set_virtual_space_list(vsl->next());
      } else {
        prev_vsl->set_next(vsl->next());
      }

1106
      vsl->purge(chunk_manager);
1107 1108
      dec_reserved_words(vsl->reserved_words());
      dec_committed_words(vsl->committed_words());
1109 1110 1111 1112 1113 1114 1115 1116 1117
      dec_virtual_space_count();
      purged_vsl = vsl;
      delete vsl;
    } else {
      prev_vsl = vsl;
    }
  }
#ifdef ASSERT
  if (purged_vsl != NULL) {
1118 1119
    // List should be stable enough to use an iterator here.
    VirtualSpaceListIterator iter(virtual_space_list());
1120 1121 1122 1123 1124 1125 1126 1127
    while (iter.repeat()) {
      VirtualSpaceNode* vsl = iter.get_next();
      assert(vsl != purged_vsl, "Purge of vsl failed");
    }
  }
#endif
}

1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144

// This function looks at the mmap regions in the metaspace without locking.
// The chunks are added with store ordering and not deleted except for at
// unloading time during a safepoint.
bool VirtualSpaceList::contains(const void* ptr) {
  // List should be stable enough to use an iterator here because removing virtual
  // space nodes is only allowed at a safepoint.
  VirtualSpaceListIterator iter(virtual_space_list());
  while (iter.repeat()) {
    VirtualSpaceNode* vsn = iter.get_next();
    if (vsn->contains(ptr)) {
      return true;
    }
  }
  return false;
}

1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
void VirtualSpaceList::retire_current_virtual_space() {
  assert_lock_strong(SpaceManager::expand_lock());

  VirtualSpaceNode* vsn = current_virtual_space();

  ChunkManager* cm = is_class() ? Metaspace::chunk_manager_class() :
                                  Metaspace::chunk_manager_metadata();

  vsn->retire(cm);
}

void VirtualSpaceNode::retire(ChunkManager* chunk_manager) {
  for (int i = (int)MediumIndex; i >= (int)ZeroIndex; --i) {
    ChunkIndex index = (ChunkIndex)i;
    size_t chunk_size = chunk_manager->free_chunks(index)->size();

    while (free_words_in_vs() >= chunk_size) {
      DEBUG_ONLY(verify_container_count();)
      Metachunk* chunk = get_chunk_vs(chunk_size);
      assert(chunk != NULL, "allocation should have been successful");

      chunk_manager->return_chunks(index, chunk);
      chunk_manager->inc_free_chunks_total(chunk_size);
      DEBUG_ONLY(verify_container_count();)
    }
  }
  assert(free_words_in_vs() == 0, "should be empty now");
}

1174
VirtualSpaceList::VirtualSpaceList(size_t word_size) :
1175 1176 1177
                                   _is_class(false),
                                   _virtual_space_list(NULL),
                                   _current_virtual_space(NULL),
1178 1179
                                   _reserved_words(0),
                                   _committed_words(0),
1180 1181 1182
                                   _virtual_space_count(0) {
  MutexLockerEx cl(SpaceManager::expand_lock(),
                   Mutex::_no_safepoint_check_flag);
1183
  create_new_virtual_space(word_size);
1184 1185 1186 1187 1188 1189
}

VirtualSpaceList::VirtualSpaceList(ReservedSpace rs) :
                                   _is_class(true),
                                   _virtual_space_list(NULL),
                                   _current_virtual_space(NULL),
1190 1191
                                   _reserved_words(0),
                                   _committed_words(0),
1192 1193 1194 1195 1196
                                   _virtual_space_count(0) {
  MutexLockerEx cl(SpaceManager::expand_lock(),
                   Mutex::_no_safepoint_check_flag);
  VirtualSpaceNode* class_entry = new VirtualSpaceNode(rs);
  bool succeeded = class_entry->initialize();
1197 1198 1199
  if (succeeded) {
    link_vs(class_entry);
  }
1200 1201
}

1202 1203 1204 1205
size_t VirtualSpaceList::free_bytes() {
  return virtual_space_list()->free_words_in_vs() * BytesPerWord;
}

1206
// Allocate another meta virtual space and add it to the list.
1207
bool VirtualSpaceList::create_new_virtual_space(size_t vs_word_size) {
1208
  assert_lock_strong(SpaceManager::expand_lock());
1209 1210 1211 1212 1213 1214 1215 1216

  if (is_class()) {
    assert(false, "We currently don't support more than one VirtualSpace for"
                  " the compressed class space. The initialization of the"
                  " CCS uses another code path and should not hit this path.");
    return false;
  }

1217
  if (vs_word_size == 0) {
1218
    assert(false, "vs_word_size should always be at least _reserve_alignment large.");
1219 1220
    return false;
  }
1221

1222 1223
  // Reserve the space
  size_t vs_byte_size = vs_word_size * BytesPerWord;
1224
  assert_is_size_aligned(vs_byte_size, Metaspace::reserve_alignment());
1225 1226 1227 1228 1229 1230 1231

  // Allocate the meta virtual space and initialize it.
  VirtualSpaceNode* new_entry = new VirtualSpaceNode(vs_byte_size);
  if (!new_entry->initialize()) {
    delete new_entry;
    return false;
  } else {
1232 1233
    assert(new_entry->reserved_words() == vs_word_size,
        "Reserved memory size differs from requested memory size");
1234 1235
    // ensure lock-free iteration sees fully initialized node
    OrderAccess::storestore();
1236
    link_vs(new_entry);
1237 1238 1239 1240
    return true;
  }
}

1241
void VirtualSpaceList::link_vs(VirtualSpaceNode* new_entry) {
1242 1243 1244 1245 1246 1247
  if (virtual_space_list() == NULL) {
      set_virtual_space_list(new_entry);
  } else {
    current_virtual_space()->set_next(new_entry);
  }
  set_current_virtual_space(new_entry);
1248 1249
  inc_reserved_words(new_entry->reserved_words());
  inc_committed_words(new_entry->committed_words());
1250 1251 1252 1253 1254 1255
  inc_virtual_space_count();
#ifdef ASSERT
  new_entry->mangle();
#endif
  if (TraceMetavirtualspaceAllocation && Verbose) {
    VirtualSpaceNode* vsl = current_virtual_space();
1256
    vsl->print_on(gclog_or_tty);
1257 1258 1259
  }
}

1260 1261 1262
bool VirtualSpaceList::expand_node_by(VirtualSpaceNode* node,
                                      size_t min_words,
                                      size_t preferred_words) {
1263 1264
  size_t before = node->committed_words();

1265
  bool result = node->expand_by(min_words, preferred_words);
1266 1267 1268 1269

  size_t after = node->committed_words();

  // after and before can be the same if the memory was pre-committed.
1270
  assert(after >= before, "Inconsistency");
1271 1272 1273 1274 1275
  inc_committed_words(after - before);

  return result;
}

1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298
bool VirtualSpaceList::expand_by(size_t min_words, size_t preferred_words) {
  assert_is_size_aligned(min_words,       Metaspace::commit_alignment_words());
  assert_is_size_aligned(preferred_words, Metaspace::commit_alignment_words());
  assert(min_words <= preferred_words, "Invalid arguments");

  if (!MetaspaceGC::can_expand(min_words, this->is_class())) {
    return  false;
  }

  size_t allowed_expansion_words = MetaspaceGC::allowed_expansion();
  if (allowed_expansion_words < min_words) {
    return false;
  }

  size_t max_expansion_words = MIN2(preferred_words, allowed_expansion_words);

  // Commit more memory from the the current virtual space.
  bool vs_expanded = expand_node_by(current_virtual_space(),
                                    min_words,
                                    max_expansion_words);
  if (vs_expanded) {
    return true;
  }
1299
  retire_current_virtual_space();
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321

  // Get another virtual space.
  size_t grow_vs_words = MAX2((size_t)VirtualSpaceSize, preferred_words);
  grow_vs_words = align_size_up(grow_vs_words, Metaspace::reserve_alignment_words());

  if (create_new_virtual_space(grow_vs_words)) {
    if (current_virtual_space()->is_pre_committed()) {
      // The memory was pre-committed, so we are done here.
      assert(min_words <= current_virtual_space()->committed_words(),
          "The new VirtualSpace was pre-committed, so it"
          "should be large enough to fit the alloc request.");
      return true;
    }

    return expand_node_by(current_virtual_space(),
                          min_words,
                          max_expansion_words);
  }

  return false;
}

1322
Metachunk* VirtualSpaceList::get_new_chunk(size_t word_size,
1323 1324
                                           size_t grow_chunks_by_words,
                                           size_t medium_chunk_bunch) {
1325

1326 1327
  // Allocate a chunk out of the current virtual space.
  Metachunk* next = current_virtual_space()->get_chunk_vs(grow_chunks_by_words);
1328

1329 1330
  if (next != NULL) {
    return next;
1331 1332
  }

1333 1334
  // The expand amount is currently only determined by the requested sizes
  // and not how much committed memory is left in the current virtual space.
1335

1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
  size_t min_word_size       = align_size_up(grow_chunks_by_words, Metaspace::commit_alignment_words());
  size_t preferred_word_size = align_size_up(medium_chunk_bunch,   Metaspace::commit_alignment_words());
  if (min_word_size >= preferred_word_size) {
    // Can happen when humongous chunks are allocated.
    preferred_word_size = min_word_size;
  }

  bool expanded = expand_by(min_word_size, preferred_word_size);
  if (expanded) {
    next = current_virtual_space()->get_chunk_vs(grow_chunks_by_words);
    assert(next != NULL, "The allocation was expected to succeed after the expansion");
  }

   return next;
1350 1351
}

1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
void VirtualSpaceList::print_on(outputStream* st) const {
  if (TraceMetadataChunkAllocation && Verbose) {
    VirtualSpaceListIterator iter(virtual_space_list());
    while (iter.repeat()) {
      VirtualSpaceNode* node = iter.get_next();
      node->print_on(st);
    }
  }
}

// MetaspaceGC methods

// VM_CollectForMetadataAllocation is the vm operation used to GC.
// Within the VM operation after the GC the attempt to allocate the metadata
// should succeed.  If the GC did not free enough space for the metaspace
// allocation, the HWM is increased so that another virtualspace will be
// allocated for the metadata.  With perm gen the increase in the perm
// gen had bounds, MinMetaspaceExpansion and MaxMetaspaceExpansion.  The
// metaspace policy uses those as the small and large steps for the HWM.
//
// After the GC the compute_new_size() for MetaspaceGC is called to
// resize the capacity of the metaspaces.  The current implementation
1374
// is based on the flags MinMetaspaceFreeRatio and MaxMetaspaceFreeRatio used
1375
// to resize the Java heap by some GC's.  New flags can be implemented
1376
// if really needed.  MinMetaspaceFreeRatio is used to calculate how much
1377
// free space is desirable in the metaspace capacity to decide how much
1378
// to increase the HWM.  MaxMetaspaceFreeRatio is used to decide how much
1379 1380 1381 1382 1383 1384
// free space is desirable in the metaspace capacity before decreasing
// the HWM.

// Calculate the amount to increase the high water mark (HWM).
// Increase by a minimum amount (MinMetaspaceExpansion) so that
// another expansion is not requested too soon.  If that is not
1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
// enough to satisfy the allocation, increase by MaxMetaspaceExpansion.
// If that is still not enough, expand by the size of the allocation
// plus some.
size_t MetaspaceGC::delta_capacity_until_GC(size_t bytes) {
  size_t min_delta = MinMetaspaceExpansion;
  size_t max_delta = MaxMetaspaceExpansion;
  size_t delta = align_size_up(bytes, Metaspace::commit_alignment());

  if (delta <= min_delta) {
    delta = min_delta;
  } else if (delta <= max_delta) {
1396 1397 1398
    // Don't want to hit the high water mark on the next
    // allocation so make the delta greater than just enough
    // for this allocation.
1399 1400 1401 1402 1403
    delta = max_delta;
  } else {
    // This allocation is large but the next ones are probably not
    // so increase by the minimum.
    delta = delta + min_delta;
1404
  }
1405 1406 1407 1408

  assert_is_size_aligned(delta, Metaspace::commit_alignment());

  return delta;
1409 1410
}

1411 1412 1413 1414 1415
size_t MetaspaceGC::capacity_until_GC() {
  size_t value = (size_t)OrderAccess::load_ptr_acquire(&_capacity_until_GC);
  assert(value >= MetaspaceSize, "Not initialied properly?");
  return value;
}
1416

1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428
size_t MetaspaceGC::inc_capacity_until_GC(size_t v) {
  assert_is_size_aligned(v, Metaspace::commit_alignment());

  return (size_t)Atomic::add_ptr(v, &_capacity_until_GC);
}

size_t MetaspaceGC::dec_capacity_until_GC(size_t v) {
  assert_is_size_aligned(v, Metaspace::commit_alignment());

  return (size_t)Atomic::add_ptr(-(intptr_t)v, &_capacity_until_GC);
}

1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439
void MetaspaceGC::initialize() {
  // Set the high-water mark to MaxMetapaceSize during VM initializaton since
  // we can't do a GC during initialization.
  _capacity_until_GC = MaxMetaspaceSize;
}

void MetaspaceGC::post_initialize() {
  // Reset the high-water mark once the VM initialization is done.
  _capacity_until_GC = MAX2(MetaspaceAux::committed_bytes(), MetaspaceSize);
}

1440 1441 1442 1443 1444
bool MetaspaceGC::can_expand(size_t word_size, bool is_class) {
  // Check if the compressed class space is full.
  if (is_class && Metaspace::using_class_space()) {
    size_t class_committed = MetaspaceAux::committed_bytes(Metaspace::ClassType);
    if (class_committed + word_size * BytesPerWord > CompressedClassSpaceSize) {
1445 1446
      return false;
    }
1447 1448
  }

1449 1450 1451 1452 1453
  // Check if the user has imposed a limit on the metaspace memory.
  size_t committed_bytes = MetaspaceAux::committed_bytes();
  if (committed_bytes + word_size * BytesPerWord > MaxMetaspaceSize) {
    return false;
  }
1454

1455 1456 1457 1458 1459 1460
  return true;
}

size_t MetaspaceGC::allowed_expansion() {
  size_t committed_bytes = MetaspaceAux::committed_bytes();
  size_t capacity_until_gc = capacity_until_GC();
1461

1462 1463 1464
  assert(capacity_until_gc >= committed_bytes,
        err_msg("capacity_until_gc: " SIZE_FORMAT " < committed_bytes: " SIZE_FORMAT,
                capacity_until_gc, committed_bytes));
1465

1466
  size_t left_until_max  = MaxMetaspaceSize - committed_bytes;
1467 1468
  size_t left_until_GC = capacity_until_gc - committed_bytes;
  size_t left_to_commit = MIN2(left_until_GC, left_until_max);
1469

1470 1471
  return left_to_commit / BytesPerWord;
}
1472 1473 1474 1475 1476 1477

void MetaspaceGC::compute_new_size() {
  assert(_shrink_factor <= 100, "invalid shrink factor");
  uint current_shrink_factor = _shrink_factor;
  _shrink_factor = 0;

1478 1479 1480 1481 1482 1483 1484 1485 1486
  // Using committed_bytes() for used_after_gc is an overestimation, since the
  // chunk free lists are included in committed_bytes() and the memory in an
  // un-fragmented chunk free list is available for future allocations.
  // However, if the chunk free lists becomes fragmented, then the memory may
  // not be available for future allocations and the memory is therefore "in use".
  // Including the chunk free lists in the definition of "in use" is therefore
  // necessary. Not including the chunk free lists can cause capacity_until_GC to
  // shrink below committed_bytes() and this has caused serious bugs in the past.
  const size_t used_after_gc = MetaspaceAux::committed_bytes();
1487
  const size_t capacity_until_GC = MetaspaceGC::capacity_until_GC();
1488

1489
  const double minimum_free_percentage = MinMetaspaceFreeRatio / 100.0;
1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506
  const double maximum_used_percentage = 1.0 - minimum_free_percentage;

  const double min_tmp = used_after_gc / maximum_used_percentage;
  size_t minimum_desired_capacity =
    (size_t)MIN2(min_tmp, double(max_uintx));
  // Don't shrink less than the initial generation size
  minimum_desired_capacity = MAX2(minimum_desired_capacity,
                                  MetaspaceSize);

  if (PrintGCDetails && Verbose) {
    gclog_or_tty->print_cr("\nMetaspaceGC::compute_new_size: ");
    gclog_or_tty->print_cr("  "
                  "  minimum_free_percentage: %6.2f"
                  "  maximum_used_percentage: %6.2f",
                  minimum_free_percentage,
                  maximum_used_percentage);
    gclog_or_tty->print_cr("  "
1507 1508
                  "   used_after_gc       : %6.1fKB",
                  used_after_gc / (double) K);
1509 1510 1511
  }


1512
  size_t shrink_bytes = 0;
1513 1514 1515 1516
  if (capacity_until_GC < minimum_desired_capacity) {
    // If we have less capacity below the metaspace HWM, then
    // increment the HWM.
    size_t expand_bytes = minimum_desired_capacity - capacity_until_GC;
1517
    expand_bytes = align_size_up(expand_bytes, Metaspace::commit_alignment());
1518 1519
    // Don't expand unless it's significant
    if (expand_bytes >= MinMetaspaceExpansion) {
1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534
      size_t new_capacity_until_GC = MetaspaceGC::inc_capacity_until_GC(expand_bytes);
      Metaspace::tracer()->report_gc_threshold(capacity_until_GC,
                                               new_capacity_until_GC,
                                               MetaspaceGCThresholdUpdater::ComputeNewSize);
      if (PrintGCDetails && Verbose) {
        gclog_or_tty->print_cr("    expanding:"
                      "  minimum_desired_capacity: %6.1fKB"
                      "  expand_bytes: %6.1fKB"
                      "  MinMetaspaceExpansion: %6.1fKB"
                      "  new metaspace HWM:  %6.1fKB",
                      minimum_desired_capacity / (double) K,
                      expand_bytes / (double) K,
                      MinMetaspaceExpansion / (double) K,
                      new_capacity_until_GC / (double) K);
      }
1535 1536 1537 1538 1539 1540
    }
    return;
  }

  // No expansion, now see if we want to shrink
  // We would never want to shrink more than this
1541 1542 1543
  size_t max_shrink_bytes = capacity_until_GC - minimum_desired_capacity;
  assert(max_shrink_bytes >= 0, err_msg("max_shrink_bytes " SIZE_FORMAT,
    max_shrink_bytes));
1544 1545

  // Should shrinking be considered?
1546 1547
  if (MaxMetaspaceFreeRatio < 100) {
    const double maximum_free_percentage = MaxMetaspaceFreeRatio / 100.0;
1548 1549 1550 1551 1552
    const double minimum_used_percentage = 1.0 - maximum_free_percentage;
    const double max_tmp = used_after_gc / minimum_used_percentage;
    size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(max_uintx));
    maximum_desired_capacity = MAX2(maximum_desired_capacity,
                                    MetaspaceSize);
1553
    if (PrintGCDetails && Verbose) {
1554 1555 1556 1557 1558 1559
      gclog_or_tty->print_cr("  "
                             "  maximum_free_percentage: %6.2f"
                             "  minimum_used_percentage: %6.2f",
                             maximum_free_percentage,
                             minimum_used_percentage);
      gclog_or_tty->print_cr("  "
1560 1561
                             "  minimum_desired_capacity: %6.1fKB"
                             "  maximum_desired_capacity: %6.1fKB",
1562 1563 1564 1565 1566 1567 1568 1569 1570
                             minimum_desired_capacity / (double) K,
                             maximum_desired_capacity / (double) K);
    }

    assert(minimum_desired_capacity <= maximum_desired_capacity,
           "sanity check");

    if (capacity_until_GC > maximum_desired_capacity) {
      // Capacity too large, compute shrinking size
1571
      shrink_bytes = capacity_until_GC - maximum_desired_capacity;
1572 1573 1574 1575 1576 1577
      // We don't want shrink all the way back to initSize if people call
      // System.gc(), because some programs do that between "phases" and then
      // we'd just have to grow the heap up again for the next phase.  So we
      // damp the shrinking: 0% on the first call, 10% on the second call, 40%
      // on the third call, and 100% by the fourth call.  But if we recompute
      // size without shrinking, it goes back to 0%.
1578
      shrink_bytes = shrink_bytes / 100 * current_shrink_factor;
1579 1580 1581

      shrink_bytes = align_size_down(shrink_bytes, Metaspace::commit_alignment());

1582
      assert(shrink_bytes <= max_shrink_bytes,
1583
        err_msg("invalid shrink size " SIZE_FORMAT " not <= " SIZE_FORMAT,
1584
          shrink_bytes, max_shrink_bytes));
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597
      if (current_shrink_factor == 0) {
        _shrink_factor = 10;
      } else {
        _shrink_factor = MIN2(current_shrink_factor * 4, (uint) 100);
      }
      if (PrintGCDetails && Verbose) {
        gclog_or_tty->print_cr("  "
                      "  shrinking:"
                      "  initSize: %.1fK"
                      "  maximum_desired_capacity: %.1fK",
                      MetaspaceSize / (double) K,
                      maximum_desired_capacity / (double) K);
        gclog_or_tty->print_cr("  "
1598
                      "  shrink_bytes: %.1fK"
1599 1600 1601
                      "  current_shrink_factor: %d"
                      "  new shrink factor: %d"
                      "  MinMetaspaceExpansion: %.1fK",
1602
                      shrink_bytes / (double) K,
1603 1604 1605 1606 1607 1608 1609 1610
                      current_shrink_factor,
                      _shrink_factor,
                      MinMetaspaceExpansion / (double) K);
      }
    }
  }

  // Don't shrink unless it's significant
1611 1612
  if (shrink_bytes >= MinMetaspaceExpansion &&
      ((capacity_until_GC - shrink_bytes) >= MetaspaceSize)) {
1613 1614 1615 1616
    size_t new_capacity_until_GC = MetaspaceGC::dec_capacity_until_GC(shrink_bytes);
    Metaspace::tracer()->report_gc_threshold(capacity_until_GC,
                                             new_capacity_until_GC,
                                             MetaspaceGCThresholdUpdater::ComputeNewSize);
1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649
  }
}

// Metadebug methods

void Metadebug::init_allocation_fail_alot_count() {
  if (MetadataAllocationFailALot) {
    _allocation_fail_alot_count =
      1+(long)((double)MetadataAllocationFailALotInterval*os::random()/(max_jint+1.0));
  }
}

#ifdef ASSERT
bool Metadebug::test_metadata_failure() {
  if (MetadataAllocationFailALot &&
      Threads::is_vm_complete()) {
    if (_allocation_fail_alot_count > 0) {
      _allocation_fail_alot_count--;
    } else {
      if (TraceMetadataChunkAllocation && Verbose) {
        gclog_or_tty->print_cr("Metadata allocation failing for "
                               "MetadataAllocationFailALot");
      }
      init_allocation_fail_alot_count();
      return true;
    }
  }
  return false;
}
#endif

// ChunkManager methods

E
ehelin 已提交
1650
size_t ChunkManager::free_chunks_total_words() {
1651 1652 1653
  return _free_chunks_total;
}

E
ehelin 已提交
1654 1655
size_t ChunkManager::free_chunks_total_bytes() {
  return free_chunks_total_words() * BytesPerWord;
1656 1657 1658 1659 1660 1661 1662 1663 1664
}

size_t ChunkManager::free_chunks_count() {
#ifdef ASSERT
  if (!UseConcMarkSweepGC && !SpaceManager::expand_lock()->is_locked()) {
    MutexLockerEx cl(SpaceManager::expand_lock(),
                     Mutex::_no_safepoint_check_flag);
    // This lock is only needed in debug because the verification
    // of the _free_chunks_totals walks the list of free chunks
1665
    slow_locked_verify_free_chunks_count();
1666 1667
  }
#endif
1668
  return _free_chunks_count;
1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701
}

void ChunkManager::locked_verify_free_chunks_total() {
  assert_lock_strong(SpaceManager::expand_lock());
  assert(sum_free_chunks() == _free_chunks_total,
    err_msg("_free_chunks_total " SIZE_FORMAT " is not the"
           " same as sum " SIZE_FORMAT, _free_chunks_total,
           sum_free_chunks()));
}

void ChunkManager::verify_free_chunks_total() {
  MutexLockerEx cl(SpaceManager::expand_lock(),
                     Mutex::_no_safepoint_check_flag);
  locked_verify_free_chunks_total();
}

void ChunkManager::locked_verify_free_chunks_count() {
  assert_lock_strong(SpaceManager::expand_lock());
  assert(sum_free_chunks_count() == _free_chunks_count,
    err_msg("_free_chunks_count " SIZE_FORMAT " is not the"
           " same as sum " SIZE_FORMAT, _free_chunks_count,
           sum_free_chunks_count()));
}

void ChunkManager::verify_free_chunks_count() {
#ifdef ASSERT
  MutexLockerEx cl(SpaceManager::expand_lock(),
                     Mutex::_no_safepoint_check_flag);
  locked_verify_free_chunks_count();
#endif
}

void ChunkManager::verify() {
1702 1703 1704
  MutexLockerEx cl(SpaceManager::expand_lock(),
                     Mutex::_no_safepoint_check_flag);
  locked_verify();
1705 1706 1707 1708
}

void ChunkManager::locked_verify() {
  locked_verify_free_chunks_count();
1709
  locked_verify_free_chunks_total();
1710 1711 1712 1713
}

void ChunkManager::locked_print_free_chunks(outputStream* st) {
  assert_lock_strong(SpaceManager::expand_lock());
1714
  st->print_cr("Free chunk total " SIZE_FORMAT "  count " SIZE_FORMAT,
1715 1716 1717 1718 1719
                _free_chunks_total, _free_chunks_count);
}

void ChunkManager::locked_print_sum_free_chunks(outputStream* st) {
  assert_lock_strong(SpaceManager::expand_lock());
1720
  st->print_cr("Sum free chunk total " SIZE_FORMAT "  count " SIZE_FORMAT,
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731
                sum_free_chunks(), sum_free_chunks_count());
}
ChunkList* ChunkManager::free_chunks(ChunkIndex index) {
  return &_free_chunks[index];
}

// These methods that sum the free chunk lists are used in printing
// methods that are used in product builds.
size_t ChunkManager::sum_free_chunks() {
  assert_lock_strong(SpaceManager::expand_lock());
  size_t result = 0;
1732
  for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) {
1733 1734 1735 1736 1737 1738
    ChunkList* list = free_chunks(i);

    if (list == NULL) {
      continue;
    }

1739
    result = result + list->count() * list->size();
1740
  }
1741
  result = result + humongous_dictionary()->total_size();
1742 1743 1744 1745 1746 1747
  return result;
}

size_t ChunkManager::sum_free_chunks_count() {
  assert_lock_strong(SpaceManager::expand_lock());
  size_t count = 0;
1748
  for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) {
1749 1750 1751 1752
    ChunkList* list = free_chunks(i);
    if (list == NULL) {
      continue;
    }
1753
    count = count + list->count();
1754
  }
1755
  count = count + humongous_dictionary()->total_free_blocks();
1756 1757 1758 1759
  return count;
}

ChunkList* ChunkManager::find_free_chunks_list(size_t word_size) {
1760 1761 1762
  ChunkIndex index = list_index(word_size);
  assert(index < HumongousIndex, "No humongous list");
  return free_chunks(index);
1763 1764 1765 1766 1767
}

Metachunk* ChunkManager::free_chunks_get(size_t word_size) {
  assert_lock_strong(SpaceManager::expand_lock());

1768
  slow_locked_verify();
1769

1770
  Metachunk* chunk = NULL;
1771
  if (list_index(word_size) != HumongousIndex) {
1772 1773
    ChunkList* free_list = find_free_chunks_list(word_size);
    assert(free_list != NULL, "Sanity check");
1774

1775 1776 1777 1778 1779
    chunk = free_list->head();

    if (chunk == NULL) {
      return NULL;
    }
1780 1781

    // Remove the chunk as the head of the list.
1782
    free_list->remove_chunk(chunk);
1783

1784
    if (TraceMetadataChunkAllocation && Verbose) {
1785 1786 1787
      gclog_or_tty->print_cr("ChunkManager::free_chunks_get: free_list "
                             PTR_FORMAT " head " PTR_FORMAT " size " SIZE_FORMAT,
                             free_list, chunk, chunk->word_size());
1788 1789
    }
  } else {
1790 1791 1792 1793
    chunk = humongous_dictionary()->get_chunk(
      word_size,
      FreeBlockDictionary<Metachunk>::atLeast);

1794
    if (chunk == NULL) {
1795
      return NULL;
1796
    }
1797 1798 1799 1800 1801 1802 1803 1804

    if (TraceMetadataHumongousAllocation) {
      size_t waste = chunk->word_size() - word_size;
      gclog_or_tty->print_cr("Free list allocate humongous chunk size "
                             SIZE_FORMAT " for requested size " SIZE_FORMAT
                             " waste " SIZE_FORMAT,
                             chunk->word_size(), word_size, waste);
    }
1805
  }
1806

1807
  // Chunk is being removed from the chunks free list.
1808
  dec_free_chunks_total(chunk->word_size());
1809

1810 1811 1812
  // Remove it from the links to this freelist
  chunk->set_next(NULL);
  chunk->set_prev(NULL);
1813 1814 1815
#ifdef ASSERT
  // Chunk is no longer on any freelist. Setting to false make container_count_slow()
  // work.
1816
  chunk->set_is_tagged_free(false);
1817
#endif
1818 1819
  chunk->container()->inc_container_count();

1820
  slow_locked_verify();
1821 1822 1823 1824 1825
  return chunk;
}

Metachunk* ChunkManager::chunk_freelist_allocate(size_t word_size) {
  assert_lock_strong(SpaceManager::expand_lock());
1826
  slow_locked_verify();
1827 1828 1829 1830 1831 1832 1833

  // Take from the beginning of the list
  Metachunk* chunk = free_chunks_get(word_size);
  if (chunk == NULL) {
    return NULL;
  }

1834 1835 1836
  assert((word_size <= chunk->word_size()) ||
         list_index(chunk->word_size() == HumongousIndex),
         "Non-humongous variable sized chunk");
1837
  if (TraceMetadataChunkAllocation) {
1838 1839 1840
    size_t list_count;
    if (list_index(word_size) < HumongousIndex) {
      ChunkList* list = find_free_chunks_list(word_size);
1841
      list_count = list->count();
1842 1843 1844
    } else {
      list_count = humongous_dictionary()->total_count();
    }
1845 1846 1847 1848
    gclog_or_tty->print("ChunkManager::chunk_freelist_allocate: " PTR_FORMAT " chunk "
                        PTR_FORMAT "  size " SIZE_FORMAT " count " SIZE_FORMAT " ",
                        this, chunk, chunk->word_size(), list_count);
    locked_print_free_chunks(gclog_or_tty);
1849 1850 1851 1852 1853
  }

  return chunk;
}

1854
void ChunkManager::print_on(outputStream* out) const {
1855
  if (PrintFLSStatistics != 0) {
1856
    const_cast<ChunkManager *>(this)->humongous_dictionary()->report_statistics();
1857 1858 1859
  }
}

1860 1861
// SpaceManager methods

1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887
void SpaceManager::get_initial_chunk_sizes(Metaspace::MetaspaceType type,
                                           size_t* chunk_word_size,
                                           size_t* class_chunk_word_size) {
  switch (type) {
  case Metaspace::BootMetaspaceType:
    *chunk_word_size = Metaspace::first_chunk_word_size();
    *class_chunk_word_size = Metaspace::first_class_chunk_word_size();
    break;
  case Metaspace::ROMetaspaceType:
    *chunk_word_size = SharedReadOnlySize / wordSize;
    *class_chunk_word_size = ClassSpecializedChunk;
    break;
  case Metaspace::ReadWriteMetaspaceType:
    *chunk_word_size = SharedReadWriteSize / wordSize;
    *class_chunk_word_size = ClassSpecializedChunk;
    break;
  case Metaspace::AnonymousMetaspaceType:
  case Metaspace::ReflectionMetaspaceType:
    *chunk_word_size = SpecializedChunk;
    *class_chunk_word_size = ClassSpecializedChunk;
    break;
  default:
    *chunk_word_size = SmallChunk;
    *class_chunk_word_size = ClassSmallChunk;
    break;
  }
1888
  assert(*chunk_word_size != 0 && *class_chunk_word_size != 0,
1889 1890
    err_msg("Initial chunks sizes bad: data  " SIZE_FORMAT
            " class " SIZE_FORMAT,
1891
            *chunk_word_size, *class_chunk_word_size));
1892 1893
}

1894 1895 1896
size_t SpaceManager::sum_free_in_chunks_in_use() const {
  MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
  size_t free = 0;
1897
  for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909
    Metachunk* chunk = chunks_in_use(i);
    while (chunk != NULL) {
      free += chunk->free_word_size();
      chunk = chunk->next();
    }
  }
  return free;
}

size_t SpaceManager::sum_waste_in_chunks_in_use() const {
  MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
  size_t result = 0;
1910
  for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1911 1912
   result += sum_waste_in_chunks_in_use(i);
  }
1913

1914 1915 1916 1917 1918 1919 1920 1921
  return result;
}

size_t SpaceManager::sum_waste_in_chunks_in_use(ChunkIndex index) const {
  size_t result = 0;
  Metachunk* chunk = chunks_in_use(index);
  // Count the free space in all the chunk but not the
  // current chunk from which allocations are still being done.
1922 1923
  while (chunk != NULL) {
    if (chunk != current_chunk()) {
1924
      result += chunk->free_word_size();
1925
    }
1926
    chunk = chunk->next();
1927 1928 1929 1930 1931
  }
  return result;
}

size_t SpaceManager::sum_capacity_in_chunks_in_use() const {
1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947
  // For CMS use "allocated_chunks_words()" which does not need the
  // Metaspace lock.  For the other collectors sum over the
  // lists.  Use both methods as a check that "allocated_chunks_words()"
  // is correct.  That is, sum_capacity_in_chunks() is too expensive
  // to use in the product and allocated_chunks_words() should be used
  // but allow for  checking that allocated_chunks_words() returns the same
  // value as sum_capacity_in_chunks_in_use() which is the definitive
  // answer.
  if (UseConcMarkSweepGC) {
    return allocated_chunks_words();
  } else {
    MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
    size_t sum = 0;
    for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
      Metachunk* chunk = chunks_in_use(i);
      while (chunk != NULL) {
1948
        sum += chunk->word_size();
1949 1950
        chunk = chunk->next();
      }
1951 1952
    }
  return sum;
1953
  }
1954 1955 1956 1957
}

size_t SpaceManager::sum_count_in_chunks_in_use() {
  size_t count = 0;
1958
  for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1959 1960
    count = count + sum_count_in_chunks_in_use(i);
  }
1961

1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978
  return count;
}

size_t SpaceManager::sum_count_in_chunks_in_use(ChunkIndex i) {
  size_t count = 0;
  Metachunk* chunk = chunks_in_use(i);
  while (chunk != NULL) {
    count++;
    chunk = chunk->next();
  }
  return count;
}


size_t SpaceManager::sum_used_in_chunks_in_use() const {
  MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
  size_t used = 0;
1979
  for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990
    Metachunk* chunk = chunks_in_use(i);
    while (chunk != NULL) {
      used += chunk->used_word_size();
      chunk = chunk->next();
    }
  }
  return used;
}

void SpaceManager::locked_print_chunks_in_use_on(outputStream* st) const {

1991 1992 1993 1994 1995 1996 1997 1998
  for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
    Metachunk* chunk = chunks_in_use(i);
    st->print("SpaceManager: %s " PTR_FORMAT,
                 chunk_size_name(i), chunk);
    if (chunk != NULL) {
      st->print_cr(" free " SIZE_FORMAT,
                   chunk->free_word_size());
    } else {
1999
      st->cr();
2000 2001
    }
  }
2002

2003 2004
  chunk_manager()->locked_print_free_chunks(st);
  chunk_manager()->locked_print_sum_free_chunks(st);
2005 2006 2007 2008 2009 2010 2011 2012 2013 2014
}

size_t SpaceManager::calc_chunk_size(size_t word_size) {

  // Decide between a small chunk and a medium chunk.  Up to
  // _small_chunk_limit small chunks can be allocated but
  // once a medium chunk has been allocated, no more small
  // chunks will be allocated.
  size_t chunk_word_size;
  if (chunks_in_use(MediumIndex) == NULL &&
2015
      sum_count_in_chunks_in_use(SmallIndex) < _small_chunk_limit) {
2016 2017 2018
    chunk_word_size = (size_t) small_chunk_size();
    if (word_size + Metachunk::overhead() > small_chunk_size()) {
      chunk_word_size = medium_chunk_size();
2019 2020
    }
  } else {
2021
    chunk_word_size = medium_chunk_size();
2022 2023
  }

2024 2025 2026
  // Might still need a humongous chunk.  Enforce
  // humongous allocations sizes to be aligned up to
  // the smallest chunk size.
2027 2028
  size_t if_humongous_sized_chunk =
    align_size_up(word_size + Metachunk::overhead(),
2029
                  smallest_chunk_size());
2030
  chunk_word_size =
2031
    MAX2((size_t) chunk_word_size, if_humongous_sized_chunk);
2032

2033 2034 2035 2036 2037
  assert(!SpaceManager::is_humongous(word_size) ||
         chunk_word_size == if_humongous_sized_chunk,
         err_msg("Size calculation is wrong, word_size " SIZE_FORMAT
                 " chunk_word_size " SIZE_FORMAT,
                 word_size, chunk_word_size));
2038 2039 2040 2041 2042 2043
  if (TraceMetadataHumongousAllocation &&
      SpaceManager::is_humongous(word_size)) {
    gclog_or_tty->print_cr("Metadata humongous allocation:");
    gclog_or_tty->print_cr("  word_size " PTR_FORMAT, word_size);
    gclog_or_tty->print_cr("  chunk_word_size " PTR_FORMAT,
                           chunk_word_size);
2044
    gclog_or_tty->print_cr("    chunk overhead " PTR_FORMAT,
2045 2046 2047 2048 2049
                           Metachunk::overhead());
  }
  return chunk_word_size;
}

2050 2051 2052 2053 2054 2055 2056 2057 2058
void SpaceManager::track_metaspace_memory_usage() {
  if (is_init_completed()) {
    if (is_class()) {
      MemoryService::track_compressed_class_memory_usage();
    }
    MemoryService::track_metaspace_memory_usage();
  }
}

2059
MetaWord* SpaceManager::grow_and_allocate(size_t word_size) {
2060 2061 2062 2063 2064 2065 2066 2067
  assert(vs_list()->current_virtual_space() != NULL,
         "Should have been set");
  assert(current_chunk() == NULL ||
         current_chunk()->allocate(word_size) == NULL,
         "Don't need to expand");
  MutexLockerEx cl(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);

  if (TraceMetadataChunkAllocation && Verbose) {
2068 2069 2070 2071 2072 2073
    size_t words_left = 0;
    size_t words_used = 0;
    if (current_chunk() != NULL) {
      words_left = current_chunk()->free_word_size();
      words_used = current_chunk()->used_word_size();
    }
2074
    gclog_or_tty->print_cr("SpaceManager::grow_and_allocate for " SIZE_FORMAT
2075 2076 2077
                           " words " SIZE_FORMAT " words used " SIZE_FORMAT
                           " words left",
                            word_size, words_used, words_left);
2078 2079 2080 2081
  }

  // Get another chunk out of the virtual space
  size_t grow_chunks_by_words = calc_chunk_size(word_size);
2082
  Metachunk* next = get_new_chunk(word_size, grow_chunks_by_words);
2083

2084 2085
  MetaWord* mem = NULL;

2086 2087 2088 2089 2090
  // If a chunk was available, add it to the in-use chunk list
  // and do an allocation from it.
  if (next != NULL) {
    // Add to this manager's list of chunks in use.
    add_chunk(next, false);
2091
    mem = next->allocate(word_size);
2092
  }
2093

2094 2095 2096
  // Track metaspace memory usage statistic.
  track_metaspace_memory_usage();

2097
  return mem;
2098 2099 2100 2101
}

void SpaceManager::print_on(outputStream* st) const {

2102
  for (ChunkIndex i = ZeroIndex;
2103
       i < NumberOfInUseLists ;
2104 2105 2106 2107 2108 2109 2110 2111 2112 2113
       i = next_chunk_index(i) ) {
    st->print_cr("  chunks_in_use " PTR_FORMAT " chunk size " PTR_FORMAT,
                 chunks_in_use(i),
                 chunks_in_use(i) == NULL ? 0 : chunks_in_use(i)->word_size());
  }
  st->print_cr("    waste:  Small " SIZE_FORMAT " Medium " SIZE_FORMAT
               " Humongous " SIZE_FORMAT,
               sum_waste_in_chunks_in_use(SmallIndex),
               sum_waste_in_chunks_in_use(MediumIndex),
               sum_waste_in_chunks_in_use(HumongousIndex));
2114 2115 2116 2117 2118
  // block free lists
  if (block_freelists() != NULL) {
    st->print_cr("total in block free lists " SIZE_FORMAT,
      block_freelists()->total_size());
  }
2119 2120
}

2121
SpaceManager::SpaceManager(Metaspace::MetadataType mdtype,
2122
                           Mutex* lock) :
2123
  _mdtype(mdtype),
2124 2125 2126
  _allocated_blocks_words(0),
  _allocated_chunks_words(0),
  _allocated_chunks_count(0),
2127 2128 2129 2130 2131
  _lock(lock)
{
  initialize();
}

2132 2133 2134 2135 2136 2137 2138
void SpaceManager::inc_size_metrics(size_t words) {
  assert_lock_strong(SpaceManager::expand_lock());
  // Total of allocated Metachunks and allocated Metachunks count
  // for each SpaceManager
  _allocated_chunks_words = _allocated_chunks_words + words;
  _allocated_chunks_count++;
  // Global total of capacity in allocated Metachunks
2139
  MetaspaceAux::inc_capacity(mdtype(), words);
2140 2141 2142 2143 2144
  // Global total of allocated Metablocks.
  // used_words_slow() includes the overhead in each
  // Metachunk so include it in the used when the
  // Metachunk is first added (so only added once per
  // Metachunk).
2145
  MetaspaceAux::inc_used(mdtype(), Metachunk::overhead());
2146 2147 2148 2149 2150 2151
}

void SpaceManager::inc_used_metrics(size_t words) {
  // Add to the per SpaceManager total
  Atomic::add_ptr(words, &_allocated_blocks_words);
  // Add to the global total
2152
  MetaspaceAux::inc_used(mdtype(), words);
2153 2154 2155
}

void SpaceManager::dec_total_from_size_metrics() {
2156 2157
  MetaspaceAux::dec_capacity(mdtype(), allocated_chunks_words());
  MetaspaceAux::dec_used(mdtype(), allocated_blocks_words());
2158
  // Also deduct the overhead per Metachunk
2159
  MetaspaceAux::dec_used(mdtype(), allocated_chunks_count() * Metachunk::overhead());
2160 2161
}

2162
void SpaceManager::initialize() {
2163
  Metadebug::init_allocation_fail_alot_count();
2164
  for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
2165 2166 2167 2168 2169 2170 2171 2172
    _chunks_in_use[i] = NULL;
  }
  _current_chunk = NULL;
  if (TraceMetadataChunkAllocation && Verbose) {
    gclog_or_tty->print_cr("SpaceManager(): " PTR_FORMAT, this);
  }
}

2173 2174 2175 2176 2177 2178 2179 2180 2181
void ChunkManager::return_chunks(ChunkIndex index, Metachunk* chunks) {
  if (chunks == NULL) {
    return;
  }
  ChunkList* list = free_chunks(index);
  assert(list->size() == chunks->word_size(), "Mismatch in chunk sizes");
  assert_lock_strong(SpaceManager::expand_lock());
  Metachunk* cur = chunks;

2182
  // This returns chunks one at a time.  If a new
2183 2184 2185 2186
  // class List can be created that is a base class
  // of FreeList then something like FreeList::prepend()
  // can be used in place of this loop
  while (cur != NULL) {
2187 2188
    assert(cur->container() != NULL, "Container should have been set");
    cur->container()->dec_container_count();
2189 2190 2191
    // Capture the next link before it is changed
    // by the call to return_chunk_at_head();
    Metachunk* next = cur->next();
2192
    DEBUG_ONLY(cur->set_is_tagged_free(true);)
2193 2194 2195 2196 2197
    list->return_chunk_at_head(cur);
    cur = next;
  }
}

2198
SpaceManager::~SpaceManager() {
2199
  // This call this->_lock which can't be done while holding expand_lock()
2200 2201 2202 2203
  assert(sum_capacity_in_chunks_in_use() == allocated_chunks_words(),
    err_msg("sum_capacity_in_chunks_in_use() " SIZE_FORMAT
            " allocated_chunks_words() " SIZE_FORMAT,
            sum_capacity_in_chunks_in_use(), allocated_chunks_words()));
2204

2205 2206 2207
  MutexLockerEx fcl(SpaceManager::expand_lock(),
                    Mutex::_no_safepoint_check_flag);

2208
  chunk_manager()->slow_locked_verify();
2209

2210 2211
  dec_total_from_size_metrics();

2212 2213 2214 2215 2216
  if (TraceMetadataChunkAllocation && Verbose) {
    gclog_or_tty->print_cr("~SpaceManager(): " PTR_FORMAT, this);
    locked_print_chunks_in_use_on(gclog_or_tty);
  }

2217 2218
  // Do not mangle freed Metachunks.  The chunk size inside Metachunks
  // is during the freeing of a VirtualSpaceNodes.
2219

2220 2221
  // Have to update before the chunks_in_use lists are emptied
  // below.
2222 2223
  chunk_manager()->inc_free_chunks_total(allocated_chunks_words(),
                                         sum_count_in_chunks_in_use());
2224 2225 2226 2227

  // Add all the chunks in use by this space manager
  // to the global list of free chunks.

2228 2229 2230 2231 2232 2233 2234 2235 2236 2237
  // Follow each list of chunks-in-use and add them to the
  // free lists.  Each list is NULL terminated.

  for (ChunkIndex i = ZeroIndex; i < HumongousIndex; i = next_chunk_index(i)) {
    if (TraceMetadataChunkAllocation && Verbose) {
      gclog_or_tty->print_cr("returned %d %s chunks to freelist",
                             sum_count_in_chunks_in_use(i),
                             chunk_size_name(i));
    }
    Metachunk* chunks = chunks_in_use(i);
2238
    chunk_manager()->return_chunks(i, chunks);
2239 2240 2241
    set_chunks_in_use(i, NULL);
    if (TraceMetadataChunkAllocation && Verbose) {
      gclog_or_tty->print_cr("updated freelist count %d %s",
2242
                             chunk_manager()->free_chunks(i)->count(),
2243 2244 2245
                             chunk_size_name(i));
    }
    assert(i != HumongousIndex, "Humongous chunks are handled explicitly later");
2246 2247
  }

2248 2249 2250 2251
  // The medium chunk case may be optimized by passing the head and
  // tail of the medium chunk list to add_at_head().  The tail is often
  // the current chunk but there are probably exceptions.

2252
  // Humongous chunks
2253 2254 2255 2256 2257 2258
  if (TraceMetadataChunkAllocation && Verbose) {
    gclog_or_tty->print_cr("returned %d %s humongous chunks to dictionary",
                            sum_count_in_chunks_in_use(HumongousIndex),
                            chunk_size_name(HumongousIndex));
    gclog_or_tty->print("Humongous chunk dictionary: ");
  }
2259 2260 2261
  // Humongous chunks are never the current chunk.
  Metachunk* humongous_chunks = chunks_in_use(HumongousIndex);

2262 2263
  while (humongous_chunks != NULL) {
#ifdef ASSERT
2264
    humongous_chunks->set_is_tagged_free(true);
2265
#endif
2266 2267 2268 2269 2270 2271 2272
    if (TraceMetadataChunkAllocation && Verbose) {
      gclog_or_tty->print(PTR_FORMAT " (" SIZE_FORMAT ") ",
                          humongous_chunks,
                          humongous_chunks->word_size());
    }
    assert(humongous_chunks->word_size() == (size_t)
           align_size_up(humongous_chunks->word_size(),
2273
                             smallest_chunk_size()),
2274
           err_msg("Humongous chunk size is wrong: word size " SIZE_FORMAT
2275
                   " granularity %d",
2276
                   humongous_chunks->word_size(), smallest_chunk_size()));
2277
    Metachunk* next_humongous_chunks = humongous_chunks->next();
2278
    humongous_chunks->container()->dec_container_count();
2279
    chunk_manager()->humongous_dictionary()->return_chunk(humongous_chunks);
2280
    humongous_chunks = next_humongous_chunks;
2281
  }
2282
  if (TraceMetadataChunkAllocation && Verbose) {
2283
    gclog_or_tty->cr();
2284
    gclog_or_tty->print_cr("updated dictionary count %d %s",
2285
                     chunk_manager()->humongous_dictionary()->total_count(),
2286 2287
                     chunk_size_name(HumongousIndex));
  }
2288
  chunk_manager()->slow_locked_verify();
2289 2290
}

2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
const char* SpaceManager::chunk_size_name(ChunkIndex index) const {
  switch (index) {
    case SpecializedIndex:
      return "Specialized";
    case SmallIndex:
      return "Small";
    case MediumIndex:
      return "Medium";
    case HumongousIndex:
      return "Humongous";
    default:
      return NULL;
  }
}

ChunkIndex ChunkManager::list_index(size_t size) {
  switch (size) {
    case SpecializedChunk:
      assert(SpecializedChunk == ClassSpecializedChunk,
             "Need branch for ClassSpecializedChunk");
      return SpecializedIndex;
    case SmallChunk:
    case ClassSmallChunk:
      return SmallIndex;
    case MediumChunk:
    case ClassMediumChunk:
      return MediumIndex;
    default:
2319
      assert(size > MediumChunk || size > ClassMediumChunk,
2320 2321 2322 2323 2324
             "Not a humongous chunk");
      return HumongousIndex;
  }
}

2325
void SpaceManager::deallocate(MetaWord* p, size_t word_size) {
2326
  assert_lock_strong(_lock);
2327
  size_t raw_word_size = get_raw_word_size(word_size);
2328
  size_t min_size = TreeChunk<Metablock, FreeList<Metablock> >::min_size();
2329
  assert(raw_word_size >= min_size,
2330
         err_msg("Should not deallocate dark matter " SIZE_FORMAT "<" SIZE_FORMAT, word_size, min_size));
2331
  block_freelists()->return_block(p, raw_word_size);
2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343
}

// Adds a chunk to the list of chunks in use.
void SpaceManager::add_chunk(Metachunk* new_chunk, bool make_current) {

  assert(new_chunk != NULL, "Should not be NULL");
  assert(new_chunk->next() == NULL, "Should not be on a list");

  new_chunk->reset_empty();

  // Find the correct list and and set the current
  // chunk for that list.
2344
  ChunkIndex index = ChunkManager::list_index(new_chunk->word_size());
2345

2346
  if (index != HumongousIndex) {
2347
    retire_current_chunk();
2348
    set_current_chunk(new_chunk);
2349 2350 2351
    new_chunk->set_next(chunks_in_use(index));
    set_chunks_in_use(index, new_chunk);
  } else {
2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365
    // For null class loader data and DumpSharedSpaces, the first chunk isn't
    // small, so small will be null.  Link this first chunk as the current
    // chunk.
    if (make_current) {
      // Set as the current chunk but otherwise treat as a humongous chunk.
      set_current_chunk(new_chunk);
    }
    // Link at head.  The _current_chunk only points to a humongous chunk for
    // the null class loader metaspace (class and data virtual space managers)
    // any humongous chunks so will not point to the tail
    // of the humongous chunks list.
    new_chunk->set_next(chunks_in_use(HumongousIndex));
    set_chunks_in_use(HumongousIndex, new_chunk);

2366
    assert(new_chunk->word_size() > medium_chunk_size(), "List inconsistency");
2367 2368
  }

2369 2370 2371
  // Add to the running sum of capacity
  inc_size_metrics(new_chunk->word_size());

2372 2373 2374 2375 2376
  assert(new_chunk->is_empty(), "Not ready for reuse");
  if (TraceMetadataChunkAllocation && Verbose) {
    gclog_or_tty->print("SpaceManager::add_chunk: %d) ",
                        sum_count_in_chunks_in_use());
    new_chunk->print_on(gclog_or_tty);
2377
    chunk_manager()->locked_print_free_chunks(gclog_or_tty);
2378 2379 2380
  }
}

2381 2382 2383
void SpaceManager::retire_current_chunk() {
  if (current_chunk() != NULL) {
    size_t remaining_words = current_chunk()->free_word_size();
2384
    if (remaining_words >= TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
2385 2386 2387 2388 2389 2390
      block_freelists()->return_block(current_chunk()->allocate(remaining_words), remaining_words);
      inc_used_metrics(remaining_words);
    }
  }
}

2391 2392
Metachunk* SpaceManager::get_new_chunk(size_t word_size,
                                       size_t grow_chunks_by_words) {
2393 2394
  // Get a chunk from the chunk freelist
  Metachunk* next = chunk_manager()->chunk_freelist_allocate(grow_chunks_by_words);
2395

2396 2397 2398 2399 2400
  if (next == NULL) {
    next = vs_list()->get_new_chunk(word_size,
                                    grow_chunks_by_words,
                                    medium_chunk_bunch());
  }
2401

S
stefank 已提交
2402
  if (TraceMetadataHumongousAllocation && next != NULL &&
2403
      SpaceManager::is_humongous(next->word_size())) {
S
stefank 已提交
2404 2405
    gclog_or_tty->print_cr("  new humongous chunk word size "
                           PTR_FORMAT, next->word_size());
2406 2407 2408 2409 2410
  }

  return next;
}

2411 2412 2413
MetaWord* SpaceManager::allocate(size_t word_size) {
  MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);

2414
  size_t raw_word_size = get_raw_word_size(word_size);
2415
  BlockFreelist* fl =  block_freelists();
2416
  MetaWord* p = NULL;
2417 2418 2419 2420 2421
  // Allocation from the dictionary is expensive in the sense that
  // the dictionary has to be searched for a size.  Don't allocate
  // from the dictionary until it starts to get fat.  Is this
  // a reasonable policy?  Maybe an skinny dictionary is fast enough
  // for allocations.  Do some profiling.  JJJ
2422 2423
  if (fl->total_size() > allocation_from_dictionary_limit) {
    p = fl->get_block(raw_word_size);
2424
  }
2425 2426
  if (p == NULL) {
    p = allocate_work(raw_word_size);
2427 2428
  }

2429
  return p;
2430 2431 2432 2433
}

// Returns the address of spaced allocated for "word_size".
// This methods does not know about blocks (Metablocks)
2434
MetaWord* SpaceManager::allocate_work(size_t word_size) {
2435 2436 2437 2438 2439 2440 2441
  assert_lock_strong(_lock);
#ifdef ASSERT
  if (Metadebug::test_metadata_failure()) {
    return NULL;
  }
#endif
  // Is there space in the current chunk?
2442
  MetaWord* result = NULL;
2443 2444 2445 2446 2447 2448

  // For DumpSharedSpaces, only allocate out of the current chunk which is
  // never null because we gave it the size we wanted.   Caller reports out
  // of memory if this returns null.
  if (DumpSharedSpaces) {
    assert(current_chunk() != NULL, "should never happen");
2449
    inc_used_metrics(word_size);
2450 2451
    return current_chunk()->allocate(word_size); // caller handles null result
  }
2452

2453 2454 2455 2456 2457 2458 2459
  if (current_chunk() != NULL) {
    result = current_chunk()->allocate(word_size);
  }

  if (result == NULL) {
    result = grow_and_allocate(word_size);
  }
2460 2461

  if (result != NULL) {
2462
    inc_used_metrics(word_size);
2463 2464
    assert(result != (MetaWord*) chunks_in_use(MediumIndex),
           "Head of the list is being allocated");
2465 2466 2467 2468 2469 2470 2471 2472 2473
  }

  return result;
}

void SpaceManager::verify() {
  // If there are blocks in the dictionary, then
  // verfication of chunks does not work since
  // being in the dictionary alters a chunk.
2474
  if (block_freelists()->total_size() == 0) {
2475
    for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
2476 2477 2478
      Metachunk* curr = chunks_in_use(i);
      while (curr != NULL) {
        curr->verify();
2479
        verify_chunk_size(curr);
2480 2481 2482 2483 2484 2485
        curr = curr->next();
      }
    }
  }
}

2486 2487
void SpaceManager::verify_chunk_size(Metachunk* chunk) {
  assert(is_humongous(chunk->word_size()) ||
2488 2489 2490
         chunk->word_size() == medium_chunk_size() ||
         chunk->word_size() == small_chunk_size() ||
         chunk->word_size() == specialized_chunk_size(),
2491 2492 2493 2494
         "Chunk size is wrong");
  return;
}

2495
#ifdef ASSERT
2496
void SpaceManager::verify_allocated_blocks_words() {
2497
  // Verification is only guaranteed at a safepoint.
2498 2499 2500
  assert(SafepointSynchronize::is_at_safepoint() || !Universe::is_fully_initialized(),
    "Verification can fail if the applications is running");
  assert(allocated_blocks_words() == sum_used_in_chunks_in_use(),
2501 2502
    err_msg("allocation total is not consistent " SIZE_FORMAT
            " vs " SIZE_FORMAT,
2503
            allocated_blocks_words(), sum_used_in_chunks_in_use()));
2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515
}

#endif

void SpaceManager::dump(outputStream* const out) const {
  size_t curr_total = 0;
  size_t waste = 0;
  uint i = 0;
  size_t used = 0;
  size_t capacity = 0;

  // Add up statistics for all chunks in this SpaceManager.
2516
  for (ChunkIndex index = ZeroIndex;
2517
       index < NumberOfInUseLists;
2518 2519 2520 2521 2522 2523 2524 2525
       index = next_chunk_index(index)) {
    for (Metachunk* curr = chunks_in_use(index);
         curr != NULL;
         curr = curr->next()) {
      out->print("%d) ", i++);
      curr->print_on(out);
      curr_total += curr->word_size();
      used += curr->used_word_size();
2526
      capacity += curr->word_size();
2527 2528 2529 2530
      waste += curr->free_word_size() + curr->overhead();;
    }
  }

S
stefank 已提交
2531 2532 2533 2534
  if (TraceMetadataChunkAllocation && Verbose) {
    block_freelists()->print_on(out);
  }

2535
  size_t free = current_chunk() == NULL ? 0 : current_chunk()->free_word_size();
2536 2537 2538 2539 2540 2541 2542 2543
  // Free space isn't wasted.
  waste -= free;

  out->print_cr("total of all chunks "  SIZE_FORMAT " used " SIZE_FORMAT
                " free " SIZE_FORMAT " capacity " SIZE_FORMAT
                " waste " SIZE_FORMAT, curr_total, used, free, capacity, waste);
}

2544
#ifndef PRODUCT
2545
void SpaceManager::mangle_freed_chunks() {
2546
  for (ChunkIndex index = ZeroIndex;
2547
       index < NumberOfInUseLists;
2548 2549 2550 2551 2552 2553 2554 2555
       index = next_chunk_index(index)) {
    for (Metachunk* curr = chunks_in_use(index);
         curr != NULL;
         curr = curr->next()) {
      curr->mangle();
    }
  }
}
2556
#endif // PRODUCT
2557 2558 2559

// MetaspaceAux

2560

2561 2562
size_t MetaspaceAux::_capacity_words[] = {0, 0};
size_t MetaspaceAux::_used_words[] = {0, 0};
2563

2564 2565 2566 2567 2568
size_t MetaspaceAux::free_bytes(Metaspace::MetadataType mdtype) {
  VirtualSpaceList* list = Metaspace::get_space_list(mdtype);
  return list == NULL ? 0 : list->free_bytes();
}

2569
size_t MetaspaceAux::free_bytes() {
2570
  return free_bytes(Metaspace::ClassType) + free_bytes(Metaspace::NonClassType);
2571 2572
}

2573
void MetaspaceAux::dec_capacity(Metaspace::MetadataType mdtype, size_t words) {
2574
  assert_lock_strong(SpaceManager::expand_lock());
2575
  assert(words <= capacity_words(mdtype),
2576
    err_msg("About to decrement below 0: words " SIZE_FORMAT
2577 2578 2579
            " is greater than _capacity_words[%u] " SIZE_FORMAT,
            words, mdtype, capacity_words(mdtype)));
  _capacity_words[mdtype] -= words;
2580 2581
}

2582
void MetaspaceAux::inc_capacity(Metaspace::MetadataType mdtype, size_t words) {
2583 2584
  assert_lock_strong(SpaceManager::expand_lock());
  // Needs to be atomic
2585
  _capacity_words[mdtype] += words;
2586 2587
}

2588
void MetaspaceAux::dec_used(Metaspace::MetadataType mdtype, size_t words) {
2589
  assert(words <= used_words(mdtype),
2590
    err_msg("About to decrement below 0: words " SIZE_FORMAT
2591 2592
            " is greater than _used_words[%u] " SIZE_FORMAT,
            words, mdtype, used_words(mdtype)));
2593 2594 2595 2596 2597
  // For CMS deallocation of the Metaspaces occurs during the
  // sweep which is a concurrent phase.  Protection by the expand_lock()
  // is not enough since allocation is on a per Metaspace basis
  // and protected by the Metaspace lock.
  jlong minus_words = (jlong) - (jlong) words;
2598
  Atomic::add_ptr(minus_words, &_used_words[mdtype]);
2599 2600
}

2601
void MetaspaceAux::inc_used(Metaspace::MetadataType mdtype, size_t words) {
2602
  // _used_words tracks allocations for
2603 2604 2605
  // each piece of metadata.  Those allocations are
  // generally done concurrently by different application
  // threads so must be done atomically.
2606
  Atomic::add_ptr(words, &_used_words[mdtype]);
2607 2608 2609
}

size_t MetaspaceAux::used_bytes_slow(Metaspace::MetadataType mdtype) {
2610 2611 2612 2613
  size_t used = 0;
  ClassLoaderDataGraphMetaspaceIterator iter;
  while (iter.repeat()) {
    Metaspace* msp = iter.get_next();
2614
    // Sum allocated_blocks_words for each metaspace
2615
    if (msp != NULL) {
2616
      used += msp->used_words_slow(mdtype);
2617 2618 2619 2620 2621
    }
  }
  return used * BytesPerWord;
}

E
ehelin 已提交
2622
size_t MetaspaceAux::free_bytes_slow(Metaspace::MetadataType mdtype) {
2623 2624 2625 2626 2627
  size_t free = 0;
  ClassLoaderDataGraphMetaspaceIterator iter;
  while (iter.repeat()) {
    Metaspace* msp = iter.get_next();
    if (msp != NULL) {
E
ehelin 已提交
2628
      free += msp->free_words_slow(mdtype);
2629 2630 2631 2632 2633
    }
  }
  return free * BytesPerWord;
}

2634
size_t MetaspaceAux::capacity_bytes_slow(Metaspace::MetadataType mdtype) {
2635 2636 2637
  if ((mdtype == Metaspace::ClassType) && !Metaspace::using_class_space()) {
    return 0;
  }
2638 2639 2640
  // Don't count the space in the freelists.  That space will be
  // added to the capacity calculation as needed.
  size_t capacity = 0;
2641 2642 2643 2644
  ClassLoaderDataGraphMetaspaceIterator iter;
  while (iter.repeat()) {
    Metaspace* msp = iter.get_next();
    if (msp != NULL) {
2645
      capacity += msp->capacity_words_slow(mdtype);
2646 2647 2648 2649 2650
    }
  }
  return capacity * BytesPerWord;
}

E
ehelin 已提交
2651 2652
size_t MetaspaceAux::capacity_bytes_slow() {
#ifdef PRODUCT
2653
  // Use capacity_bytes() in PRODUCT instead of this function.
E
ehelin 已提交
2654 2655 2656 2657
  guarantee(false, "Should not call capacity_bytes_slow() in the PRODUCT");
#endif
  size_t class_capacity = capacity_bytes_slow(Metaspace::ClassType);
  size_t non_class_capacity = capacity_bytes_slow(Metaspace::NonClassType);
2658 2659
  assert(capacity_bytes() == class_capacity + non_class_capacity,
      err_msg("bad accounting: capacity_bytes() " SIZE_FORMAT
E
ehelin 已提交
2660 2661
        " class_capacity + non_class_capacity " SIZE_FORMAT
        " class_capacity " SIZE_FORMAT " non_class_capacity " SIZE_FORMAT,
2662
        capacity_bytes(), class_capacity + non_class_capacity,
E
ehelin 已提交
2663 2664 2665 2666 2667 2668
        class_capacity, non_class_capacity));

  return class_capacity + non_class_capacity;
}

size_t MetaspaceAux::reserved_bytes(Metaspace::MetadataType mdtype) {
2669
  VirtualSpaceList* list = Metaspace::get_space_list(mdtype);
2670 2671 2672 2673 2674 2675
  return list == NULL ? 0 : list->reserved_bytes();
}

size_t MetaspaceAux::committed_bytes(Metaspace::MetadataType mdtype) {
  VirtualSpaceList* list = Metaspace::get_space_list(mdtype);
  return list == NULL ? 0 : list->committed_bytes();
2676 2677
}

E
ehelin 已提交
2678
size_t MetaspaceAux::min_chunk_size_words() { return Metaspace::first_chunk_word_size(); }
2679

E
ehelin 已提交
2680
size_t MetaspaceAux::free_chunks_total_words(Metaspace::MetadataType mdtype) {
2681 2682
  ChunkManager* chunk_manager = Metaspace::get_chunk_manager(mdtype);
  if (chunk_manager == NULL) {
2683 2684
    return 0;
  }
2685 2686
  chunk_manager->slow_verify();
  return chunk_manager->free_chunks_total_words();
2687 2688
}

E
ehelin 已提交
2689 2690
size_t MetaspaceAux::free_chunks_total_bytes(Metaspace::MetadataType mdtype) {
  return free_chunks_total_words(mdtype) * BytesPerWord;
2691 2692
}

E
ehelin 已提交
2693 2694 2695
size_t MetaspaceAux::free_chunks_total_words() {
  return free_chunks_total_words(Metaspace::ClassType) +
         free_chunks_total_words(Metaspace::NonClassType);
2696 2697
}

E
ehelin 已提交
2698 2699
size_t MetaspaceAux::free_chunks_total_bytes() {
  return free_chunks_total_words() * BytesPerWord;
2700 2701
}

2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714
bool MetaspaceAux::has_chunk_free_list(Metaspace::MetadataType mdtype) {
  return Metaspace::get_chunk_manager(mdtype) != NULL;
}

MetaspaceChunkFreeListSummary MetaspaceAux::chunk_free_list_summary(Metaspace::MetadataType mdtype) {
  if (!has_chunk_free_list(mdtype)) {
    return MetaspaceChunkFreeListSummary();
  }

  const ChunkManager* cm = Metaspace::get_chunk_manager(mdtype);
  return cm->chunk_free_list_summary();
}

2715 2716 2717 2718 2719
void MetaspaceAux::print_metaspace_change(size_t prev_metadata_used) {
  gclog_or_tty->print(", [Metaspace:");
  if (PrintGCDetails && Verbose) {
    gclog_or_tty->print(" "  SIZE_FORMAT
                        "->" SIZE_FORMAT
2720
                        "("  SIZE_FORMAT ")",
2721
                        prev_metadata_used,
2722
                        used_bytes(),
E
ehelin 已提交
2723
                        reserved_bytes());
2724 2725 2726
  } else {
    gclog_or_tty->print(" "  SIZE_FORMAT "K"
                        "->" SIZE_FORMAT "K"
2727
                        "("  SIZE_FORMAT "K)",
E
ehelin 已提交
2728
                        prev_metadata_used/K,
2729
                        used_bytes()/K,
E
ehelin 已提交
2730
                        reserved_bytes()/K);
2731 2732 2733 2734 2735 2736 2737 2738 2739
  }

  gclog_or_tty->print("]");
}

// This is printed when PrintGCDetails
void MetaspaceAux::print_on(outputStream* out) {
  Metaspace::MetadataType nct = Metaspace::NonClassType;

2740 2741 2742 2743 2744
  out->print_cr(" Metaspace       "
                "used "      SIZE_FORMAT "K, "
                "capacity "  SIZE_FORMAT "K, "
                "committed " SIZE_FORMAT "K, "
                "reserved "  SIZE_FORMAT "K",
2745 2746
                used_bytes()/K,
                capacity_bytes()/K,
2747 2748 2749
                committed_bytes()/K,
                reserved_bytes()/K);

2750 2751 2752
  if (Metaspace::using_class_space()) {
    Metaspace::MetadataType ct = Metaspace::ClassType;
    out->print_cr("  class space    "
2753 2754 2755 2756
                  "used "      SIZE_FORMAT "K, "
                  "capacity "  SIZE_FORMAT "K, "
                  "committed " SIZE_FORMAT "K, "
                  "reserved "  SIZE_FORMAT "K",
2757 2758
                  used_bytes(ct)/K,
                  capacity_bytes(ct)/K,
2759
                  committed_bytes(ct)/K,
E
ehelin 已提交
2760
                  reserved_bytes(ct)/K);
2761
  }
2762 2763 2764 2765 2766
}

// Print information for class space and data space separately.
// This is almost the same as above.
void MetaspaceAux::print_on(outputStream* out, Metaspace::MetadataType mdtype) {
E
ehelin 已提交
2767
  size_t free_chunks_capacity_bytes = free_chunks_total_bytes(mdtype);
2768 2769
  size_t capacity_bytes = capacity_bytes_slow(mdtype);
  size_t used_bytes = used_bytes_slow(mdtype);
E
ehelin 已提交
2770
  size_t free_bytes = free_bytes_slow(mdtype);
2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781
  size_t used_and_free = used_bytes + free_bytes +
                           free_chunks_capacity_bytes;
  out->print_cr("  Chunk accounting: used in chunks " SIZE_FORMAT
             "K + unused in chunks " SIZE_FORMAT "K  + "
             " capacity in free chunks " SIZE_FORMAT "K = " SIZE_FORMAT
             "K  capacity in allocated chunks " SIZE_FORMAT "K",
             used_bytes / K,
             free_bytes / K,
             free_chunks_capacity_bytes / K,
             used_and_free / K,
             capacity_bytes / K);
2782 2783
  // Accounting can only be correct if we got the values during a safepoint
  assert(!SafepointSynchronize::is_at_safepoint() || used_and_free == capacity_bytes, "Accounting is wrong");
2784 2785
}

2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811
// Print total fragmentation for class metaspaces
void MetaspaceAux::print_class_waste(outputStream* out) {
  assert(Metaspace::using_class_space(), "class metaspace not used");
  size_t cls_specialized_waste = 0, cls_small_waste = 0, cls_medium_waste = 0;
  size_t cls_specialized_count = 0, cls_small_count = 0, cls_medium_count = 0, cls_humongous_count = 0;
  ClassLoaderDataGraphMetaspaceIterator iter;
  while (iter.repeat()) {
    Metaspace* msp = iter.get_next();
    if (msp != NULL) {
      cls_specialized_waste += msp->class_vsm()->sum_waste_in_chunks_in_use(SpecializedIndex);
      cls_specialized_count += msp->class_vsm()->sum_count_in_chunks_in_use(SpecializedIndex);
      cls_small_waste += msp->class_vsm()->sum_waste_in_chunks_in_use(SmallIndex);
      cls_small_count += msp->class_vsm()->sum_count_in_chunks_in_use(SmallIndex);
      cls_medium_waste += msp->class_vsm()->sum_waste_in_chunks_in_use(MediumIndex);
      cls_medium_count += msp->class_vsm()->sum_count_in_chunks_in_use(MediumIndex);
      cls_humongous_count += msp->class_vsm()->sum_count_in_chunks_in_use(HumongousIndex);
    }
  }
  out->print_cr(" class: " SIZE_FORMAT " specialized(s) " SIZE_FORMAT ", "
                SIZE_FORMAT " small(s) " SIZE_FORMAT ", "
                SIZE_FORMAT " medium(s) " SIZE_FORMAT ", "
                "large count " SIZE_FORMAT,
                cls_specialized_count, cls_specialized_waste,
                cls_small_count, cls_small_waste,
                cls_medium_count, cls_medium_waste, cls_humongous_count);
}
2812

2813 2814
// Print total fragmentation for data and class metaspaces separately
void MetaspaceAux::print_waste(outputStream* out) {
2815 2816
  size_t specialized_waste = 0, small_waste = 0, medium_waste = 0;
  size_t specialized_count = 0, small_count = 0, medium_count = 0, humongous_count = 0;
2817 2818 2819 2820 2821

  ClassLoaderDataGraphMetaspaceIterator iter;
  while (iter.repeat()) {
    Metaspace* msp = iter.get_next();
    if (msp != NULL) {
2822 2823
      specialized_waste += msp->vsm()->sum_waste_in_chunks_in_use(SpecializedIndex);
      specialized_count += msp->vsm()->sum_count_in_chunks_in_use(SpecializedIndex);
2824
      small_waste += msp->vsm()->sum_waste_in_chunks_in_use(SmallIndex);
2825
      small_count += msp->vsm()->sum_count_in_chunks_in_use(SmallIndex);
2826
      medium_waste += msp->vsm()->sum_waste_in_chunks_in_use(MediumIndex);
2827
      medium_count += msp->vsm()->sum_count_in_chunks_in_use(MediumIndex);
2828
      humongous_count += msp->vsm()->sum_count_in_chunks_in_use(HumongousIndex);
2829 2830 2831
    }
  }
  out->print_cr("Total fragmentation waste (words) doesn't count free space");
2832 2833
  out->print_cr("  data: " SIZE_FORMAT " specialized(s) " SIZE_FORMAT ", "
                        SIZE_FORMAT " small(s) " SIZE_FORMAT ", "
2834 2835
                        SIZE_FORMAT " medium(s) " SIZE_FORMAT ", "
                        "large count " SIZE_FORMAT,
2836
             specialized_count, specialized_waste, small_count,
2837
             small_waste, medium_count, medium_waste, humongous_count);
2838 2839 2840
  if (Metaspace::using_class_space()) {
    print_class_waste(out);
  }
2841 2842 2843 2844 2845 2846 2847 2848 2849 2850
}

// Dump global metaspace things from the end of ClassLoaderDataGraph
void MetaspaceAux::dump(outputStream* out) {
  out->print_cr("All Metaspace:");
  out->print("data space: "); print_on(out, Metaspace::NonClassType);
  out->print("class space: "); print_on(out, Metaspace::ClassType);
  print_waste(out);
}

2851
void MetaspaceAux::verify_free_chunks() {
2852
  Metaspace::chunk_manager_metadata()->verify();
2853
  if (Metaspace::using_class_space()) {
2854
    Metaspace::chunk_manager_class()->verify();
2855
  }
2856 2857
}

2858 2859
void MetaspaceAux::verify_capacity() {
#ifdef ASSERT
2860
  size_t running_sum_capacity_bytes = capacity_bytes();
2861
  // For purposes of the running sum of capacity, verify against capacity
2862 2863
  size_t capacity_in_use_bytes = capacity_bytes_slow();
  assert(running_sum_capacity_bytes == capacity_in_use_bytes,
2864
    err_msg("capacity_words() * BytesPerWord " SIZE_FORMAT
2865 2866
            " capacity_bytes_slow()" SIZE_FORMAT,
            running_sum_capacity_bytes, capacity_in_use_bytes));
2867 2868 2869 2870
  for (Metaspace::MetadataType i = Metaspace::ClassType;
       i < Metaspace:: MetadataTypeCount;
       i = (Metaspace::MetadataType)(i + 1)) {
    size_t capacity_in_use_bytes = capacity_bytes_slow(i);
2871 2872
    assert(capacity_bytes(i) == capacity_in_use_bytes,
      err_msg("capacity_bytes(%u) " SIZE_FORMAT
2873
              " capacity_bytes_slow(%u)" SIZE_FORMAT,
2874
              i, capacity_bytes(i), i, capacity_in_use_bytes));
2875
  }
2876 2877 2878 2879 2880
#endif
}

void MetaspaceAux::verify_used() {
#ifdef ASSERT
2881
  size_t running_sum_used_bytes = used_bytes();
2882
  // For purposes of the running sum of used, verify against used
2883
  size_t used_in_use_bytes = used_bytes_slow();
2884 2885
  assert(used_bytes() == used_in_use_bytes,
    err_msg("used_bytes() " SIZE_FORMAT
2886
            " used_bytes_slow()" SIZE_FORMAT,
2887
            used_bytes(), used_in_use_bytes));
2888 2889 2890 2891
  for (Metaspace::MetadataType i = Metaspace::ClassType;
       i < Metaspace:: MetadataTypeCount;
       i = (Metaspace::MetadataType)(i + 1)) {
    size_t used_in_use_bytes = used_bytes_slow(i);
2892 2893
    assert(used_bytes(i) == used_in_use_bytes,
      err_msg("used_bytes(%u) " SIZE_FORMAT
2894
              " used_bytes_slow(%u)" SIZE_FORMAT,
2895
              i, used_bytes(i), i, used_in_use_bytes));
2896
  }
2897 2898 2899 2900 2901 2902 2903 2904 2905
#endif
}

void MetaspaceAux::verify_metrics() {
  verify_capacity();
  verify_used();
}


2906 2907 2908
// Metaspace methods

size_t Metaspace::_first_chunk_word_size = 0;
2909
size_t Metaspace::_first_class_chunk_word_size = 0;
2910

2911 2912 2913
size_t Metaspace::_commit_alignment = 0;
size_t Metaspace::_reserve_alignment = 0;

2914 2915
Metaspace::Metaspace(Mutex* lock, MetaspaceType type) {
  initialize(lock, type);
2916 2917 2918 2919
}

Metaspace::~Metaspace() {
  delete _vsm;
2920 2921 2922
  if (using_class_space()) {
    delete _class_vsm;
  }
2923 2924 2925 2926 2927
}

VirtualSpaceList* Metaspace::_space_list = NULL;
VirtualSpaceList* Metaspace::_class_space_list = NULL;

2928 2929 2930
ChunkManager* Metaspace::_chunk_manager_metadata = NULL;
ChunkManager* Metaspace::_chunk_manager_class = NULL;

2931 2932
#define VIRTUALSPACEMULTIPLIER 2

2933
#ifdef _LP64
2934 2935
static const uint64_t UnscaledClassSpaceMax = (uint64_t(max_juint) + 1);

2936 2937 2938 2939 2940 2941 2942
void Metaspace::set_narrow_klass_base_and_shift(address metaspace_base, address cds_base) {
  // Figure out the narrow_klass_base and the narrow_klass_shift.  The
  // narrow_klass_base is the lower of the metaspace base and the cds base
  // (if cds is enabled).  The narrow_klass_shift depends on the distance
  // between the lower base and higher address.
  address lower_base;
  address higher_address;
2943
#if INCLUDE_CDS
2944 2945
  if (UseSharedSpaces) {
    higher_address = MAX2((address)(cds_base + FileMapInfo::shared_spaces_size()),
2946
                          (address)(metaspace_base + compressed_class_space_size()));
2947
    lower_base = MIN2(metaspace_base, cds_base);
2948 2949 2950
  } else
#endif
  {
2951
    higher_address = metaspace_base + compressed_class_space_size();
2952
    lower_base = metaspace_base;
2953 2954 2955 2956 2957 2958

    uint64_t klass_encoding_max = UnscaledClassSpaceMax << LogKlassAlignmentInBytes;
    // If compressed class space fits in lower 32G, we don't need a base.
    if (higher_address <= (address)klass_encoding_max) {
      lower_base = 0; // effectively lower base is zero.
    }
2959
  }
2960

2961
  Universe::set_narrow_klass_base(lower_base);
2962

2963
  if ((uint64_t)(higher_address - lower_base) <= UnscaledClassSpaceMax) {
2964 2965 2966 2967 2968 2969 2970
    Universe::set_narrow_klass_shift(0);
  } else {
    assert(!UseSharedSpaces, "Cannot shift with UseSharedSpaces");
    Universe::set_narrow_klass_shift(LogKlassAlignmentInBytes);
  }
}

2971
#if INCLUDE_CDS
2972 2973 2974 2975
// Return TRUE if the specified metaspace_base and cds_base are close enough
// to work with compressed klass pointers.
bool Metaspace::can_use_cds_with_metaspace_addr(char* metaspace_base, address cds_base) {
  assert(cds_base != 0 && UseSharedSpaces, "Only use with CDS");
2976
  assert(UseCompressedClassPointers, "Only use with CompressedKlassPtrs");
2977 2978
  address lower_base = MIN2((address)metaspace_base, cds_base);
  address higher_address = MAX2((address)(cds_base + FileMapInfo::shared_spaces_size()),
2979
                                (address)(metaspace_base + compressed_class_space_size()));
2980
  return ((uint64_t)(higher_address - lower_base) <= UnscaledClassSpaceMax);
2981
}
2982
#endif
2983 2984 2985 2986

// Try to allocate the metaspace at the requested addr.
void Metaspace::allocate_metaspace_compressed_klass_ptrs(char* requested_addr, address cds_base) {
  assert(using_class_space(), "called improperly");
2987
  assert(UseCompressedClassPointers, "Only use with CompressedKlassPtrs");
2988
  assert(compressed_class_space_size() < KlassEncodingMetaspaceMax,
2989
         "Metaspace size is too big");
2990 2991 2992
  assert_is_ptr_aligned(requested_addr, _reserve_alignment);
  assert_is_ptr_aligned(cds_base, _reserve_alignment);
  assert_is_size_aligned(compressed_class_space_size(), _reserve_alignment);
2993 2994 2995

  // Don't use large pages for the class space.
  bool large_pages = false;
2996

2997
  ReservedSpace metaspace_rs = ReservedSpace(compressed_class_space_size(),
2998 2999 3000
                                             _reserve_alignment,
                                             large_pages,
                                             requested_addr, 0);
3001
  if (!metaspace_rs.is_reserved()) {
3002
#if INCLUDE_CDS
3003
    if (UseSharedSpaces) {
3004 3005
      size_t increment = align_size_up(1*G, _reserve_alignment);

3006 3007 3008 3009
      // Keep trying to allocate the metaspace, increasing the requested_addr
      // by 1GB each time, until we reach an address that will no longer allow
      // use of CDS with compressed klass pointers.
      char *addr = requested_addr;
3010 3011 3012
      while (!metaspace_rs.is_reserved() && (addr + increment > addr) &&
             can_use_cds_with_metaspace_addr(addr + increment, cds_base)) {
        addr = addr + increment;
3013
        metaspace_rs = ReservedSpace(compressed_class_space_size(),
3014
                                     _reserve_alignment, large_pages, addr, 0);
3015 3016
      }
    }
3017
#endif
3018 3019
    // If no successful allocation then try to allocate the space anywhere.  If
    // that fails then OOM doom.  At this point we cannot try allocating the
3020 3021 3022
    // metaspace as if UseCompressedClassPointers is off because too much
    // initialization has happened that depends on UseCompressedClassPointers.
    // So, UseCompressedClassPointers cannot be turned off at this point.
3023
    if (!metaspace_rs.is_reserved()) {
3024
      metaspace_rs = ReservedSpace(compressed_class_space_size(),
3025
                                   _reserve_alignment, large_pages);
3026 3027
      if (!metaspace_rs.is_reserved()) {
        vm_exit_during_initialization(err_msg("Could not allocate metaspace: %d bytes",
3028
                                              compressed_class_space_size()));
3029 3030 3031 3032 3033 3034 3035
      }
    }
  }

  // If we got here then the metaspace got allocated.
  MemTracker::record_virtual_memory_type((address)metaspace_rs.base(), mtClass);

3036
#if INCLUDE_CDS
3037 3038 3039 3040 3041
  // Verify that we can use shared spaces.  Otherwise, turn off CDS.
  if (UseSharedSpaces && !can_use_cds_with_metaspace_addr(metaspace_rs.base(), cds_base)) {
    FileMapInfo::stop_sharing_and_unmap(
        "Could not allocate metaspace at a compatible address");
  }
3042
#endif
3043 3044 3045 3046 3047 3048 3049 3050
  set_narrow_klass_base_and_shift((address)metaspace_rs.base(),
                                  UseSharedSpaces ? (address)cds_base : 0);

  initialize_class_space(metaspace_rs);

  if (PrintCompressedOopsMode || (PrintMiscellaneous && Verbose)) {
    gclog_or_tty->print_cr("Narrow klass base: " PTR_FORMAT ", Narrow klass shift: " SIZE_FORMAT,
                            Universe::narrow_klass_base(), Universe::narrow_klass_shift());
3051 3052
    gclog_or_tty->print_cr("Compressed class space size: " SIZE_FORMAT " Address: " PTR_FORMAT " Req Addr: " PTR_FORMAT,
                           compressed_class_space_size(), metaspace_rs.base(), requested_addr);
3053 3054 3055
  }
}

3056
// For UseCompressedClassPointers the class space is reserved above the top of
3057 3058 3059
// the Java heap.  The argument passed in is at the base of the compressed space.
void Metaspace::initialize_class_space(ReservedSpace rs) {
  // The reserved space size may be bigger because of alignment, esp with UseLargePages
3060 3061
  assert(rs.size() >= CompressedClassSpaceSize,
         err_msg(SIZE_FORMAT " != " UINTX_FORMAT, rs.size(), CompressedClassSpaceSize));
3062 3063
  assert(using_class_space(), "Must be using class space");
  _class_space_list = new VirtualSpaceList(rs);
3064
  _chunk_manager_class = new ChunkManager(SpecializedChunk, ClassSmallChunk, ClassMediumChunk);
3065 3066 3067 3068

  if (!_class_space_list->initialization_succeeded()) {
    vm_exit_during_initialization("Failed to setup compressed class space virtual space list.");
  }
3069 3070 3071 3072
}

#endif

3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094
void Metaspace::ergo_initialize() {
  if (DumpSharedSpaces) {
    // Using large pages when dumping the shared archive is currently not implemented.
    FLAG_SET_ERGO(bool, UseLargePagesInMetaspace, false);
  }

  size_t page_size = os::vm_page_size();
  if (UseLargePages && UseLargePagesInMetaspace) {
    page_size = os::large_page_size();
  }

  _commit_alignment  = page_size;
  _reserve_alignment = MAX2(page_size, (size_t)os::vm_allocation_granularity());

  // Do not use FLAG_SET_ERGO to update MaxMetaspaceSize, since this will
  // override if MaxMetaspaceSize was set on the command line or not.
  // This information is needed later to conform to the specification of the
  // java.lang.management.MemoryUsage API.
  //
  // Ideally, we would be able to set the default value of MaxMetaspaceSize in
  // globals.hpp to the aligned value, but this is not possible, since the
  // alignment depends on other flags being parsed.
3095
  MaxMetaspaceSize = align_size_down_bounded(MaxMetaspaceSize, _reserve_alignment);
3096 3097 3098 3099 3100

  if (MetaspaceSize > MaxMetaspaceSize) {
    MetaspaceSize = MaxMetaspaceSize;
  }

3101
  MetaspaceSize = align_size_down_bounded(MetaspaceSize, _commit_alignment);
3102 3103 3104 3105 3106 3107 3108

  assert(MetaspaceSize <= MaxMetaspaceSize, "MetaspaceSize should be limited by MaxMetaspaceSize");

  if (MetaspaceSize < 256*K) {
    vm_exit_during_initialization("Too small initial Metaspace size");
  }

3109 3110
  MinMetaspaceExpansion = align_size_down_bounded(MinMetaspaceExpansion, _commit_alignment);
  MaxMetaspaceExpansion = align_size_down_bounded(MaxMetaspaceExpansion, _commit_alignment);
3111

3112
  CompressedClassSpaceSize = align_size_down_bounded(CompressedClassSpaceSize, _reserve_alignment);
3113
  set_compressed_class_space_size(CompressedClassSpaceSize);
3114 3115
}

3116
void Metaspace::global_initialize() {
3117 3118
  MetaspaceGC::initialize();

3119 3120
  // Initialize the alignment for shared spaces.
  int max_alignment = os::vm_page_size();
3121 3122
  size_t cds_total = 0;

3123 3124 3125
  MetaspaceShared::set_max_alignment(max_alignment);

  if (DumpSharedSpaces) {
3126
#if INCLUDE_CDS
3127
    SharedReadOnlySize  = align_size_up(SharedReadOnlySize,  max_alignment);
3128
    SharedReadWriteSize = align_size_up(SharedReadWriteSize, max_alignment);
3129 3130
    SharedMiscDataSize  = align_size_up(SharedMiscDataSize,  max_alignment);
    SharedMiscCodeSize  = align_size_up(SharedMiscCodeSize,  max_alignment);
3131 3132 3133 3134

    // Initialize with the sum of the shared space sizes.  The read-only
    // and read write metaspace chunks will be allocated out of this and the
    // remainder is the misc code and data chunks.
3135
    cds_total = FileMapInfo::shared_spaces_size();
3136
    cds_total = align_size_up(cds_total, _reserve_alignment);
3137
    _space_list = new VirtualSpaceList(cds_total/wordSize);
3138
    _chunk_manager_metadata = new ChunkManager(SpecializedChunk, SmallChunk, MediumChunk);
3139

3140 3141 3142 3143
    if (!_space_list->initialization_succeeded()) {
      vm_exit_during_initialization("Unable to dump shared archive.", NULL);
    }

3144
#ifdef _LP64
3145
    if (cds_total + compressed_class_space_size() > UnscaledClassSpaceMax) {
3146 3147 3148
      vm_exit_during_initialization("Unable to dump shared archive.",
          err_msg("Size of archive (" SIZE_FORMAT ") + compressed class space ("
                  SIZE_FORMAT ") == total (" SIZE_FORMAT ") is larger than compressed "
3149 3150
                  "klass limit: " SIZE_FORMAT, cds_total, compressed_class_space_size(),
                  cds_total + compressed_class_space_size(), UnscaledClassSpaceMax));
3151 3152
    }

3153 3154
    // Set the compressed klass pointer base so that decoding of these pointers works
    // properly when creating the shared archive.
3155 3156
    assert(UseCompressedOops && UseCompressedClassPointers,
      "UseCompressedOops and UseCompressedClassPointers must be set");
3157 3158 3159 3160 3161 3162 3163
    Universe::set_narrow_klass_base((address)_space_list->current_virtual_space()->bottom());
    if (TraceMetavirtualspaceAllocation && Verbose) {
      gclog_or_tty->print_cr("Setting_narrow_klass_base to Address: " PTR_FORMAT,
                             _space_list->current_virtual_space()->bottom());
    }

    Universe::set_narrow_klass_shift(0);
3164 3165
#endif // _LP64
#endif // INCLUDE_CDS
3166
  } else {
3167
#if INCLUDE_CDS
3168 3169 3170
    // If using shared space, open the file that contains the shared space
    // and map in the memory before initializing the rest of metaspace (so
    // the addresses don't conflict)
3171
    address cds_address = NULL;
3172 3173 3174 3175 3176 3177 3178 3179
    if (UseSharedSpaces) {
      FileMapInfo* mapinfo = new FileMapInfo();

      // Open the shared archive file, read and validate the header. If
      // initialization fails, shared spaces [UseSharedSpaces] are
      // disabled and the file is closed.
      // Map in spaces now also
      if (mapinfo->initialize() && MetaspaceShared::map_shared_spaces(mapinfo)) {
3180 3181
        cds_total = FileMapInfo::shared_spaces_size();
        cds_address = (address)mapinfo->region_base(0);
3182 3183 3184 3185
      } else {
        assert(!mapinfo->is_open() && !UseSharedSpaces,
               "archive file not closed or shared spaces not disabled.");
      }
3186
    }
3187
#endif // INCLUDE_CDS
3188
#ifdef _LP64
3189
    // If UseCompressedClassPointers is set then allocate the metaspace area
3190 3191 3192
    // above the heap and above the CDS area (if it exists).
    if (using_class_space()) {
      if (UseSharedSpaces) {
3193
#if INCLUDE_CDS
3194 3195 3196
        char* cds_end = (char*)(cds_address + cds_total);
        cds_end = (char *)align_ptr_up(cds_end, _reserve_alignment);
        allocate_metaspace_compressed_klass_ptrs(cds_end, cds_address);
3197
#endif
3198
      } else {
3199 3200
        char* base = (char*)align_ptr_up(Universe::heap()->reserved_region().end(), _reserve_alignment);
        allocate_metaspace_compressed_klass_ptrs(base, 0);
3201
      }
3202
    }
3203
#endif // _LP64
3204

3205
    // Initialize these before initializing the VirtualSpaceList
3206
    _first_chunk_word_size = InitialBootClassLoaderMetaspaceSize / BytesPerWord;
3207 3208 3209 3210 3211
    _first_chunk_word_size = align_word_size_up(_first_chunk_word_size);
    // Make the first class chunk bigger than a medium chunk so it's not put
    // on the medium chunk list.   The next chunk will be small and progress
    // from there.  This size calculated by -version.
    _first_class_chunk_word_size = MIN2((size_t)MediumChunk*6,
3212
                                       (CompressedClassSpaceSize/BytesPerWord)*2);
3213
    _first_class_chunk_word_size = align_word_size_up(_first_class_chunk_word_size);
3214 3215
    // Arbitrarily set the initial virtual space to a multiple
    // of the boot class loader size.
3216 3217 3218
    size_t word_size = VIRTUALSPACEMULTIPLIER * _first_chunk_word_size;
    word_size = align_size_up(word_size, Metaspace::reserve_alignment_words());

3219 3220
    // Initialize the list of virtual spaces.
    _space_list = new VirtualSpaceList(word_size);
3221
    _chunk_manager_metadata = new ChunkManager(SpecializedChunk, SmallChunk, MediumChunk);
3222 3223 3224 3225

    if (!_space_list->initialization_succeeded()) {
      vm_exit_during_initialization("Unable to setup metadata virtual space list.", NULL);
    }
3226
  }
3227

3228
  _tracer = new MetaspaceTracer();
3229 3230
}

3231 3232 3233 3234
void Metaspace::post_initialize() {
  MetaspaceGC::post_initialize();
}

3235 3236 3237 3238 3239 3240 3241
Metachunk* Metaspace::get_initialization_chunk(MetadataType mdtype,
                                               size_t chunk_word_size,
                                               size_t chunk_bunch) {
  // Get a chunk from the chunk freelist
  Metachunk* chunk = get_chunk_manager(mdtype)->chunk_freelist_allocate(chunk_word_size);
  if (chunk != NULL) {
    return chunk;
3242
  }
3243

3244
  return get_space_list(mdtype)->get_new_chunk(chunk_word_size, chunk_word_size, chunk_bunch);
3245 3246
}

3247
void Metaspace::initialize(Mutex* lock, MetaspaceType type) {
3248 3249 3250

  assert(space_list() != NULL,
    "Metadata VirtualSpaceList has not been initialized");
3251 3252
  assert(chunk_manager_metadata() != NULL,
    "Metadata ChunkManager has not been initialized");
3253

3254
  _vsm = new SpaceManager(NonClassType, lock);
3255 3256 3257
  if (_vsm == NULL) {
    return;
  }
3258 3259
  size_t word_size;
  size_t class_word_size;
3260
  vsm()->get_initial_chunk_sizes(type, &word_size, &class_word_size);
3261

3262
  if (using_class_space()) {
3263 3264 3265 3266
  assert(class_space_list() != NULL,
    "Class VirtualSpaceList has not been initialized");
  assert(chunk_manager_class() != NULL,
    "Class ChunkManager has not been initialized");
3267

3268
    // Allocate SpaceManager for classes.
3269
    _class_vsm = new SpaceManager(ClassType, lock);
3270 3271 3272
    if (_class_vsm == NULL) {
      return;
    }
3273 3274 3275 3276 3277
  }

  MutexLockerEx cl(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);

  // Allocate chunk for metadata objects
3278 3279 3280
  Metachunk* new_chunk = get_initialization_chunk(NonClassType,
                                                  word_size,
                                                  vsm()->medium_chunk_bunch());
3281 3282 3283 3284 3285 3286 3287
  assert(!DumpSharedSpaces || new_chunk != NULL, "should have enough space for both chunks");
  if (new_chunk != NULL) {
    // Add to this manager's list of chunks in use and current_chunk().
    vsm()->add_chunk(new_chunk, true);
  }

  // Allocate chunk for class metadata objects
3288
  if (using_class_space()) {
3289 3290 3291
    Metachunk* class_chunk = get_initialization_chunk(ClassType,
                                                      class_word_size,
                                                      class_vsm()->medium_chunk_bunch());
3292 3293 3294
    if (class_chunk != NULL) {
      class_vsm()->add_chunk(class_chunk, true);
    }
3295
  }
3296 3297 3298

  _alloc_record_head = NULL;
  _alloc_record_tail = NULL;
3299 3300
}

3301 3302 3303 3304 3305
size_t Metaspace::align_word_size_up(size_t word_size) {
  size_t byte_size = word_size * wordSize;
  return ReservedSpace::allocation_align_size_up(byte_size) / wordSize;
}

3306 3307
MetaWord* Metaspace::allocate(size_t word_size, MetadataType mdtype) {
  // DumpSharedSpaces doesn't use class metadata area (yet)
3308
  // Also, don't use class_vsm() unless UseCompressedClassPointers is true.
3309
  if (is_class_space_allocation(mdtype)) {
3310
    return  class_vsm()->allocate(word_size);
3311
  } else {
3312
    return  vsm()->allocate(word_size);
3313 3314 3315
  }
}

3316
MetaWord* Metaspace::expand_and_allocate(size_t word_size, MetadataType mdtype) {
3317 3318 3319 3320
  size_t delta_bytes = MetaspaceGC::delta_capacity_until_GC(word_size * BytesPerWord);
  assert(delta_bytes > 0, "Must be");

  size_t after_inc = MetaspaceGC::inc_capacity_until_GC(delta_bytes);
3321 3322

  // capacity_until_GC might be updated concurrently, must calculate previous value.
3323 3324
  size_t before_inc = after_inc - delta_bytes;

3325 3326
  tracer()->report_gc_threshold(before_inc, after_inc,
                                MetaspaceGCThresholdUpdater::ExpandAndAllocate);
3327 3328
  if (PrintGCDetails && Verbose) {
    gclog_or_tty->print_cr("Increase capacity to GC from " SIZE_FORMAT
3329
        " to " SIZE_FORMAT, before_inc, after_inc);
3330
  }
3331

3332
  return allocate(word_size, mdtype);
3333 3334
}

3335 3336 3337 3338 3339 3340 3341
// Space allocated in the Metaspace.  This may
// be across several metadata virtual spaces.
char* Metaspace::bottom() const {
  assert(DumpSharedSpaces, "only useful and valid for dumping shared spaces");
  return (char*)vsm()->current_chunk()->bottom();
}

3342
size_t Metaspace::used_words_slow(MetadataType mdtype) const {
3343 3344 3345 3346 3347
  if (mdtype == ClassType) {
    return using_class_space() ? class_vsm()->sum_used_in_chunks_in_use() : 0;
  } else {
    return vsm()->sum_used_in_chunks_in_use();  // includes overhead!
  }
3348 3349
}

E
ehelin 已提交
3350
size_t Metaspace::free_words_slow(MetadataType mdtype) const {
3351 3352 3353 3354 3355
  if (mdtype == ClassType) {
    return using_class_space() ? class_vsm()->sum_free_in_chunks_in_use() : 0;
  } else {
    return vsm()->sum_free_in_chunks_in_use();
  }
3356 3357 3358 3359 3360 3361 3362
}

// Space capacity in the Metaspace.  It includes
// space in the list of chunks from which allocations
// have been made. Don't include space in the global freelist and
// in the space available in the dictionary which
// is already counted in some chunk.
3363
size_t Metaspace::capacity_words_slow(MetadataType mdtype) const {
3364 3365 3366 3367 3368
  if (mdtype == ClassType) {
    return using_class_space() ? class_vsm()->sum_capacity_in_chunks_in_use() : 0;
  } else {
    return vsm()->sum_capacity_in_chunks_in_use();
  }
3369 3370
}

3371 3372 3373 3374 3375 3376 3377 3378
size_t Metaspace::used_bytes_slow(MetadataType mdtype) const {
  return used_words_slow(mdtype) * BytesPerWord;
}

size_t Metaspace::capacity_bytes_slow(MetadataType mdtype) const {
  return capacity_words_slow(mdtype) * BytesPerWord;
}

3379 3380
void Metaspace::deallocate(MetaWord* ptr, size_t word_size, bool is_class) {
  if (SafepointSynchronize::is_at_safepoint()) {
3381 3382 3383 3384
    if (DumpSharedSpaces && PrintSharedSpaces) {
      record_deallocation(ptr, vsm()->get_raw_word_size(word_size));
    }

3385
    assert(Thread::current()->is_VM_thread(), "should be the VM thread");
3386
    // Don't take Heap_lock
3387
    MutexLockerEx ml(vsm()->lock(), Mutex::_no_safepoint_check_flag);
3388
    if (word_size < TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
3389 3390 3391 3392 3393 3394
      // Dark matter.  Too small for dictionary.
#ifdef ASSERT
      Copy::fill_to_words((HeapWord*)ptr, word_size, 0xf5f5f5f5);
#endif
      return;
    }
3395 3396
    if (is_class && using_class_space()) {
      class_vsm()->deallocate(ptr, word_size);
3397
    } else {
3398
      vsm()->deallocate(ptr, word_size);
3399 3400
    }
  } else {
3401
    MutexLockerEx ml(vsm()->lock(), Mutex::_no_safepoint_check_flag);
3402

3403
    if (word_size < TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
3404 3405 3406 3407 3408 3409
      // Dark matter.  Too small for dictionary.
#ifdef ASSERT
      Copy::fill_to_words((HeapWord*)ptr, word_size, 0xf5f5f5f5);
#endif
      return;
    }
3410
    if (is_class && using_class_space()) {
3411
      class_vsm()->deallocate(ptr, word_size);
3412
    } else {
3413
      vsm()->deallocate(ptr, word_size);
3414 3415 3416 3417
    }
  }
}

3418

3419
MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
3420
                              bool read_only, MetaspaceObj::Type type, TRAPS) {
3421 3422 3423 3424 3425 3426 3427
  if (HAS_PENDING_EXCEPTION) {
    assert(false, "Should not allocate with exception pending");
    return NULL;  // caller does a CHECK_NULL too
  }

  assert(loader_data != NULL, "Should never pass around a NULL loader_data. "
        "ClassLoaderData::the_null_class_loader_data() should have been used.");
3428

3429 3430 3431 3432
  // Allocate in metaspaces without taking out a lock, because it deadlocks
  // with the SymbolTable_lock.  Dumping is single threaded for now.  We'll have
  // to revisit this for application class data sharing.
  if (DumpSharedSpaces) {
3433 3434
    assert(type > MetaspaceObj::UnknownType && type < MetaspaceObj::_number_of_types, "sanity");
    Metaspace* space = read_only ? loader_data->ro_metaspace() : loader_data->rw_metaspace();
3435
    MetaWord* result = space->allocate(word_size, NonClassType);
3436 3437 3438
    if (result == NULL) {
      report_out_of_shared_space(read_only ? SharedReadOnly : SharedReadWrite);
    }
3439 3440 3441
    if (PrintSharedSpaces) {
      space->record_allocation(result, type, space->vsm()->get_raw_word_size(word_size));
    }
3442 3443 3444 3445 3446

    // Zero initialize.
    Copy::fill_to_aligned_words((HeapWord*)result, word_size, 0);

    return result;
3447 3448
  }

3449 3450 3451 3452
  MetadataType mdtype = (type == MetaspaceObj::ClassType) ? ClassType : NonClassType;

  // Try to allocate metadata.
  MetaWord* result = loader_data->metaspace_non_null()->allocate(word_size, mdtype);
3453 3454

  if (result == NULL) {
3455 3456
    tracer()->report_metaspace_allocation_failure(loader_data, word_size, type, mdtype);

3457 3458 3459
    // Allocation failed.
    if (is_init_completed()) {
      // Only start a GC if the bootstrapping has completed.
3460

3461 3462 3463
      // Try to clean out some memory and retry.
      result = Universe::heap()->collector_policy()->satisfy_failed_metadata_allocation(
          loader_data, word_size, mdtype);
3464 3465
    }
  }
3466 3467

  if (result == NULL) {
3468
    report_metadata_oome(loader_data, word_size, type, mdtype, CHECK_NULL);
3469 3470
  }

3471 3472 3473 3474
  // Zero initialize.
  Copy::fill_to_aligned_words((HeapWord*)result, word_size, 0);

  return result;
3475 3476
}

3477 3478 3479 3480 3481
size_t Metaspace::class_chunk_size(size_t word_size) {
  assert(using_class_space(), "Has to use class space");
  return class_vsm()->calc_chunk_size(word_size);
}

3482 3483 3484
void Metaspace::report_metadata_oome(ClassLoaderData* loader_data, size_t word_size, MetaspaceObj::Type type, MetadataType mdtype, TRAPS) {
  tracer()->report_metadata_oom(loader_data, word_size, type, mdtype);

3485 3486 3487 3488 3489 3490 3491 3492 3493 3494
  // If result is still null, we are out of memory.
  if (Verbose && TraceMetadataChunkAllocation) {
    gclog_or_tty->print_cr("Metaspace allocation failed for size "
        SIZE_FORMAT, word_size);
    if (loader_data->metaspace_or_null() != NULL) {
      loader_data->dump(gclog_or_tty);
    }
    MetaspaceAux::dump(gclog_or_tty);
  }

3495 3496 3497 3498 3499 3500 3501 3502 3503
  bool out_of_compressed_class_space = false;
  if (is_class_space_allocation(mdtype)) {
    Metaspace* metaspace = loader_data->metaspace_non_null();
    out_of_compressed_class_space =
      MetaspaceAux::committed_bytes(Metaspace::ClassType) +
      (metaspace->class_chunk_size(word_size) * BytesPerWord) >
      CompressedClassSpaceSize;
  }

3504
  // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support
3505 3506 3507
  const char* space_string = out_of_compressed_class_space ?
    "Compressed class space" : "Metaspace";

3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519
  report_java_out_of_memory(space_string);

  if (JvmtiExport::should_post_resource_exhausted()) {
    JvmtiExport::post_resource_exhausted(
        JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR,
        space_string);
  }

  if (!is_init_completed()) {
    vm_exit_during_initialization("OutOfMemoryError", space_string);
  }

3520
  if (out_of_compressed_class_space) {
3521 3522 3523 3524 3525 3526
    THROW_OOP(Universe::out_of_memory_error_class_metaspace());
  } else {
    THROW_OOP(Universe::out_of_memory_error_metaspace());
  }
}

3527 3528 3529 3530 3531 3532 3533 3534 3535 3536
const char* Metaspace::metadata_type_name(Metaspace::MetadataType mdtype) {
  switch (mdtype) {
    case Metaspace::ClassType: return "Class";
    case Metaspace::NonClassType: return "Metadata";
    default:
      assert(false, err_msg("Got bad mdtype: %d", (int) mdtype));
      return NULL;
  }
}

3537 3538 3539
void Metaspace::record_allocation(void* ptr, MetaspaceObj::Type type, size_t word_size) {
  assert(DumpSharedSpaces, "sanity");

3540 3541 3542
  int byte_size = (int)word_size * HeapWordSize;
  AllocRecord *rec = new AllocRecord((address)ptr, type, byte_size);

3543 3544
  if (_alloc_record_head == NULL) {
    _alloc_record_head = _alloc_record_tail = rec;
3545
  } else if (_alloc_record_tail->_ptr + _alloc_record_tail->_byte_size == (address)ptr) {
3546 3547
    _alloc_record_tail->_next = rec;
    _alloc_record_tail = rec;
3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583
  } else {
    // slow linear search, but this doesn't happen that often, and only when dumping
    for (AllocRecord *old = _alloc_record_head; old; old = old->_next) {
      if (old->_ptr == ptr) {
        assert(old->_type == MetaspaceObj::DeallocatedType, "sanity");
        int remain_bytes = old->_byte_size - byte_size;
        assert(remain_bytes >= 0, "sanity");
        old->_type = type;

        if (remain_bytes == 0) {
          delete(rec);
        } else {
          address remain_ptr = address(ptr) + byte_size;
          rec->_ptr = remain_ptr;
          rec->_byte_size = remain_bytes;
          rec->_type = MetaspaceObj::DeallocatedType;
          rec->_next = old->_next;
          old->_byte_size = byte_size;
          old->_next = rec;
        }
        return;
      }
    }
    assert(0, "reallocating a freed pointer that was not recorded");
  }
}

void Metaspace::record_deallocation(void* ptr, size_t word_size) {
  assert(DumpSharedSpaces, "sanity");

  for (AllocRecord *rec = _alloc_record_head; rec; rec = rec->_next) {
    if (rec->_ptr == ptr) {
      assert(rec->_byte_size == (int)word_size * HeapWordSize, "sanity");
      rec->_type = MetaspaceObj::DeallocatedType;
      return;
    }
3584
  }
3585 3586

  assert(0, "deallocating a pointer that was not recorded");
3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608
}

void Metaspace::iterate(Metaspace::AllocRecordClosure *closure) {
  assert(DumpSharedSpaces, "unimplemented for !DumpSharedSpaces");

  address last_addr = (address)bottom();

  for (AllocRecord *rec = _alloc_record_head; rec; rec = rec->_next) {
    address ptr = rec->_ptr;
    if (last_addr < ptr) {
      closure->doit(last_addr, MetaspaceObj::UnknownType, ptr - last_addr);
    }
    closure->doit(ptr, rec->_type, rec->_byte_size);
    last_addr = ptr + rec->_byte_size;
  }

  address top = ((address)bottom()) + used_bytes_slow(Metaspace::NonClassType);
  if (last_addr < top) {
    closure->doit(last_addr, MetaspaceObj::UnknownType, top - last_addr);
  }
}

3609 3610 3611 3612
void Metaspace::purge(MetadataType mdtype) {
  get_space_list(mdtype)->purge(get_chunk_manager(mdtype));
}

3613 3614 3615
void Metaspace::purge() {
  MutexLockerEx cl(SpaceManager::expand_lock(),
                   Mutex::_no_safepoint_check_flag);
3616
  purge(NonClassType);
3617
  if (using_class_space()) {
3618
    purge(ClassType);
3619
  }
3620 3621
}

3622 3623 3624
void Metaspace::print_on(outputStream* out) const {
  // Print both class virtual space counts and metaspace.
  if (Verbose) {
3625 3626
    vsm()->print_on(out);
    if (using_class_space()) {
3627
      class_vsm()->print_on(out);
3628
    }
3629 3630 3631
  }
}

3632
bool Metaspace::contains(const void* ptr) {
3633 3634
  if (UseSharedSpaces && MetaspaceShared::is_in_shared_space(ptr)) {
    return true;
3635
  }
3636 3637 3638 3639 3640 3641

  if (using_class_space() && get_space_list(ClassType)->contains(ptr)) {
     return true;
  }

  return get_space_list(NonClassType)->contains(ptr);
3642 3643 3644 3645
}

void Metaspace::verify() {
  vsm()->verify();
3646 3647 3648
  if (using_class_space()) {
    class_vsm()->verify();
  }
3649 3650 3651 3652 3653
}

void Metaspace::dump(outputStream* const out) const {
  out->print_cr("\nVirtual space manager: " INTPTR_FORMAT, vsm());
  vsm()->dump(out);
3654 3655 3656 3657
  if (using_class_space()) {
    out->print_cr("\nClass space manager: " INTPTR_FORMAT, class_vsm());
    class_vsm()->dump(out);
  }
3658
}
3659 3660 3661 3662 3663

/////////////// Unit tests ///////////////

#ifndef PRODUCT

3664
class TestMetaspaceAuxTest : AllStatic {
3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703
 public:
  static void test_reserved() {
    size_t reserved = MetaspaceAux::reserved_bytes();

    assert(reserved > 0, "assert");

    size_t committed  = MetaspaceAux::committed_bytes();
    assert(committed <= reserved, "assert");

    size_t reserved_metadata = MetaspaceAux::reserved_bytes(Metaspace::NonClassType);
    assert(reserved_metadata > 0, "assert");
    assert(reserved_metadata <= reserved, "assert");

    if (UseCompressedClassPointers) {
      size_t reserved_class    = MetaspaceAux::reserved_bytes(Metaspace::ClassType);
      assert(reserved_class > 0, "assert");
      assert(reserved_class < reserved, "assert");
    }
  }

  static void test_committed() {
    size_t committed = MetaspaceAux::committed_bytes();

    assert(committed > 0, "assert");

    size_t reserved  = MetaspaceAux::reserved_bytes();
    assert(committed <= reserved, "assert");

    size_t committed_metadata = MetaspaceAux::committed_bytes(Metaspace::NonClassType);
    assert(committed_metadata > 0, "assert");
    assert(committed_metadata <= committed, "assert");

    if (UseCompressedClassPointers) {
      size_t committed_class    = MetaspaceAux::committed_bytes(Metaspace::ClassType);
      assert(committed_class > 0, "assert");
      assert(committed_class < committed, "assert");
    }
  }

3704 3705 3706 3707 3708 3709 3710 3711 3712 3713
  static void test_virtual_space_list_large_chunk() {
    VirtualSpaceList* vs_list = new VirtualSpaceList(os::vm_allocation_granularity());
    MutexLockerEx cl(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);
    // A size larger than VirtualSpaceSize (256k) and add one page to make it _not_ be
    // vm_allocation_granularity aligned on Windows.
    size_t large_size = (size_t)(2*256*K + (os::vm_page_size()/BytesPerWord));
    large_size += (os::vm_page_size()/BytesPerWord);
    vs_list->get_new_chunk(large_size, large_size, 0);
  }

3714 3715 3716
  static void test() {
    test_reserved();
    test_committed();
3717
    test_virtual_space_list_large_chunk();
3718 3719 3720
  }
};

3721 3722
void TestMetaspaceAux_test() {
  TestMetaspaceAuxTest::test();
3723 3724
}

3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808
class TestVirtualSpaceNodeTest {
  static void chunk_up(size_t words_left, size_t& num_medium_chunks,
                                          size_t& num_small_chunks,
                                          size_t& num_specialized_chunks) {
    num_medium_chunks = words_left / MediumChunk;
    words_left = words_left % MediumChunk;

    num_small_chunks = words_left / SmallChunk;
    words_left = words_left % SmallChunk;
    // how many specialized chunks can we get?
    num_specialized_chunks = words_left / SpecializedChunk;
    assert(words_left % SpecializedChunk == 0, "should be nothing left");
  }

 public:
  static void test() {
    MutexLockerEx ml(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);
    const size_t vsn_test_size_words = MediumChunk  * 4;
    const size_t vsn_test_size_bytes = vsn_test_size_words * BytesPerWord;

    // The chunk sizes must be multiples of eachother, or this will fail
    STATIC_ASSERT(MediumChunk % SmallChunk == 0);
    STATIC_ASSERT(SmallChunk % SpecializedChunk == 0);

    { // No committed memory in VSN
      ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
      VirtualSpaceNode vsn(vsn_test_size_bytes);
      vsn.initialize();
      vsn.retire(&cm);
      assert(cm.sum_free_chunks_count() == 0, "did not commit any memory in the VSN");
    }

    { // All of VSN is committed, half is used by chunks
      ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
      VirtualSpaceNode vsn(vsn_test_size_bytes);
      vsn.initialize();
      vsn.expand_by(vsn_test_size_words, vsn_test_size_words);
      vsn.get_chunk_vs(MediumChunk);
      vsn.get_chunk_vs(MediumChunk);
      vsn.retire(&cm);
      assert(cm.sum_free_chunks_count() == 2, "should have been memory left for 2 medium chunks");
      assert(cm.sum_free_chunks() == 2*MediumChunk, "sizes should add up");
    }

    { // 4 pages of VSN is committed, some is used by chunks
      ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
      VirtualSpaceNode vsn(vsn_test_size_bytes);
      const size_t page_chunks = 4 * (size_t)os::vm_page_size() / BytesPerWord;
      assert(page_chunks < MediumChunk, "Test expects medium chunks to be at least 4*page_size");
      vsn.initialize();
      vsn.expand_by(page_chunks, page_chunks);
      vsn.get_chunk_vs(SmallChunk);
      vsn.get_chunk_vs(SpecializedChunk);
      vsn.retire(&cm);

      // committed - used = words left to retire
      const size_t words_left = page_chunks - SmallChunk - SpecializedChunk;

      size_t num_medium_chunks, num_small_chunks, num_spec_chunks;
      chunk_up(words_left, num_medium_chunks, num_small_chunks, num_spec_chunks);

      assert(num_medium_chunks == 0, "should not get any medium chunks");
      assert(cm.sum_free_chunks_count() == (num_small_chunks + num_spec_chunks), "should be space for 3 chunks");
      assert(cm.sum_free_chunks() == words_left, "sizes should add up");
    }

    { // Half of VSN is committed, a humongous chunk is used
      ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
      VirtualSpaceNode vsn(vsn_test_size_bytes);
      vsn.initialize();
      vsn.expand_by(MediumChunk * 2, MediumChunk * 2);
      vsn.get_chunk_vs(MediumChunk + SpecializedChunk); // Humongous chunks will be aligned up to MediumChunk + SpecializedChunk
      vsn.retire(&cm);

      const size_t words_left = MediumChunk * 2 - (MediumChunk + SpecializedChunk);
      size_t num_medium_chunks, num_small_chunks, num_spec_chunks;
      chunk_up(words_left, num_medium_chunks, num_small_chunks, num_spec_chunks);

      assert(num_medium_chunks == 0, "should not get any medium chunks");
      assert(cm.sum_free_chunks_count() == (num_small_chunks + num_spec_chunks), "should be space for 3 chunks");
      assert(cm.sum_free_chunks() == words_left, "sizes should add up");
    }

  }
3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879

#define assert_is_available_positive(word_size) \
  assert(vsn.is_available(word_size), \
    err_msg(#word_size ": " PTR_FORMAT " bytes were not available in " \
            "VirtualSpaceNode [" PTR_FORMAT ", " PTR_FORMAT ")", \
            (uintptr_t)(word_size * BytesPerWord), vsn.bottom(), vsn.end()));

#define assert_is_available_negative(word_size) \
  assert(!vsn.is_available(word_size), \
    err_msg(#word_size ": " PTR_FORMAT " bytes should not be available in " \
            "VirtualSpaceNode [" PTR_FORMAT ", " PTR_FORMAT ")", \
            (uintptr_t)(word_size * BytesPerWord), vsn.bottom(), vsn.end()));

  static void test_is_available_positive() {
    // Reserve some memory.
    VirtualSpaceNode vsn(os::vm_allocation_granularity());
    assert(vsn.initialize(), "Failed to setup VirtualSpaceNode");

    // Commit some memory.
    size_t commit_word_size = os::vm_allocation_granularity() / BytesPerWord;
    bool expanded = vsn.expand_by(commit_word_size, commit_word_size);
    assert(expanded, "Failed to commit");

    // Check that is_available accepts the committed size.
    assert_is_available_positive(commit_word_size);

    // Check that is_available accepts half the committed size.
    size_t expand_word_size = commit_word_size / 2;
    assert_is_available_positive(expand_word_size);
  }

  static void test_is_available_negative() {
    // Reserve some memory.
    VirtualSpaceNode vsn(os::vm_allocation_granularity());
    assert(vsn.initialize(), "Failed to setup VirtualSpaceNode");

    // Commit some memory.
    size_t commit_word_size = os::vm_allocation_granularity() / BytesPerWord;
    bool expanded = vsn.expand_by(commit_word_size, commit_word_size);
    assert(expanded, "Failed to commit");

    // Check that is_available doesn't accept a too large size.
    size_t two_times_commit_word_size = commit_word_size * 2;
    assert_is_available_negative(two_times_commit_word_size);
  }

  static void test_is_available_overflow() {
    // Reserve some memory.
    VirtualSpaceNode vsn(os::vm_allocation_granularity());
    assert(vsn.initialize(), "Failed to setup VirtualSpaceNode");

    // Commit some memory.
    size_t commit_word_size = os::vm_allocation_granularity() / BytesPerWord;
    bool expanded = vsn.expand_by(commit_word_size, commit_word_size);
    assert(expanded, "Failed to commit");

    // Calculate a size that will overflow the virtual space size.
    void* virtual_space_max = (void*)(uintptr_t)-1;
    size_t bottom_to_max = pointer_delta(virtual_space_max, vsn.bottom(), 1);
    size_t overflow_size = bottom_to_max + BytesPerWord;
    size_t overflow_word_size = overflow_size / BytesPerWord;

    // Check that is_available can handle the overflow.
    assert_is_available_negative(overflow_word_size);
  }

  static void test_is_available() {
    TestVirtualSpaceNodeTest::test_is_available_positive();
    TestVirtualSpaceNodeTest::test_is_available_negative();
    TestVirtualSpaceNodeTest::test_is_available_overflow();
  }
3880 3881 3882 3883
};

void TestVirtualSpaceNode_test() {
  TestVirtualSpaceNodeTest::test();
3884
  TestVirtualSpaceNodeTest::test_is_available();
3885
}
3886
#endif