diff --git a/src/os/solaris/vm/os_solaris.cpp b/src/os/solaris/vm/os_solaris.cpp index a270c162fbf4cb8d4ae2827dbeb29ac4b1923a19..ce6765e8d5094f07bfe7324ce710375223bc9da8 100644 --- a/src/os/solaris/vm/os_solaris.cpp +++ b/src/os/solaris/vm/os_solaris.cpp @@ -122,6 +122,13 @@ struct memcntl_mha { # define MADV_ACCESS_MANY 8 /* many processes to access heavily */ #endif +#ifndef LGRP_RSRC_CPU +# define LGRP_RSRC_CPU 0 /* CPU resources */ +#endif +#ifndef LGRP_RSRC_MEM +# define LGRP_RSRC_MEM 1 /* memory resources */ +#endif + // Some more macros from sys/mman.h that are not present in Solaris 8. #ifndef MAX_MEMINFO_CNT @@ -2640,8 +2647,13 @@ size_t os::numa_get_leaf_groups(int *ids, size_t size) { return 1; } if (!r) { + // That's a leaf node. assert (bottom <= cur, "Sanity check"); - ids[bottom++] = ids[cur]; + // Check if the node has memory + if (Solaris::lgrp_resources(Solaris::lgrp_cookie(), ids[cur], + NULL, 0, LGRP_RSRC_MEM) > 0) { + ids[bottom++] = ids[cur]; + } } top += r; cur++; @@ -2664,11 +2676,20 @@ bool os::numa_topology_changed() { // Get the group id of the current LWP. int os::numa_get_group_id() { - int lgrp_id = os::Solaris::lgrp_home(P_LWPID, P_MYID); + int lgrp_id = Solaris::lgrp_home(P_LWPID, P_MYID); if (lgrp_id == -1) { return 0; } - return lgrp_id; + const int size = os::numa_get_groups_num(); + int *ids = (int*)alloca(size * sizeof(int)); + + // Get the ids of all lgroups with memory; r is the count. + int r = Solaris::lgrp_resources(Solaris::lgrp_cookie(), lgrp_id, + (Solaris::lgrp_id_t*)ids, size, LGRP_RSRC_MEM); + if (r <= 0) { + return 0; + } + return ids[os::random() % r]; } // Request information about the page. @@ -4353,6 +4374,7 @@ os::Solaris::lgrp_init_func_t os::Solaris::_lgrp_init; os::Solaris::lgrp_fini_func_t os::Solaris::_lgrp_fini; os::Solaris::lgrp_root_func_t os::Solaris::_lgrp_root; os::Solaris::lgrp_children_func_t os::Solaris::_lgrp_children; +os::Solaris::lgrp_resources_func_t os::Solaris::_lgrp_resources; os::Solaris::lgrp_nlgrps_func_t os::Solaris::_lgrp_nlgrps; os::Solaris::lgrp_cookie_stale_func_t os::Solaris::_lgrp_cookie_stale; os::Solaris::lgrp_cookie_t os::Solaris::_lgrp_cookie = 0; @@ -4555,6 +4577,7 @@ void os::Solaris::liblgrp_init() { os::Solaris::set_lgrp_fini(CAST_TO_FN_PTR(lgrp_fini_func_t, dlsym(handle, "lgrp_fini"))); os::Solaris::set_lgrp_root(CAST_TO_FN_PTR(lgrp_root_func_t, dlsym(handle, "lgrp_root"))); os::Solaris::set_lgrp_children(CAST_TO_FN_PTR(lgrp_children_func_t, dlsym(handle, "lgrp_children"))); + os::Solaris::set_lgrp_resources(CAST_TO_FN_PTR(lgrp_resources_func_t, dlsym(handle, "lgrp_resources"))); os::Solaris::set_lgrp_nlgrps(CAST_TO_FN_PTR(lgrp_nlgrps_func_t, dlsym(handle, "lgrp_nlgrps"))); os::Solaris::set_lgrp_cookie_stale(CAST_TO_FN_PTR(lgrp_cookie_stale_func_t, dlsym(handle, "lgrp_cookie_stale"))); diff --git a/src/os/solaris/vm/os_solaris.hpp b/src/os/solaris/vm/os_solaris.hpp index b66bcb2bf6fc01a8aa690e1b6ba821b8b9282d63..545802ae1581eb5e2b11cdf06b9814671b962d6c 100644 --- a/src/os/solaris/vm/os_solaris.hpp +++ b/src/os/solaris/vm/os_solaris.hpp @@ -66,6 +66,7 @@ class Solaris { typedef uintptr_t lgrp_cookie_t; typedef id_t lgrp_id_t; + typedef int lgrp_rsrc_t; typedef enum lgrp_view { LGRP_VIEW_CALLER, /* what's available to the caller */ LGRP_VIEW_OS /* what's available to operating system */ @@ -77,6 +78,9 @@ class Solaris { typedef lgrp_id_t (*lgrp_root_func_t)(lgrp_cookie_t cookie); typedef int (*lgrp_children_func_t)(lgrp_cookie_t cookie, lgrp_id_t parent, lgrp_id_t *lgrp_array, uint_t lgrp_array_size); + typedef int (*lgrp_resources_func_t)(lgrp_cookie_t cookie, lgrp_id_t lgrp, + lgrp_id_t *lgrp_array, uint_t lgrp_array_size, + lgrp_rsrc_t type); typedef int (*lgrp_nlgrps_func_t)(lgrp_cookie_t cookie); typedef int (*lgrp_cookie_stale_func_t)(lgrp_cookie_t cookie); typedef int (*meminfo_func_t)(const uint64_t inaddr[], int addr_count, @@ -88,6 +92,7 @@ class Solaris { static lgrp_fini_func_t _lgrp_fini; static lgrp_root_func_t _lgrp_root; static lgrp_children_func_t _lgrp_children; + static lgrp_resources_func_t _lgrp_resources; static lgrp_nlgrps_func_t _lgrp_nlgrps; static lgrp_cookie_stale_func_t _lgrp_cookie_stale; static lgrp_cookie_t _lgrp_cookie; @@ -109,7 +114,6 @@ class Solaris { static int (*get_libjsig_version)(); static void save_preinstalled_handler(int, struct sigaction&); static void check_signal_handler(int sig); - // For overridable signals static int _SIGinterrupt; // user-overridable INTERRUPT_SIGNAL static int _SIGasync; // user-overridable ASYNC_SIGNAL @@ -253,8 +257,9 @@ class Solaris { static void set_lgrp_init(lgrp_init_func_t func) { _lgrp_init = func; } static void set_lgrp_fini(lgrp_fini_func_t func) { _lgrp_fini = func; } static void set_lgrp_root(lgrp_root_func_t func) { _lgrp_root = func; } - static void set_lgrp_children(lgrp_children_func_t func) { _lgrp_children = func; } - static void set_lgrp_nlgrps(lgrp_nlgrps_func_t func) { _lgrp_nlgrps = func; } + static void set_lgrp_children(lgrp_children_func_t func) { _lgrp_children = func; } + static void set_lgrp_resources(lgrp_resources_func_t func) { _lgrp_resources = func; } + static void set_lgrp_nlgrps(lgrp_nlgrps_func_t func) { _lgrp_nlgrps = func; } static void set_lgrp_cookie_stale(lgrp_cookie_stale_func_t func) { _lgrp_cookie_stale = func; } static void set_lgrp_cookie(lgrp_cookie_t cookie) { _lgrp_cookie = cookie; } @@ -266,6 +271,12 @@ class Solaris { lgrp_id_t *lgrp_array, uint_t lgrp_array_size) { return _lgrp_children != NULL ? _lgrp_children(cookie, parent, lgrp_array, lgrp_array_size) : -1; } + static int lgrp_resources(lgrp_cookie_t cookie, lgrp_id_t lgrp, + lgrp_id_t *lgrp_array, uint_t lgrp_array_size, + lgrp_rsrc_t type) { + return _lgrp_resources != NULL ? _lgrp_resources(cookie, lgrp, lgrp_array, lgrp_array_size, type) : -1; + } + static int lgrp_nlgrps(lgrp_cookie_t cookie) { return _lgrp_nlgrps != NULL ? _lgrp_nlgrps(cookie) : -1; } static int lgrp_cookie_stale(lgrp_cookie_t cookie) { return _lgrp_cookie_stale != NULL ? _lgrp_cookie_stale(cookie) : -1; diff --git a/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp b/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp index 87ca3afc4a3e7d6b06c233ffb81434a717e82def..3115b6b112777368c52438006c10b5664d06f492 100644 --- a/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp @@ -329,7 +329,7 @@ class Par_PushOrMarkClosure: public OopClosure { class CMSKeepAliveClosure: public OopClosure { private: CMSCollector* _collector; - MemRegion _span; + const MemRegion _span; CMSMarkStack* _mark_stack; CMSBitMap* _bit_map; protected: @@ -340,7 +340,9 @@ class CMSKeepAliveClosure: public OopClosure { _collector(collector), _span(span), _bit_map(bit_map), - _mark_stack(mark_stack) { } + _mark_stack(mark_stack) { + assert(!_span.is_empty(), "Empty span could spell trouble"); + } virtual void do_oop(oop* p); virtual void do_oop(narrowOop* p); inline void do_oop_nv(oop* p) { CMSKeepAliveClosure::do_oop_work(p); } diff --git a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp index 689ede0ea057b5bb0c5b5454b459d8d9a69429e5..8ab7bdd1b58c4ba334f94dd249cf3f2f2e8e8be0 100644 --- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp @@ -520,7 +520,10 @@ CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen, -1 /* lock-free */, "No_lock" /* dummy */), _modUnionClosure(&_modUnionTable), _modUnionClosurePar(&_modUnionTable), - _is_alive_closure(&_markBitMap), + // Adjust my span to cover old (cms) gen and perm gen + _span(cmsGen->reserved()._union(permGen->reserved())), + // Construct the is_alive_closure with _span & markBitMap + _is_alive_closure(_span, &_markBitMap), _restart_addr(NULL), _overflow_list(NULL), _preserved_oop_stack(NULL), @@ -572,11 +575,6 @@ CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen, _cmsGen->cmsSpace()->set_collector(this); _permGen->cmsSpace()->set_collector(this); - // Adjust my span to cover old (cms) gen and perm gen - _span = _cmsGen->reserved()._union(_permGen->reserved()); - // Initialize the span of is_alive_closure - _is_alive_closure.set_span(_span); - // Allocate MUT and marking bit map { MutexLockerEx x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag); @@ -5496,7 +5494,7 @@ class CMSRefProcTaskProxy: public AbstractGangTask { typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; CMSCollector* _collector; CMSBitMap* _mark_bit_map; - MemRegion _span; + const MemRegion _span; OopTaskQueueSet* _task_queues; ParallelTaskTerminator _term; ProcessTask& _task; @@ -5513,7 +5511,10 @@ public: _collector(collector), _span(span), _mark_bit_map(mark_bit_map), _task_queues(task_queues), _term(total_workers, task_queues) - { } + { + assert(_collector->_span.equals(_span) && !_span.is_empty(), + "Inconsistency in _span"); + } OopTaskQueueSet* task_queues() { return _task_queues; } @@ -5530,11 +5531,12 @@ public: }; void CMSRefProcTaskProxy::work(int i) { + assert(_collector->_span.equals(_span), "Inconsistency in _span"); CMSParKeepAliveClosure par_keep_alive(_collector, _span, _mark_bit_map, work_queue(i)); CMSParDrainMarkingStackClosure par_drain_stack(_collector, _span, _mark_bit_map, work_queue(i)); - CMSIsAliveClosure is_alive_closure(_mark_bit_map); + CMSIsAliveClosure is_alive_closure(_span, _mark_bit_map); _task.work(i, is_alive_closure, par_keep_alive, par_drain_stack); if (_task.marks_oops_alive()) { do_work_steal(i, &par_drain_stack, &par_keep_alive, diff --git a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp index ea44e417b833d2a39facf69569106c3631eb10c0..6d5546eeb9533705d42c835408810d2881524fd4 100644 --- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp @@ -435,23 +435,22 @@ class CMSStats VALUE_OBJ_CLASS_SPEC { // if the object is "live" (reachable). Used in weak // reference processing. class CMSIsAliveClosure: public BoolObjectClosure { - MemRegion _span; + const MemRegion _span; const CMSBitMap* _bit_map; friend class CMSCollector; - protected: - void set_span(MemRegion span) { _span = span; } public: - CMSIsAliveClosure(CMSBitMap* bit_map): - _bit_map(bit_map) { } - CMSIsAliveClosure(MemRegion span, CMSBitMap* bit_map): _span(span), - _bit_map(bit_map) { } + _bit_map(bit_map) { + assert(!span.is_empty(), "Empty span could spell trouble"); + } + void do_object(oop obj) { assert(false, "not to be invoked"); } + bool do_object_b(oop obj); }; @@ -600,7 +599,7 @@ class CMSCollector: public CHeapObj { // ("Weak") Reference processing support ReferenceProcessor* _ref_processor; CMSIsAliveClosure _is_alive_closure; - // keep this textually after _markBitMap; c'tor dependency + // keep this textually after _markBitMap and _span; c'tor dependency ConcurrentMarkSweepThread* _cmsThread; // the thread doing the work ModUnionClosure _modUnionClosure; diff --git a/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp b/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp index dd0ed2efc018ccfb65d76c7bc61418881ed8d75a..959c4e1b6bfed8e960083805bb52aad50392900e 100644 --- a/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp +++ b/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp @@ -62,10 +62,10 @@ void MutableNUMASpace::ensure_parsability() { for (int i = 0; i < lgrp_spaces()->length(); i++) { LGRPSpace *ls = lgrp_spaces()->at(i); MutableSpace *s = ls->space(); - if (!s->contains(top())) { + if (s->top() < top()) { // For all spaces preceeding the one containing top() if (s->free_in_words() > 0) { SharedHeap::fill_region_with_object(MemRegion(s->top(), s->end())); - size_t area_touched_words = pointer_delta(s->end(), s->top(), sizeof(HeapWordSize)); + size_t area_touched_words = pointer_delta(s->end(), s->top()); #ifndef ASSERT if (!ZapUnusedHeapArea) { area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)), @@ -88,7 +88,6 @@ void MutableNUMASpace::ensure_parsability() { ls->add_invalid_region(invalid); } - s->set_top(s->end()); } } else { if (!os::numa_has_static_binding()) { @@ -99,8 +98,12 @@ void MutableNUMASpace::ensure_parsability() { if (ZapUnusedHeapArea) { MemRegion invalid(s->top(), s->end()); ls->add_invalid_region(invalid); - } else break; + } else { + return; + } #endif + } else { + return; } } } @@ -658,9 +661,12 @@ HeapWord* MutableNUMASpace::allocate(size_t size) { MutableSpace *s = lgrp_spaces()->at(i)->space(); HeapWord *p = s->allocate(size); - if (p != NULL && s->free_in_words() < (size_t)oopDesc::header_size()) { - s->set_top(s->top() - size); - p = NULL; + if (p != NULL) { + size_t remainder = s->free_in_words(); + if (remainder < (size_t)oopDesc::header_size() && remainder > 0) { + s->set_top(s->top() - size); + p = NULL; + } } if (p != NULL) { if (top() < s->top()) { // Keep _top updated. @@ -693,11 +699,14 @@ HeapWord* MutableNUMASpace::cas_allocate(size_t size) { } MutableSpace *s = lgrp_spaces()->at(i)->space(); HeapWord *p = s->cas_allocate(size); - if (p != NULL && s->free_in_words() < (size_t)oopDesc::header_size()) { - if (s->cas_deallocate(p, size)) { - // We were the last to allocate and created a fragment less than - // a minimal object. - p = NULL; + if (p != NULL) { + size_t remainder = pointer_delta(s->end(), p); + if (remainder < (size_t)oopDesc::header_size() && remainder > 0) { + if (s->cas_deallocate(p, size)) { + // We were the last to allocate and created a fragment less than + // a minimal object. + p = NULL; + } } } if (p != NULL) { @@ -738,6 +747,9 @@ void MutableNUMASpace::print_on(outputStream* st) const { st->print(" lgrp %d", ls->lgrp_id()); ls->space()->print_on(st); if (NUMAStats) { + for (int i = 0; i < lgrp_spaces()->length(); i++) { + lgrp_spaces()->at(i)->accumulate_statistics(page_size()); + } st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n", ls->space_stats()->_local_space / K, ls->space_stats()->_remote_space / K,