/* * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. * */ #ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_HPP #define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_HPP #include "gc_implementation/shared/gSpaceCounters.hpp" #include "gc_implementation/shared/gcStats.hpp" #include "gc_implementation/shared/generationCounters.hpp" #include "memory/freeBlockDictionary.hpp" #include "memory/generation.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/virtualspace.hpp" #include "services/memoryService.hpp" #include "utilities/bitMap.inline.hpp" #include "utilities/stack.inline.hpp" #include "utilities/taskqueue.hpp" #include "utilities/yieldingWorkgroup.hpp" // ConcurrentMarkSweepGeneration is in support of a concurrent // mark-sweep old generation in the Detlefs-Printezis--Boehm-Demers-Schenker // style. We assume, for now, that this generation is always the // seniormost generation and for simplicity // in the first implementation, that this generation is a single compactible // space. Neither of these restrictions appears essential, and will be // relaxed in the future when more time is available to implement the // greater generality (and there's a need for it). // // Concurrent mode failures are currently handled by // means of a sliding mark-compact. class CMSAdaptiveSizePolicy; class CMSConcMarkingTask; class CMSGCAdaptivePolicyCounters; class ConcurrentMarkSweepGeneration; class ConcurrentMarkSweepPolicy; class ConcurrentMarkSweepThread; class CompactibleFreeListSpace; class FreeChunk; class PromotionInfo; class ScanMarkedObjectsAgainCarefullyClosure; // A generic CMS bit map. It's the basis for both the CMS marking bit map // as well as for the mod union table (in each case only a subset of the // methods are used). This is essentially a wrapper around the BitMap class, // with one bit per (1<<_shifter) HeapWords. (i.e. for the marking bit map, // we have _shifter == 0. and for the mod union table we have // shifter == CardTableModRefBS::card_shift - LogHeapWordSize.) // XXX 64-bit issues in BitMap? class CMSBitMap VALUE_OBJ_CLASS_SPEC { friend class VMStructs; HeapWord* _bmStartWord; // base address of range covered by map size_t _bmWordSize; // map size (in #HeapWords covered) const int _shifter; // shifts to convert HeapWord to bit position VirtualSpace _virtual_space; // underlying the bit map BitMap _bm; // the bit map itself public: Mutex* const _lock; // mutex protecting _bm; public: // constructor CMSBitMap(int shifter, int mutex_rank, const char* mutex_name); // allocates the actual storage for the map bool allocate(MemRegion mr); // field getter Mutex* lock() const { return _lock; } // locking verifier convenience function void assert_locked() const PRODUCT_RETURN; // inquiries HeapWord* startWord() const { return _bmStartWord; } size_t sizeInWords() const { return _bmWordSize; } size_t sizeInBits() const { return _bm.size(); } // the following is one past the last word in space HeapWord* endWord() const { return _bmStartWord + _bmWordSize; } // reading marks bool isMarked(HeapWord* addr) const; bool par_isMarked(HeapWord* addr) const; // do not lock checks bool isUnmarked(HeapWord* addr) const; bool isAllClear() const; // writing marks void mark(HeapWord* addr); // For marking by parallel GC threads; // returns true if we did, false if another thread did bool par_mark(HeapWord* addr); void mark_range(MemRegion mr); void par_mark_range(MemRegion mr); void mark_large_range(MemRegion mr); void par_mark_large_range(MemRegion mr); void par_clear(HeapWord* addr); // For unmarking by parallel GC threads. void clear_range(MemRegion mr); void par_clear_range(MemRegion mr); void clear_large_range(MemRegion mr); void par_clear_large_range(MemRegion mr); void clear_all(); void clear_all_incrementally(); // Not yet implemented!! NOT_PRODUCT( // checks the memory region for validity void region_invariant(MemRegion mr); ) // iteration void iterate(BitMapClosure* cl) { _bm.iterate(cl); } void iterate(BitMapClosure* cl, HeapWord* left, HeapWord* right); void dirty_range_iterate_clear(MemRegionClosure* cl); void dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl); // auxiliary support for iteration HeapWord* getNextMarkedWordAddress(HeapWord* addr) const; HeapWord* getNextMarkedWordAddress(HeapWord* start_addr, HeapWord* end_addr) const; HeapWord* getNextUnmarkedWordAddress(HeapWord* addr) const; HeapWord* getNextUnmarkedWordAddress(HeapWord* start_addr, HeapWord* end_addr) const; MemRegion getAndClearMarkedRegion(HeapWord* addr); MemRegion getAndClearMarkedRegion(HeapWord* start_addr, HeapWord* end_addr); // conversion utilities HeapWord* offsetToHeapWord(size_t offset) const; size_t heapWordToOffset(HeapWord* addr) const; size_t heapWordDiffToOffsetDiff(size_t diff) const; // debugging // is this address range covered by the bit-map? NOT_PRODUCT( bool covers(MemRegion mr) const; bool covers(HeapWord* start, size_t size = 0) const; ) void verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) PRODUCT_RETURN; }; // Represents a marking stack used by the CMS collector. // Ideally this should be GrowableArray<> just like MSC's marking stack(s). class CMSMarkStack: public CHeapObj { // friend class CMSCollector; // to get at expasion stats further below // VirtualSpace _virtual_space; // space for the stack oop* _base; // bottom of stack size_t _index; // one more than last occupied index size_t _capacity; // max #elements Mutex _par_lock; // an advisory lock used in case of parallel access NOT_PRODUCT(size_t _max_depth;) // max depth plumbed during run protected: size_t _hit_limit; // we hit max stack size limit size_t _failed_double; // we failed expansion before hitting limit public: CMSMarkStack(): _par_lock(Mutex::event, "CMSMarkStack._par_lock", true), _hit_limit(0), _failed_double(0) {} bool allocate(size_t size); size_t capacity() const { return _capacity; } oop pop() { if (!isEmpty()) { return _base[--_index] ; } return NULL; } bool push(oop ptr) { if (isFull()) { return false; } else { _base[_index++] = ptr; NOT_PRODUCT(_max_depth = MAX2(_max_depth, _index)); return true; } } bool isEmpty() const { return _index == 0; } bool isFull() const { assert(_index <= _capacity, "buffer overflow"); return _index == _capacity; } size_t length() { return _index; } // "Parallel versions" of some of the above oop par_pop() { // lock and pop MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag); return pop(); } bool par_push(oop ptr) { // lock and push MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag); return push(ptr); } // Forcibly reset the stack, losing all of its contents. void reset() { _index = 0; } // Expand the stack, typically in response to an overflow condition void expand(); // Compute the least valued stack element. oop least_value(HeapWord* low) { oop least = (oop)low; for (size_t i = 0; i < _index; i++) { least = MIN2(least, _base[i]); } return least; } // Exposed here to allow stack expansion in || case Mutex* par_lock() { return &_par_lock; } }; class CardTableRS; class CMSParGCThreadState; class ModUnionClosure: public MemRegionClosure { protected: CMSBitMap* _t; public: ModUnionClosure(CMSBitMap* t): _t(t) { } void do_MemRegion(MemRegion mr); }; class ModUnionClosurePar: public ModUnionClosure { public: ModUnionClosurePar(CMSBitMap* t): ModUnionClosure(t) { } void do_MemRegion(MemRegion mr); }; // Survivor Chunk Array in support of parallelization of // Survivor Space rescan. class ChunkArray: public CHeapObj { size_t _index; size_t _capacity; size_t _overflows; HeapWord** _array; // storage for array public: ChunkArray() : _index(0), _capacity(0), _overflows(0), _array(NULL) {} ChunkArray(HeapWord** a, size_t c): _index(0), _capacity(c), _overflows(0), _array(a) {} HeapWord** array() { return _array; } void set_array(HeapWord** a) { _array = a; } size_t capacity() { return _capacity; } void set_capacity(size_t c) { _capacity = c; } size_t end() { assert(_index <= capacity(), err_msg("_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT "): out of bounds", _index, _capacity)); return _index; } // exclusive HeapWord* nth(size_t n) { assert(n < end(), "Out of bounds access"); return _array[n]; } void reset() { _index = 0; if (_overflows > 0 && PrintCMSStatistics > 1) { warning("CMS: ChunkArray[" SIZE_FORMAT "] overflowed " SIZE_FORMAT " times", _capacity, _overflows); } _overflows = 0; } void record_sample(HeapWord* p, size_t sz) { // For now we do not do anything with the size if (_index < _capacity) { _array[_index++] = p; } else { ++_overflows; assert(_index == _capacity, err_msg("_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT "): out of bounds at overflow#" SIZE_FORMAT, _index, _capacity, _overflows)); } } }; // // Timing, allocation and promotion statistics for gc scheduling and incremental // mode pacing. Most statistics are exponential averages. // class CMSStats VALUE_OBJ_CLASS_SPEC { private: ConcurrentMarkSweepGeneration* const _cms_gen; // The cms (old) gen. // The following are exponential averages with factor alpha: // avg = (100 - alpha) * avg + alpha * cur_sample // // The durations measure: end_time[n] - start_time[n] // The periods measure: start_time[n] - start_time[n-1] // // The cms period and duration include only concurrent collections; time spent // in foreground cms collections due to System.gc() or because of a failure to // keep up are not included. // // There are 3 alphas to "bootstrap" the statistics. The _saved_alpha is the // real value, but is used only after the first period. A value of 100 is // used for the first sample so it gets the entire weight. unsigned int _saved_alpha; // 0-100 unsigned int _gc0_alpha; unsigned int _cms_alpha; double _gc0_duration; double _gc0_period; size_t _gc0_promoted; // bytes promoted per gc0 double _cms_duration; double _cms_duration_pre_sweep; // time from initiation to start of sweep double _cms_duration_per_mb; double _cms_period; size_t _cms_allocated; // bytes of direct allocation per gc0 period // Timers. elapsedTimer _cms_timer; TimeStamp _gc0_begin_time; TimeStamp _cms_begin_time; TimeStamp _cms_end_time; // Snapshots of the amount used in the CMS generation. size_t _cms_used_at_gc0_begin; size_t _cms_used_at_gc0_end; size_t _cms_used_at_cms_begin; // Used to prevent the duty cycle from being reduced in the middle of a cms // cycle. bool _allow_duty_cycle_reduction; enum { _GC0_VALID = 0x1, _CMS_VALID = 0x2, _ALL_VALID = _GC0_VALID | _CMS_VALID }; unsigned int _valid_bits; unsigned int _icms_duty_cycle; // icms duty cycle (0-100). protected: // Return a duty cycle that avoids wild oscillations, by limiting the amount // of change between old_duty_cycle and new_duty_cycle (the latter is treated // as a recommended value). static unsigned int icms_damped_duty_cycle(unsigned int old_duty_cycle, unsigned int new_duty_cycle); unsigned int icms_update_duty_cycle_impl(); // In support of adjusting of cms trigger ratios based on history // of concurrent mode failure. double cms_free_adjustment_factor(size_t free) const; void adjust_cms_free_adjustment_factor(bool fail, size_t free); public: CMSStats(ConcurrentMarkSweepGeneration* cms_gen, unsigned int alpha = CMSExpAvgFactor); // Whether or not the statistics contain valid data; higher level statistics // cannot be called until this returns true (they require at least one young // gen and one cms cycle to have completed). bool valid() const; // Record statistics. void record_gc0_begin(); void record_gc0_end(size_t cms_gen_bytes_used); void record_cms_begin(); void record_cms_end(); // Allow management of the cms timer, which must be stopped/started around // yield points. elapsedTimer& cms_timer() { return _cms_timer; } void start_cms_timer() { _cms_timer.start(); } void stop_cms_timer() { _cms_timer.stop(); } // Basic statistics; units are seconds or bytes. double gc0_period() const { return _gc0_period; } double gc0_duration() const { return _gc0_duration; } size_t gc0_promoted() const { return _gc0_promoted; } double cms_period() const { return _cms_period; } double cms_duration() const { return _cms_duration; } double cms_duration_per_mb() const { return _cms_duration_per_mb; } size_t cms_allocated() const { return _cms_allocated; } size_t cms_used_at_gc0_end() const { return _cms_used_at_gc0_end;} // Seconds since the last background cms cycle began or ended. double cms_time_since_begin() const; double cms_time_since_end() const; // Higher level statistics--caller must check that valid() returns true before // calling. // Returns bytes promoted per second of wall clock time. double promotion_rate() const; // Returns bytes directly allocated per second of wall clock time. double cms_allocation_rate() const; // Rate at which space in the cms generation is being consumed (sum of the // above two). double cms_consumption_rate() const; // Returns an estimate of the number of seconds until the cms generation will // fill up, assuming no collection work is done. double time_until_cms_gen_full() const; // Returns an estimate of the number of seconds remaining until // the cms generation collection should start. double time_until_cms_start() const; // End of higher level statistics. // Returns the cms incremental mode duty cycle, as a percentage (0-100). unsigned int icms_duty_cycle() const { return _icms_duty_cycle; } // Update the duty cycle and return the new value. unsigned int icms_update_duty_cycle(); // Debugging. void print_on(outputStream* st) const PRODUCT_RETURN; void print() const { print_on(gclog_or_tty); } }; // A closure related to weak references processing which // we embed in the CMSCollector, since we need to pass // it to the reference processor for secondary filtering // of references based on reachability of referent; // see role of _is_alive_non_header closure in the // ReferenceProcessor class. // For objects in the CMS generation, this closure checks // if the object is "live" (reachable). Used in weak // reference processing. class CMSIsAliveClosure: public BoolObjectClosure { const MemRegion _span; const CMSBitMap* _bit_map; friend class CMSCollector; public: CMSIsAliveClosure(MemRegion span, CMSBitMap* bit_map): _span(span), _bit_map(bit_map) { assert(!span.is_empty(), "Empty span could spell trouble"); } void do_object(oop obj) { assert(false, "not to be invoked"); } bool do_object_b(oop obj); }; // Implements AbstractRefProcTaskExecutor for CMS. class CMSRefProcTaskExecutor: public AbstractRefProcTaskExecutor { public: CMSRefProcTaskExecutor(CMSCollector& collector) : _collector(collector) { } // Executes a task using worker threads. virtual void execute(ProcessTask& task); virtual void execute(EnqueueTask& task); private: CMSCollector& _collector; }; class CMSCollector: public CHeapObj { friend class VMStructs; friend class ConcurrentMarkSweepThread; friend class ConcurrentMarkSweepGeneration; friend class CompactibleFreeListSpace; friend class CMSParRemarkTask; friend class CMSConcMarkingTask; friend class CMSRefProcTaskProxy; friend class CMSRefProcTaskExecutor; friend class ScanMarkedObjectsAgainCarefullyClosure; // for sampling eden friend class SurvivorSpacePrecleanClosure; // --- ditto ------- friend class PushOrMarkClosure; // to access _restart_addr friend class Par_PushOrMarkClosure; // to access _restart_addr friend class MarkFromRootsClosure; // -- ditto -- // ... and for clearing cards friend class Par_MarkFromRootsClosure; // to access _restart_addr // ... and for clearing cards friend class Par_ConcMarkingClosure; // to access _restart_addr etc. friend class MarkFromRootsVerifyClosure; // to access _restart_addr friend class PushAndMarkVerifyClosure; // -- ditto -- friend class MarkRefsIntoAndScanClosure; // to access _overflow_list friend class PushAndMarkClosure; // -- ditto -- friend class Par_PushAndMarkClosure; // -- ditto -- friend class CMSKeepAliveClosure; // -- ditto -- friend class CMSDrainMarkingStackClosure; // -- ditto -- friend class CMSInnerParMarkAndPushClosure; // -- ditto -- NOT_PRODUCT(friend class ScanMarkedObjectsAgainClosure;) // assertion on _overflow_list friend class ReleaseForegroundGC; // to access _foregroundGCShouldWait friend class VM_CMS_Operation; friend class VM_CMS_Initial_Mark; friend class VM_CMS_Final_Remark; friend class TraceCMSMemoryManagerStats; private: jlong _time_of_last_gc; void update_time_of_last_gc(jlong now) { _time_of_last_gc = now; } OopTaskQueueSet* _task_queues; // Overflow list of grey objects, threaded through mark-word // Manipulated with CAS in the parallel/multi-threaded case. oop _overflow_list; // The following array-pair keeps track of mark words // displaced for accomodating overflow list above. // This code will likely be revisited under RFE#4922830. Stack _preserved_oop_stack; Stack _preserved_mark_stack; int* _hash_seed; // In support of multi-threaded concurrent phases YieldingFlexibleWorkGang* _conc_workers; // Performance Counters CollectorCounters* _gc_counters; // Initialization Errors bool _completed_initialization; // In support of ExplicitGCInvokesConcurrent static bool _full_gc_requested; unsigned int _collection_count_start; // Should we unload classes this concurrent cycle? bool _should_unload_classes; unsigned int _concurrent_cycles_since_last_unload; unsigned int concurrent_cycles_since_last_unload() const { return _concurrent_cycles_since_last_unload; } // Did we (allow) unload classes in the previous concurrent cycle? bool unloaded_classes_last_cycle() const { return concurrent_cycles_since_last_unload() == 0; } // Root scanning options for perm gen int _roots_scanning_options; int roots_scanning_options() const { return _roots_scanning_options; } void add_root_scanning_option(int o) { _roots_scanning_options |= o; } void remove_root_scanning_option(int o) { _roots_scanning_options &= ~o; } // Verification support CMSBitMap _verification_mark_bm; void verify_after_remark_work_1(); void verify_after_remark_work_2(); // true if any verification flag is on. bool _verifying; bool verifying() const { return _verifying; } void set_verifying(bool v) { _verifying = v; } // Collector policy ConcurrentMarkSweepPolicy* _collector_policy; ConcurrentMarkSweepPolicy* collector_policy() { return _collector_policy; } // XXX Move these to CMSStats ??? FIX ME !!! elapsedTimer _inter_sweep_timer; // time between sweeps elapsedTimer _intra_sweep_timer; // time _in_ sweeps // padded decaying average estimates of the above AdaptivePaddedAverage _inter_sweep_estimate; AdaptivePaddedAverage _intra_sweep_estimate; protected: ConcurrentMarkSweepGeneration* _cmsGen; // old gen (CMS) MemRegion _span; // span covering above two CardTableRS* _ct; // card table // CMS marking support structures CMSBitMap _markBitMap; CMSBitMap _modUnionTable; CMSMarkStack _markStack; HeapWord* _restart_addr; // in support of marking stack overflow void lower_restart_addr(HeapWord* low); // Counters in support of marking stack / work queue overflow handling: // a non-zero value indicates certain types of overflow events during // the current CMS cycle and could lead to stack resizing efforts at // an opportune future time. size_t _ser_pmc_preclean_ovflw; size_t _ser_pmc_remark_ovflw; size_t _par_pmc_remark_ovflw; size_t _ser_kac_preclean_ovflw; size_t _ser_kac_ovflw; size_t _par_kac_ovflw; NOT_PRODUCT(ssize_t _num_par_pushes;) // ("Weak") Reference processing support ReferenceProcessor* _ref_processor; CMSIsAliveClosure _is_alive_closure; // keep this textually after _markBitMap and _span; c'tor dependency ConcurrentMarkSweepThread* _cmsThread; // the thread doing the work ModUnionClosure _modUnionClosure; ModUnionClosurePar _modUnionClosurePar; // CMS abstract state machine // initial_state: Idling // next_state(Idling) = {Marking} // next_state(Marking) = {Precleaning, Sweeping} // next_state(Precleaning) = {AbortablePreclean, FinalMarking} // next_state(AbortablePreclean) = {FinalMarking} // next_state(FinalMarking) = {Sweeping} // next_state(Sweeping) = {Resizing} // next_state(Resizing) = {Resetting} // next_state(Resetting) = {Idling} // The numeric values below are chosen so that: // . _collectorState <= Idling == post-sweep && pre-mark // . _collectorState in (Idling, Sweeping) == {initial,final}marking || // precleaning || abortablePrecleanb public: enum CollectorState { Resizing = 0, Resetting = 1, Idling = 2, InitialMarking = 3, Marking = 4, Precleaning = 5, AbortablePreclean = 6, FinalMarking = 7, Sweeping = 8 }; protected: static CollectorState _collectorState; // State related to prologue/epilogue invocation for my generations bool _between_prologue_and_epilogue; // Signalling/State related to coordination between fore- and backgroud GC // Note: When the baton has been passed from background GC to foreground GC, // _foregroundGCIsActive is true and _foregroundGCShouldWait is false. static bool _foregroundGCIsActive; // true iff foreground collector is active or // wants to go active static bool _foregroundGCShouldWait; // true iff background GC is active and has not // yet passed the baton to the foreground GC // Support for CMSScheduleRemark (abortable preclean) bool _abort_preclean; bool _start_sampling; int _numYields; size_t _numDirtyCards; size_t _sweep_count; // number of full gc's since the last concurrent gc. uint _full_gcs_since_conc_gc; // occupancy used for bootstrapping stats double _bootstrap_occupancy; // timer elapsedTimer _timer; // Timing, allocation and promotion statistics, used for scheduling. CMSStats _stats; // Allocation limits installed in the young gen, used only in // CMSIncrementalMode. When an allocation in the young gen would cross one of // these limits, the cms generation is notified and the cms thread is started // or stopped, respectively. HeapWord* _icms_start_limit; HeapWord* _icms_stop_limit; enum CMS_op_type { CMS_op_checkpointRootsInitial, CMS_op_checkpointRootsFinal }; void do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause); bool stop_world_and_do(CMS_op_type op); OopTaskQueueSet* task_queues() { return _task_queues; } int* hash_seed(int i) { return &_hash_seed[i]; } YieldingFlexibleWorkGang* conc_workers() { return _conc_workers; } // Support for parallelizing Eden rescan in CMS remark phase void sample_eden(); // ... sample Eden space top private: // Support for parallelizing young gen rescan in CMS remark phase Generation* _young_gen; // the younger gen HeapWord** _top_addr; // ... Top of Eden HeapWord** _end_addr; // ... End of Eden HeapWord** _eden_chunk_array; // ... Eden partitioning array size_t _eden_chunk_index; // ... top (exclusive) of array size_t _eden_chunk_capacity; // ... max entries in array // Support for parallelizing survivor space rescan HeapWord** _survivor_chunk_array; size_t _survivor_chunk_index; size_t _survivor_chunk_capacity; size_t* _cursor; ChunkArray* _survivor_plab_array; // Support for marking stack overflow handling bool take_from_overflow_list(size_t num, CMSMarkStack* to_stack); bool par_take_from_overflow_list(size_t num, OopTaskQueue* to_work_q, int no_of_gc_threads); void push_on_overflow_list(oop p); void par_push_on_overflow_list(oop p); // the following is, obviously, not, in general, "MT-stable" bool overflow_list_is_empty() const; void preserve_mark_if_necessary(oop p); void par_preserve_mark_if_necessary(oop p); void preserve_mark_work(oop p, markOop m); void restore_preserved_marks_if_any(); NOT_PRODUCT(bool no_preserved_marks() const;) // in support of testing overflow code NOT_PRODUCT(int _overflow_counter;) NOT_PRODUCT(bool simulate_overflow();) // sequential NOT_PRODUCT(bool par_simulate_overflow();) // MT version // CMS work methods void checkpointRootsInitialWork(bool asynch); // initial checkpoint work // a return value of false indicates failure due to stack overflow bool markFromRootsWork(bool asynch); // concurrent marking work public: // FIX ME!!! only for testing bool do_marking_st(bool asynch); // single-threaded marking bool do_marking_mt(bool asynch); // multi-threaded marking private: // concurrent precleaning work size_t preclean_mod_union_table(ConcurrentMarkSweepGeneration* gen, ScanMarkedObjectsAgainCarefullyClosure* cl); size_t preclean_card_table(ConcurrentMarkSweepGeneration* gen, ScanMarkedObjectsAgainCarefullyClosure* cl); // Does precleaning work, returning a quantity indicative of // the amount of "useful work" done. size_t preclean_work(bool clean_refs, bool clean_survivors); void preclean_klasses(MarkRefsIntoAndScanClosure* cl, Mutex* freelistLock); void abortable_preclean(); // Preclean while looking for possible abort void initialize_sequential_subtasks_for_young_gen_rescan(int i); // Helper function for above; merge-sorts the per-thread plab samples void merge_survivor_plab_arrays(ContiguousSpace* surv, int no_of_gc_threads); // Resets (i.e. clears) the per-thread plab sample vectors void reset_survivor_plab_arrays(); // final (second) checkpoint work void checkpointRootsFinalWork(bool asynch, bool clear_all_soft_refs, bool init_mark_was_synchronous); // work routine for parallel version of remark void do_remark_parallel(); // work routine for non-parallel version of remark void do_remark_non_parallel(); // reference processing work routine (during second checkpoint) void refProcessingWork(bool asynch, bool clear_all_soft_refs); // concurrent sweeping work void sweepWork(ConcurrentMarkSweepGeneration* gen, bool asynch); // (concurrent) resetting of support data structures void reset(bool asynch); // Clear _expansion_cause fields of constituent generations void clear_expansion_cause(); // An auxilliary method used to record the ends of // used regions of each generation to limit the extent of sweep void save_sweep_limits(); // Resize the generations included in the collector. void compute_new_size(); // A work method used by foreground collection to determine // what type of collection (compacting or not, continuing or fresh) // it should do. void decide_foreground_collection_type(bool clear_all_soft_refs, bool* should_compact, bool* should_start_over); // A work method used by the foreground collector to do // a mark-sweep-compact. void do_compaction_work(bool clear_all_soft_refs); // A work method used by the foreground collector to do // a mark-sweep, after taking over from a possibly on-going // concurrent mark-sweep collection. void do_mark_sweep_work(bool clear_all_soft_refs, CollectorState first_state, bool should_start_over); // If the backgrould GC is active, acquire control from the background // GC and do the collection. void acquire_control_and_collect(bool full, bool clear_all_soft_refs); // For synchronizing passing of control from background to foreground // GC. waitForForegroundGC() is called by the background // collector. It if had to wait for a foreground collection, // it returns true and the background collection should assume // that the collection was finished by the foreground // collector. bool waitForForegroundGC(); // Incremental mode triggering: recompute the icms duty cycle and set the // allocation limits in the young gen. void icms_update_allocation_limits(); size_t block_size_using_printezis_bits(HeapWord* addr) const; size_t block_size_if_printezis_bits(HeapWord* addr) const; HeapWord* next_card_start_after_block(HeapWord* addr) const; void setup_cms_unloading_and_verification_state(); public: CMSCollector(ConcurrentMarkSweepGeneration* cmsGen, CardTableRS* ct, ConcurrentMarkSweepPolicy* cp); ConcurrentMarkSweepThread* cmsThread() { return _cmsThread; } ReferenceProcessor* ref_processor() { return _ref_processor; } void ref_processor_init(); Mutex* bitMapLock() const { return _markBitMap.lock(); } static CollectorState abstract_state() { return _collectorState; } bool should_abort_preclean() const; // Whether preclean should be aborted. size_t get_eden_used() const; size_t get_eden_capacity() const; ConcurrentMarkSweepGeneration* cmsGen() { return _cmsGen; } // locking checks NOT_PRODUCT(static bool have_cms_token();) // XXXPERM bool should_collect(bool full, size_t size, bool tlab); bool shouldConcurrentCollect(); void collect(bool full, bool clear_all_soft_refs, size_t size, bool tlab); void collect_in_background(bool clear_all_soft_refs); void collect_in_foreground(bool clear_all_soft_refs); // In support of ExplicitGCInvokesConcurrent static void request_full_gc(unsigned int full_gc_count); // Should we unload classes in a particular concurrent cycle? bool should_unload_classes() const { return _should_unload_classes; } void update_should_unload_classes(); void direct_allocated(HeapWord* start, size_t size); // Object is dead if not marked and current phase is sweeping. bool is_dead_obj(oop obj) const; // After a promotion (of "start"), do any necessary marking. // If "par", then it's being done by a parallel GC thread. // The last two args indicate if we need precise marking // and if so the size of the object so it can be dirtied // in its entirety. void promoted(bool par, HeapWord* start, bool is_obj_array, size_t obj_size); HeapWord* allocation_limit_reached(Space* space, HeapWord* top, size_t word_size); void getFreelistLocks() const; void releaseFreelistLocks() const; bool haveFreelistLocks() const; // GC prologue and epilogue void gc_prologue(bool full); void gc_epilogue(bool full); jlong time_of_last_gc(jlong now) { if (_collectorState <= Idling) { // gc not in progress return _time_of_last_gc; } else { // collection in progress return now; } } // Support for parallel remark of survivor space void* get_data_recorder(int thr_num); CMSBitMap* markBitMap() { return &_markBitMap; } void directAllocated(HeapWord* start, size_t size); // main CMS steps and related support void checkpointRootsInitial(bool asynch); bool markFromRoots(bool asynch); // a return value of false indicates failure // due to stack overflow void preclean(); void checkpointRootsFinal(bool asynch, bool clear_all_soft_refs, bool init_mark_was_synchronous); void sweep(bool asynch); // Check that the currently executing thread is the expected // one (foreground collector or background collector). static void check_correct_thread_executing() PRODUCT_RETURN; // XXXPERM void print_statistics() PRODUCT_RETURN; bool is_cms_reachable(HeapWord* addr); // Performance Counter Support CollectorCounters* counters() { return _gc_counters; } // timer stuff void startTimer() { assert(!_timer.is_active(), "Error"); _timer.start(); } void stopTimer() { assert( _timer.is_active(), "Error"); _timer.stop(); } void resetTimer() { assert(!_timer.is_active(), "Error"); _timer.reset(); } double timerValue() { assert(!_timer.is_active(), "Error"); return _timer.seconds(); } int yields() { return _numYields; } void resetYields() { _numYields = 0; } void incrementYields() { _numYields++; } void resetNumDirtyCards() { _numDirtyCards = 0; } void incrementNumDirtyCards(size_t num) { _numDirtyCards += num; } size_t numDirtyCards() { return _numDirtyCards; } static bool foregroundGCShouldWait() { return _foregroundGCShouldWait; } static void set_foregroundGCShouldWait(bool v) { _foregroundGCShouldWait = v; } static bool foregroundGCIsActive() { return _foregroundGCIsActive; } static void set_foregroundGCIsActive(bool v) { _foregroundGCIsActive = v; } size_t sweep_count() const { return _sweep_count; } void increment_sweep_count() { _sweep_count++; } // Timers/stats for gc scheduling and incremental mode pacing. CMSStats& stats() { return _stats; } // Convenience methods that check whether CMSIncrementalMode is enabled and // forward to the corresponding methods in ConcurrentMarkSweepThread. static void start_icms(); static void stop_icms(); // Called at the end of the cms cycle. static void disable_icms(); // Called before a foreground collection. static void enable_icms(); // Called after a foreground collection. void icms_wait(); // Called at yield points. // Adaptive size policy CMSAdaptiveSizePolicy* size_policy(); CMSGCAdaptivePolicyCounters* gc_adaptive_policy_counters(); // debugging void verify(); bool verify_after_remark(); void verify_ok_to_terminate() const PRODUCT_RETURN; void verify_work_stacks_empty() const PRODUCT_RETURN; void verify_overflow_empty() const PRODUCT_RETURN; // convenience methods in support of debugging static const size_t skip_header_HeapWords() PRODUCT_RETURN0; HeapWord* block_start(const void* p) const PRODUCT_RETURN0; // accessors CMSMarkStack* verification_mark_stack() { return &_markStack; } CMSBitMap* verification_mark_bm() { return &_verification_mark_bm; } // Initialization errors bool completed_initialization() { return _completed_initialization; } }; class CMSExpansionCause : public AllStatic { public: enum Cause { _no_expansion, _satisfy_free_ratio, _satisfy_promotion, _satisfy_allocation, _allocate_par_lab, _allocate_par_spooling_space, _adaptive_size_policy }; // Return a string describing the cause of the expansion. static const char* to_string(CMSExpansionCause::Cause cause); }; class ConcurrentMarkSweepGeneration: public CardGeneration { friend class VMStructs; friend class ConcurrentMarkSweepThread; friend class ConcurrentMarkSweep; friend class CMSCollector; protected: static CMSCollector* _collector; // the collector that collects us CompactibleFreeListSpace* _cmsSpace; // underlying space (only one for now) // Performance Counters GenerationCounters* _gen_counters; GSpaceCounters* _space_counters; // Words directly allocated, used by CMSStats. size_t _direct_allocated_words; // Non-product stat counters NOT_PRODUCT( size_t _numObjectsPromoted; size_t _numWordsPromoted; size_t _numObjectsAllocated; size_t _numWordsAllocated; ) // Used for sizing decisions bool _incremental_collection_failed; bool incremental_collection_failed() { return _incremental_collection_failed; } void set_incremental_collection_failed() { _incremental_collection_failed = true; } void clear_incremental_collection_failed() { _incremental_collection_failed = false; } // accessors void set_expansion_cause(CMSExpansionCause::Cause v) { _expansion_cause = v;} CMSExpansionCause::Cause expansion_cause() const { return _expansion_cause; } private: // For parallel young-gen GC support. CMSParGCThreadState** _par_gc_thread_states; // Reason generation was expanded CMSExpansionCause::Cause _expansion_cause; // In support of MinChunkSize being larger than min object size const double _dilatation_factor; enum CollectionTypes { Concurrent_collection_type = 0, MS_foreground_collection_type = 1, MSC_foreground_collection_type = 2, Unknown_collection_type = 3 }; CollectionTypes _debug_collection_type; // Fraction of current occupancy at which to start a CMS collection which // will collect this generation (at least). double _initiating_occupancy; protected: // Shrink generation by specified size (returns false if unable to shrink) virtual void shrink_by(size_t bytes); // Update statistics for GC virtual void update_gc_stats(int level, bool full); // Maximum available space in the generation (including uncommitted) // space. size_t max_available() const; // getter and initializer for _initiating_occupancy field. double initiating_occupancy() const { return _initiating_occupancy; } void init_initiating_occupancy(intx io, intx tr); public: ConcurrentMarkSweepGeneration(ReservedSpace rs, size_t initial_byte_size, int level, CardTableRS* ct, bool use_adaptive_freelists, FreeBlockDictionary::DictionaryChoice); // Accessors CMSCollector* collector() const { return _collector; } static void set_collector(CMSCollector* collector) { assert(_collector == NULL, "already set"); _collector = collector; } CompactibleFreeListSpace* cmsSpace() const { return _cmsSpace; } Mutex* freelistLock() const; virtual Generation::Name kind() { return Generation::ConcurrentMarkSweep; } // Adaptive size policy CMSAdaptiveSizePolicy* size_policy(); bool refs_discovery_is_atomic() const { return false; } bool refs_discovery_is_mt() const { // Note: CMS does MT-discovery during the parallel-remark // phases. Use ReferenceProcessorMTMutator to make refs // discovery MT-safe during such phases or other parallel // discovery phases in the future. This may all go away // if/when we decide that refs discovery is sufficiently // rare that the cost of the CAS's involved is in the // noise. That's a measurement that should be done, and // the code simplified if that turns out to be the case. return ConcGCThreads > 1; } // Override virtual void ref_processor_init(); // Grow generation by specified size (returns false if unable to grow) bool grow_by(size_t bytes); // Grow generation to reserved size. bool grow_to_reserved(); void clear_expansion_cause() { _expansion_cause = CMSExpansionCause::_no_expansion; } // Space enquiries size_t capacity() const; size_t used() const; size_t free() const; double occupancy() const { return ((double)used())/((double)capacity()); } size_t contiguous_available() const; size_t unsafe_max_alloc_nogc() const; // over-rides MemRegion used_region() const; MemRegion used_region_at_save_marks() const; // Does a "full" (forced) collection invoked on this generation collect // all younger generations as well? Note that the second conjunct is a // hack to allow the collection of the younger gen first if the flag is // set. This is better than using th policy's should_collect_gen0_first() // since that causes us to do an extra unnecessary pair of restart-&-stop-world. virtual bool full_collects_younger_generations() const { return UseCMSCompactAtFullCollection && !CollectGen0First; } void space_iterate(SpaceClosure* blk, bool usedOnly = false); // Support for compaction CompactibleSpace* first_compaction_space() const; // Adjust quantites in the generation affected by // the compaction. void reset_after_compaction(); // Allocation support HeapWord* allocate(size_t size, bool tlab); HeapWord* have_lock_and_allocate(size_t size, bool tlab); oop promote(oop obj, size_t obj_size); HeapWord* par_allocate(size_t size, bool tlab) { return allocate(size, tlab); } // Incremental mode triggering. HeapWord* allocation_limit_reached(Space* space, HeapWord* top, size_t word_size); // Used by CMSStats to track direct allocation. The value is sampled and // reset after each young gen collection. size_t direct_allocated_words() const { return _direct_allocated_words; } void reset_direct_allocated_words() { _direct_allocated_words = 0; } // Overrides for parallel promotion. virtual oop par_promote(int thread_num, oop obj, markOop m, size_t word_sz); // This one should not be called for CMS. virtual void par_promote_alloc_undo(int thread_num, HeapWord* obj, size_t word_sz); virtual void par_promote_alloc_done(int thread_num); virtual void par_oop_since_save_marks_iterate_done(int thread_num); virtual bool promotion_attempt_is_safe(size_t promotion_in_bytes) const; // Inform this (non-young) generation that a promotion failure was // encountered during a collection of a younger generation that // promotes into this generation. virtual void promotion_failure_occurred(); bool should_collect(bool full, size_t size, bool tlab); virtual bool should_concurrent_collect() const; virtual bool is_too_full() const; void collect(bool full, bool clear_all_soft_refs, size_t size, bool tlab); HeapWord* expand_and_allocate(size_t word_size, bool tlab, bool parallel = false); // GC prologue and epilogue void gc_prologue(bool full); void gc_prologue_work(bool full, bool registerClosure, ModUnionClosure* modUnionClosure); void gc_epilogue(bool full); void gc_epilogue_work(bool full); // Time since last GC of this generation jlong time_of_last_gc(jlong now) { return collector()->time_of_last_gc(now); } void update_time_of_last_gc(jlong now) { collector()-> update_time_of_last_gc(now); } // Allocation failure void expand(size_t bytes, size_t expand_bytes, CMSExpansionCause::Cause cause); virtual bool expand(size_t bytes, size_t expand_bytes); void shrink(size_t bytes); HeapWord* expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz); bool expand_and_ensure_spooling_space(PromotionInfo* promo); // Iteration support and related enquiries void save_marks(); bool no_allocs_since_save_marks(); void object_iterate_since_last_GC(ObjectClosure* cl); void younger_refs_iterate(OopsInGenClosure* cl); // Iteration support specific to CMS generations void save_sweep_limit(); // More iteration support virtual void oop_iterate(MemRegion mr, ExtendedOopClosure* cl); virtual void oop_iterate(ExtendedOopClosure* cl); virtual void safe_object_iterate(ObjectClosure* cl); virtual void object_iterate(ObjectClosure* cl); // Need to declare the full complement of closures, whether we'll // override them or not, or get message from the compiler: // oop_since_save_marks_iterate_nv hides virtual function... #define CMS_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl); ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DECL) // Smart allocation XXX -- move to CFLSpace? void setNearLargestChunk(); bool isNearLargestChunk(HeapWord* addr); // Get the chunk at the end of the space. Delagates to // the space. FreeChunk* find_chunk_at_end(); void post_compact(); // Debugging void prepare_for_verify(); void verify(); void print_statistics() PRODUCT_RETURN; // Performance Counters support virtual void update_counters(); virtual void update_counters(size_t used); void initialize_performance_counters(); CollectorCounters* counters() { return collector()->counters(); } // Support for parallel remark of survivor space void* get_data_recorder(int thr_num) { //Delegate to collector return collector()->get_data_recorder(thr_num); } // Printing const char* name() const; virtual const char* short_name() const { return "CMS"; } void print() const; void printOccupancy(const char* s); bool must_be_youngest() const { return false; } bool must_be_oldest() const { return true; } void compute_new_size(); CollectionTypes debug_collection_type() { return _debug_collection_type; } void rotate_debug_collection_type(); }; class ASConcurrentMarkSweepGeneration : public ConcurrentMarkSweepGeneration { // Return the size policy from the heap's collector // policy casted to CMSAdaptiveSizePolicy*. CMSAdaptiveSizePolicy* cms_size_policy() const; // Resize the generation based on the adaptive size // policy. void resize(size_t cur_promo, size_t desired_promo); // Return the GC counters from the collector policy CMSGCAdaptivePolicyCounters* gc_adaptive_policy_counters(); virtual void shrink_by(size_t bytes); public: virtual void compute_new_size(); ASConcurrentMarkSweepGeneration(ReservedSpace rs, size_t initial_byte_size, int level, CardTableRS* ct, bool use_adaptive_freelists, FreeBlockDictionary::DictionaryChoice dictionaryChoice) : ConcurrentMarkSweepGeneration(rs, initial_byte_size, level, ct, use_adaptive_freelists, dictionaryChoice) {} virtual const char* short_name() const { return "ASCMS"; } virtual Generation::Name kind() { return Generation::ASConcurrentMarkSweep; } virtual void update_counters(); virtual void update_counters(size_t used); }; // // Closures of various sorts used by CMS to accomplish its work // // This closure is used to check that a certain set of oops is empty. class FalseClosure: public OopClosure { public: void do_oop(oop* p) { guarantee(false, "Should be an empty set"); } void do_oop(narrowOop* p) { guarantee(false, "Should be an empty set"); } }; // This closure is used to do concurrent marking from the roots // following the first checkpoint. class MarkFromRootsClosure: public BitMapClosure { CMSCollector* _collector; MemRegion _span; CMSBitMap* _bitMap; CMSBitMap* _mut; CMSMarkStack* _markStack; bool _yield; int _skipBits; HeapWord* _finger; HeapWord* _threshold; DEBUG_ONLY(bool _verifying;) public: MarkFromRootsClosure(CMSCollector* collector, MemRegion span, CMSBitMap* bitMap, CMSMarkStack* markStack, bool should_yield, bool verifying = false); bool do_bit(size_t offset); void reset(HeapWord* addr); inline void do_yield_check(); private: void scanOopsInOop(HeapWord* ptr); void do_yield_work(); }; // This closure is used to do concurrent multi-threaded // marking from the roots following the first checkpoint. // XXX This should really be a subclass of The serial version // above, but i have not had the time to refactor things cleanly. // That willbe done for Dolphin. class Par_MarkFromRootsClosure: public BitMapClosure { CMSCollector* _collector; MemRegion _whole_span; MemRegion _span; CMSBitMap* _bit_map; CMSBitMap* _mut; OopTaskQueue* _work_queue; CMSMarkStack* _overflow_stack; bool _yield; int _skip_bits; HeapWord* _finger; HeapWord* _threshold; CMSConcMarkingTask* _task; public: Par_MarkFromRootsClosure(CMSConcMarkingTask* task, CMSCollector* collector, MemRegion span, CMSBitMap* bit_map, OopTaskQueue* work_queue, CMSMarkStack* overflow_stack, bool should_yield); bool do_bit(size_t offset); inline void do_yield_check(); private: void scan_oops_in_oop(HeapWord* ptr); void do_yield_work(); bool get_work_from_overflow_stack(); }; // The following closures are used to do certain kinds of verification of // CMS marking. class PushAndMarkVerifyClosure: public CMSOopClosure { CMSCollector* _collector; MemRegion _span; CMSBitMap* _verification_bm; CMSBitMap* _cms_bm; CMSMarkStack* _mark_stack; protected: void do_oop(oop p); template inline void do_oop_work(T *p) { oop obj = oopDesc::load_decode_heap_oop(p); do_oop(obj); } public: PushAndMarkVerifyClosure(CMSCollector* cms_collector, MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm, CMSMarkStack* mark_stack); void do_oop(oop* p); void do_oop(narrowOop* p); // Deal with a stack overflow condition void handle_stack_overflow(HeapWord* lost); }; class MarkFromRootsVerifyClosure: public BitMapClosure { CMSCollector* _collector; MemRegion _span; CMSBitMap* _verification_bm; CMSBitMap* _cms_bm; CMSMarkStack* _mark_stack; HeapWord* _finger; PushAndMarkVerifyClosure _pam_verify_closure; public: MarkFromRootsVerifyClosure(CMSCollector* collector, MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm, CMSMarkStack* mark_stack); bool do_bit(size_t offset); void reset(HeapWord* addr); }; // This closure is used to check that a certain set of bits is // "empty" (i.e. the bit vector doesn't have any 1-bits). class FalseBitMapClosure: public BitMapClosure { public: bool do_bit(size_t offset) { guarantee(false, "Should not have a 1 bit"); return true; } }; // This closure is used during the second checkpointing phase // to rescan the marked objects on the dirty cards in the mod // union table and the card table proper. It's invoked via // MarkFromDirtyCardsClosure below. It uses either // [Par_]MarkRefsIntoAndScanClosure (Par_ in the parallel case) // declared in genOopClosures.hpp to accomplish some of its work. // In the parallel case the bitMap is shared, so access to // it needs to be suitably synchronized for updates by embedded // closures that update it; however, this closure itself only // reads the bit_map and because it is idempotent, is immune to // reading stale values. class ScanMarkedObjectsAgainClosure: public UpwardsObjectClosure { #ifdef ASSERT CMSCollector* _collector; MemRegion _span; union { CMSMarkStack* _mark_stack; OopTaskQueue* _work_queue; }; #endif // ASSERT bool _parallel; CMSBitMap* _bit_map; union { MarkRefsIntoAndScanClosure* _scan_closure; Par_MarkRefsIntoAndScanClosure* _par_scan_closure; }; public: ScanMarkedObjectsAgainClosure(CMSCollector* collector, MemRegion span, ReferenceProcessor* rp, CMSBitMap* bit_map, CMSMarkStack* mark_stack, MarkRefsIntoAndScanClosure* cl): #ifdef ASSERT _collector(collector), _span(span), _mark_stack(mark_stack), #endif // ASSERT _parallel(false), _bit_map(bit_map), _scan_closure(cl) { } ScanMarkedObjectsAgainClosure(CMSCollector* collector, MemRegion span, ReferenceProcessor* rp, CMSBitMap* bit_map, OopTaskQueue* work_queue, Par_MarkRefsIntoAndScanClosure* cl): #ifdef ASSERT _collector(collector), _span(span), _work_queue(work_queue), #endif // ASSERT _parallel(true), _bit_map(bit_map), _par_scan_closure(cl) { } void do_object(oop obj) { guarantee(false, "Call do_object_b(oop, MemRegion) instead"); } bool do_object_b(oop obj) { guarantee(false, "Call do_object_b(oop, MemRegion) form instead"); return false; } bool do_object_bm(oop p, MemRegion mr); }; // This closure is used during the second checkpointing phase // to rescan the marked objects on the dirty cards in the mod // union table and the card table proper. It invokes // ScanMarkedObjectsAgainClosure above to accomplish much of its work. // In the parallel case, the bit map is shared and requires // synchronized access. class MarkFromDirtyCardsClosure: public MemRegionClosure { CompactibleFreeListSpace* _space; ScanMarkedObjectsAgainClosure _scan_cl; size_t _num_dirty_cards; public: MarkFromDirtyCardsClosure(CMSCollector* collector, MemRegion span, CompactibleFreeListSpace* space, CMSBitMap* bit_map, CMSMarkStack* mark_stack, MarkRefsIntoAndScanClosure* cl): _space(space), _num_dirty_cards(0), _scan_cl(collector, span, collector->ref_processor(), bit_map, mark_stack, cl) { } MarkFromDirtyCardsClosure(CMSCollector* collector, MemRegion span, CompactibleFreeListSpace* space, CMSBitMap* bit_map, OopTaskQueue* work_queue, Par_MarkRefsIntoAndScanClosure* cl): _space(space), _num_dirty_cards(0), _scan_cl(collector, span, collector->ref_processor(), bit_map, work_queue, cl) { } void do_MemRegion(MemRegion mr); void set_space(CompactibleFreeListSpace* space) { _space = space; } size_t num_dirty_cards() { return _num_dirty_cards; } }; // This closure is used in the non-product build to check // that there are no MemRegions with a certain property. class FalseMemRegionClosure: public MemRegionClosure { void do_MemRegion(MemRegion mr) { guarantee(!mr.is_empty(), "Shouldn't be empty"); guarantee(false, "Should never be here"); } }; // This closure is used during the precleaning phase // to "carefully" rescan marked objects on dirty cards. // It uses MarkRefsIntoAndScanClosure declared in genOopClosures.hpp // to accomplish some of its work. class ScanMarkedObjectsAgainCarefullyClosure: public ObjectClosureCareful { CMSCollector* _collector; MemRegion _span; bool _yield; Mutex* _freelistLock; CMSBitMap* _bitMap; CMSMarkStack* _markStack; MarkRefsIntoAndScanClosure* _scanningClosure; public: ScanMarkedObjectsAgainCarefullyClosure(CMSCollector* collector, MemRegion span, CMSBitMap* bitMap, CMSMarkStack* markStack, MarkRefsIntoAndScanClosure* cl, bool should_yield): _collector(collector), _span(span), _yield(should_yield), _bitMap(bitMap), _markStack(markStack), _scanningClosure(cl) { } void do_object(oop p) { guarantee(false, "call do_object_careful instead"); } size_t do_object_careful(oop p) { guarantee(false, "Unexpected caller"); return 0; } size_t do_object_careful_m(oop p, MemRegion mr); void setFreelistLock(Mutex* m) { _freelistLock = m; _scanningClosure->set_freelistLock(m); } private: inline bool do_yield_check(); void do_yield_work(); }; class SurvivorSpacePrecleanClosure: public ObjectClosureCareful { CMSCollector* _collector; MemRegion _span; bool _yield; CMSBitMap* _bit_map; CMSMarkStack* _mark_stack; PushAndMarkClosure* _scanning_closure; unsigned int _before_count; public: SurvivorSpacePrecleanClosure(CMSCollector* collector, MemRegion span, CMSBitMap* bit_map, CMSMarkStack* mark_stack, PushAndMarkClosure* cl, unsigned int before_count, bool should_yield): _collector(collector), _span(span), _yield(should_yield), _bit_map(bit_map), _mark_stack(mark_stack), _scanning_closure(cl), _before_count(before_count) { } void do_object(oop p) { guarantee(false, "call do_object_careful instead"); } size_t do_object_careful(oop p); size_t do_object_careful_m(oop p, MemRegion mr) { guarantee(false, "Unexpected caller"); return 0; } private: inline void do_yield_check(); void do_yield_work(); }; // This closure is used to accomplish the sweeping work // after the second checkpoint but before the concurrent reset // phase. // // Terminology // left hand chunk (LHC) - block of one or more chunks currently being // coalesced. The LHC is available for coalescing with a new chunk. // right hand chunk (RHC) - block that is currently being swept that is // free or garbage that can be coalesced with the LHC. // _inFreeRange is true if there is currently a LHC // _lastFreeRangeCoalesced is true if the LHC consists of more than one chunk. // _freeRangeInFreeLists is true if the LHC is in the free lists. // _freeFinger is the address of the current LHC class SweepClosure: public BlkClosureCareful { CMSCollector* _collector; // collector doing the work ConcurrentMarkSweepGeneration* _g; // Generation being swept CompactibleFreeListSpace* _sp; // Space being swept HeapWord* _limit;// the address at or above which the sweep should stop // because we do not expect newly garbage blocks // eligible for sweeping past that address. Mutex* _freelistLock; // Free list lock (in space) CMSBitMap* _bitMap; // Marking bit map (in // generation) bool _inFreeRange; // Indicates if we are in the // midst of a free run bool _freeRangeInFreeLists; // Often, we have just found // a free chunk and started // a new free range; we do not // eagerly remove this chunk from // the free lists unless there is // a possibility of coalescing. // When true, this flag indicates // that the _freeFinger below // points to a potentially free chunk // that may still be in the free lists bool _lastFreeRangeCoalesced; // free range contains chunks // coalesced bool _yield; // Whether sweeping should be // done with yields. For instance // when done by the foreground // collector we shouldn't yield. HeapWord* _freeFinger; // When _inFreeRange is set, the // pointer to the "left hand // chunk" size_t _freeRangeSize; // When _inFreeRange is set, this // indicates the accumulated size // of the "left hand chunk" NOT_PRODUCT( size_t _numObjectsFreed; size_t _numWordsFreed; size_t _numObjectsLive; size_t _numWordsLive; size_t _numObjectsAlreadyFree; size_t _numWordsAlreadyFree; FreeChunk* _last_fc; ) private: // Code that is common to a free chunk or garbage when // encountered during sweeping. void do_post_free_or_garbage_chunk(FreeChunk *fc, size_t chunkSize); // Process a free chunk during sweeping. void do_already_free_chunk(FreeChunk *fc); // Work method called when processing an already free or a // freshly garbage chunk to do a lookahead and possibly a // premptive flush if crossing over _limit. void lookahead_and_flush(FreeChunk* fc, size_t chunkSize); // Process a garbage chunk during sweeping. size_t do_garbage_chunk(FreeChunk *fc); // Process a live chunk during sweeping. size_t do_live_chunk(FreeChunk* fc); // Accessors. HeapWord* freeFinger() const { return _freeFinger; } void set_freeFinger(HeapWord* v) { _freeFinger = v; } bool inFreeRange() const { return _inFreeRange; } void set_inFreeRange(bool v) { _inFreeRange = v; } bool lastFreeRangeCoalesced() const { return _lastFreeRangeCoalesced; } void set_lastFreeRangeCoalesced(bool v) { _lastFreeRangeCoalesced = v; } bool freeRangeInFreeLists() const { return _freeRangeInFreeLists; } void set_freeRangeInFreeLists(bool v) { _freeRangeInFreeLists = v; } // Initialize a free range. void initialize_free_range(HeapWord* freeFinger, bool freeRangeInFreeLists); // Return this chunk to the free lists. void flush_cur_free_chunk(HeapWord* chunk, size_t size); // Check if we should yield and do so when necessary. inline void do_yield_check(HeapWord* addr); // Yield void do_yield_work(HeapWord* addr); // Debugging/Printing void print_free_block_coalesced(FreeChunk* fc) const; public: SweepClosure(CMSCollector* collector, ConcurrentMarkSweepGeneration* g, CMSBitMap* bitMap, bool should_yield); ~SweepClosure() PRODUCT_RETURN; size_t do_blk_careful(HeapWord* addr); void print() const { print_on(tty); } void print_on(outputStream *st) const; }; // Closures related to weak references processing // During CMS' weak reference processing, this is a // work-routine/closure used to complete transitive // marking of objects as live after a certain point // in which an initial set has been completely accumulated. // This closure is currently used both during the final // remark stop-world phase, as well as during the concurrent // precleaning of the discovered reference lists. class CMSDrainMarkingStackClosure: public VoidClosure { CMSCollector* _collector; MemRegion _span; CMSMarkStack* _mark_stack; CMSBitMap* _bit_map; CMSKeepAliveClosure* _keep_alive; bool _concurrent_precleaning; public: CMSDrainMarkingStackClosure(CMSCollector* collector, MemRegion span, CMSBitMap* bit_map, CMSMarkStack* mark_stack, CMSKeepAliveClosure* keep_alive, bool cpc): _collector(collector), _span(span), _bit_map(bit_map), _mark_stack(mark_stack), _keep_alive(keep_alive), _concurrent_precleaning(cpc) { assert(_concurrent_precleaning == _keep_alive->concurrent_precleaning(), "Mismatch"); } void do_void(); }; // A parallel version of CMSDrainMarkingStackClosure above. class CMSParDrainMarkingStackClosure: public VoidClosure { CMSCollector* _collector; MemRegion _span; OopTaskQueue* _work_queue; CMSBitMap* _bit_map; CMSInnerParMarkAndPushClosure _mark_and_push; public: CMSParDrainMarkingStackClosure(CMSCollector* collector, MemRegion span, CMSBitMap* bit_map, OopTaskQueue* work_queue): _collector(collector), _span(span), _bit_map(bit_map), _work_queue(work_queue), _mark_and_push(collector, span, bit_map, work_queue) { } public: void trim_queue(uint max); void do_void(); }; // Allow yielding or short-circuiting of reference list // prelceaning work. class CMSPrecleanRefsYieldClosure: public YieldClosure { CMSCollector* _collector; void do_yield_work(); public: CMSPrecleanRefsYieldClosure(CMSCollector* collector): _collector(collector) {} virtual bool should_return(); }; // Convenience class that locks free list locks for given CMS collector class FreelistLocker: public StackObj { private: CMSCollector* _collector; public: FreelistLocker(CMSCollector* collector): _collector(collector) { _collector->getFreelistLocks(); } ~FreelistLocker() { _collector->releaseFreelistLocks(); } }; // Mark all dead objects in a given space. class MarkDeadObjectsClosure: public BlkClosure { const CMSCollector* _collector; const CompactibleFreeListSpace* _sp; CMSBitMap* _live_bit_map; CMSBitMap* _dead_bit_map; public: MarkDeadObjectsClosure(const CMSCollector* collector, const CompactibleFreeListSpace* sp, CMSBitMap *live_bit_map, CMSBitMap *dead_bit_map) : _collector(collector), _sp(sp), _live_bit_map(live_bit_map), _dead_bit_map(dead_bit_map) {} size_t do_blk(HeapWord* addr); }; class TraceCMSMemoryManagerStats : public TraceMemoryManagerStats { public: TraceCMSMemoryManagerStats(CMSCollector::CollectorState phase, GCCause::Cause cause); }; #endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_HPP