diff --git a/src/hotspot/share/gc/serial/serialCompressor.cpp b/src/hotspot/share/gc/serial/serialCompressor.cpp index 363a0ec93b61a..23c03f134ddb7 100644 --- a/src/hotspot/share/gc/serial/serialCompressor.cpp +++ b/src/hotspot/share/gc/serial/serialCompressor.cpp @@ -174,36 +174,47 @@ class SCCompacter { return _spaces[index]._compaction_top; } - // Get space at index. + // Get space at index.build_ ContiguousSpace* get_space(uint index) const { return _spaces[index]._space; } // Find the start of the first object at or after addr (but not after limit). - HeapWord* first_object_in_block(HeapWord* addr, HeapWord* start, HeapWord* limit) { + HeapWord* first_object_in_block(HeapWord* addr, HeapWord* start, HeapWord* limit, TenuredSpace* bot_provider) { if (addr >= limit) return limit; if (!_mark_bitmap.is_marked(addr)) { // Easy: find next marked address. return _mark_bitmap.get_next_marked_addr(addr, limit); } else { - // Find beginning of live chunk. - HeapWord* current = _mark_bitmap.get_last_unmarked_addr(start, addr); - if (current == addr) { - // No clear bit found before search range, start of range because that must - // be the previous object. - current = start; - } else { - // Use first marked word, this must be an object. - assert(!_mark_bitmap.is_marked(current), "must not be marked"); - current = current + 1; - } // Forward-search to first object >= addr. - while (current < addr) { - size_t obj_size = cast_to_oop(current)->size(); - current += obj_size; + if (bot_provider != nullptr) { + HeapWord* reaching_into = bot_provider->block_start_const(addr); + if (reaching_into < addr) { + reaching_into = reaching_into + cast_to_oop(reaching_into)->size(); + } + assert(reaching_into >= addr, "must be"); + assert(reaching_into == limit || oopDesc::is_oop(cast_to_oop(reaching_into)), "must be"); + return reaching_into; + } else { + // Find beginning of live chunk. + HeapWord* current = _mark_bitmap.get_last_unmarked_addr(start, addr); + if (current == addr) { + // No clear bit found before search range, start of range because that must + // be the previous object. + current = start; + } else { + // Use first marked word, this must be an object. + assert(!_mark_bitmap.is_marked(current), "must not be marked"); + current = current + 1; + } + + while (current < addr) { + size_t obj_size = cast_to_oop(current)->size(); + current += obj_size; + } + assert(current >= addr, "found object start must be >= addr"); + return MIN2(current, limit); } - assert(current >= addr, "found object start must be >= addr"); - return MIN2(current, limit); } } @@ -213,6 +224,7 @@ class SCCompacter { HeapWord* const bottom = space->bottom(); HeapWord* const top = space->top(); + TenuredSpace* bot_space = UseNewCode && space == SerialHeap::heap()->old_gen()->space() ? SerialHeap::heap()->old_gen()->space() : nullptr; // Clear table (only required for assertion in forwardee()). DEBUG_ONLY(clear(bottom, top);) @@ -229,7 +241,7 @@ class SCCompacter { // Find first object that starts after this block and count // live words up to that object. This is how many words we // must fit into the current compaction space. - HeapWord* first_obj_after_block = first_object_in_block(current + words_per_block(), first_obj_this_block, top); + HeapWord* first_obj_after_block = first_object_in_block(current + words_per_block(), first_obj_this_block, top, bot_space); size_t live_in_block = _mark_bitmap.count_marked_words(first_obj_this_block, first_obj_after_block); while (live_in_block > pointer_delta(_spaces[_index]._space->end(), compact_top)) { @@ -398,14 +410,16 @@ void SCCompacter::compact_space(uint idx) const { // Visit all live objects in the space. while (current < top) { assert(_mark_bitmap.is_marked(current), "must be marked"); - - // Copy the object. - size_t size = cast_to_oop(current)->size(); + // Copy the whole chunk. HeapWord* compact_to = forwardee(current); + + HeapWord* next_dead = _mark_bitmap.get_next_unmarked_addr(current, top); + size_t size = pointer_delta(next_dead, current); + Copy::aligned_conjoint_words(current, compact_to, size); // Advance to next live object. - current = _mark_bitmap.get_next_marked_addr(current + size, top); + current = _mark_bitmap.get_next_marked_addr(next_dead, top); } // Reset top and unused memory @@ -637,7 +651,7 @@ class SCKeepAliveClosure: public OopClosure { bool SerialCompressor::mark_object(oop obj) { HeapWord* addr = cast_from_oop(obj); - if (!_mark_bitmap.is_marked(addr)) { + if (_mark_bitmap.set_mark(addr)) { if (StringDedup::is_enabled() && java_lang_String::is_instance(obj) && SerialStringDedup::is_candidate_from_mark(obj)) { @@ -648,7 +662,6 @@ bool SerialCompressor::mark_object(oop obj) { // which might include important class information. ContinuationGCSupport::transform_stack_chunk(obj); - _mark_bitmap.mark_range(addr, obj->size()); return true; } else { return false; @@ -674,9 +687,11 @@ void SerialCompressor::follow_object(oop obj) { assert(_mark_bitmap.is_marked(obj), "p must be marked"); if (obj->is_objArray()) { follow_array((objArrayOop)obj); + _mark_bitmap.mark_range(cast_from_oop(obj), ((objArrayOop)obj)->size()); } else { SCMarkAndPushClosure mark_and_push_closure(ClassLoaderData::_claim_stw_fullgc_mark, *this); - obj->oop_iterate(&mark_and_push_closure); + size_t size = obj->oop_iterate_size(&mark_and_push_closure); + _mark_bitmap.mark_range(cast_from_oop(obj), size); } } diff --git a/src/hotspot/share/gc/shared/markBitMap.hpp b/src/hotspot/share/gc/shared/markBitMap.hpp index 5207ebaaf9ec3..907ddaa4db66e 100644 --- a/src/hotspot/share/gc/shared/markBitMap.hpp +++ b/src/hotspot/share/gc/shared/markBitMap.hpp @@ -78,6 +78,7 @@ class MarkBitMap { return _bm.at(addr_to_offset(addr)); } + inline bool set_mark(HeapWord* addr); // Return the address corresponding to the next marked bit at or after // "addr", and before "limit", if "limit" is non-null. If there is no // such bit, returns "limit" if that is non-null, or else "endWord()". diff --git a/src/hotspot/share/gc/shared/markBitMap.inline.hpp b/src/hotspot/share/gc/shared/markBitMap.inline.hpp index 436b24535216d..2c2fbca1636ec 100644 --- a/src/hotspot/share/gc/shared/markBitMap.inline.hpp +++ b/src/hotspot/share/gc/shared/markBitMap.inline.hpp @@ -33,6 +33,10 @@ #include "utilities/align.hpp" #include "utilities/bitMap.inline.hpp" +inline bool MarkBitMap::set_mark(HeapWord* addr) { + return _bm.maybe_set_bit(addr_to_offset(addr)); +} + inline HeapWord* MarkBitMap::get_next_marked_addr(const HeapWord* const addr, HeapWord* const limit) const { assert(limit != nullptr, "limit must not be null"); diff --git a/src/hotspot/share/utilities/bitMap.hpp b/src/hotspot/share/utilities/bitMap.hpp index 8e6f57a4c257a..4f8c987ebab2c 100644 --- a/src/hotspot/share/utilities/bitMap.hpp +++ b/src/hotspot/share/utilities/bitMap.hpp @@ -214,6 +214,7 @@ class BitMap { // memory_order must be memory_order_relaxed or memory_order_acquire. bool par_at(idx_t index, atomic_memory_order memory_order = memory_order_acquire) const; + inline bool maybe_set_bit(idx_t bit); // Set or clear the specified bit. inline void set_bit(idx_t bit); inline void clear_bit(idx_t bit); diff --git a/src/hotspot/share/utilities/bitMap.inline.hpp b/src/hotspot/share/utilities/bitMap.inline.hpp index 00659a48046fe..2497b97af5161 100644 --- a/src/hotspot/share/utilities/bitMap.inline.hpp +++ b/src/hotspot/share/utilities/bitMap.inline.hpp @@ -33,6 +33,14 @@ #include "utilities/population_count.hpp" #include "utilities/powerOfTwo.hpp" +inline bool BitMap::maybe_set_bit(idx_t bit) { + verify_index(bit); + bm_word_t old_value = *word_addr(bit); + bm_word_t new_value = old_value | bit_mask(bit); + *word_addr(bit) = new_value; + return new_value != old_value; +} + inline void BitMap::set_bit(idx_t bit) { verify_index(bit); *word_addr(bit) |= bit_mask(bit);