optionally use bot

Author: Thomas Schatzl

src/hotspot/share/gc/serial/serialCompressor.cpp

@@ -174,36 +174,47 @@ class SCCompacter {
     return _spaces[index]._compaction_top;
   }
 
-  // Get space at index.
+  // Get space at index.build_
   ContiguousSpace* get_space(uint index) const {
     return _spaces[index]._space;
   }
 
   // Find the start of the first object at or after addr (but not after limit).
-  HeapWord* first_object_in_block(HeapWord* addr, HeapWord* start, HeapWord* limit) {
+  HeapWord* first_object_in_block(HeapWord* addr, HeapWord* start, HeapWord* limit, TenuredSpace* bot_provider) {
     if (addr >= limit) return limit;
     if (!_mark_bitmap.is_marked(addr)) {
       // Easy: find next marked address.
       return _mark_bitmap.get_next_marked_addr(addr, limit);
     } else {
-      // Find beginning of live chunk.
-      HeapWord* current = _mark_bitmap.get_last_unmarked_addr(start, addr);
-      if (current == addr) {
-        // No clear bit found before search range, start of range because that must
-        // be the previous object.
-        current = start;
-      } else {
-        // Use first marked word, this must be an object.
-        assert(!_mark_bitmap.is_marked(current), "must not be marked");
-        current = current + 1;
-      }
       // Forward-search to first object >= addr.
-      while (current < addr) {
-        size_t obj_size = cast_to_oop(current)->size();
-        current += obj_size;
+      if (bot_provider != nullptr) {
+        HeapWord* reaching_into = bot_provider->block_start_const(addr);
+        if (reaching_into < addr) {
+          reaching_into = reaching_into + cast_to_oop<HeapWord*>(reaching_into)->size();
+        }
+        assert(reaching_into >= addr, "must be");
+        assert(reaching_into == limit || oopDesc::is_oop(cast_to_oop<HeapWord*>(reaching_into)), "must be");
+        return reaching_into;
+      } else {
+        // Find beginning of live chunk.
+        HeapWord* current = _mark_bitmap.get_last_unmarked_addr(start, addr);
+        if (current == addr) {
+          // No clear bit found before search range, start of range because that must
+          // be the previous object.
+          current = start;
+        } else {
+          // Use first marked word, this must be an object.
+          assert(!_mark_bitmap.is_marked(current), "must not be marked");
+          current = current + 1;
+        }
+
+        while (current < addr) {
+          size_t obj_size = cast_to_oop(current)->size();
+          current += obj_size;
+        }
+        assert(current >= addr, "found object start must be >= addr");
+        return MIN2(current, limit);
       }
-      assert(current >= addr, "found object start must be >= addr");
-      return MIN2(current, limit);
     }
   }
 
@@ -213,6 +224,7 @@ class SCCompacter {
     HeapWord* const bottom = space->bottom();
     HeapWord* const top = space->top();
 
+    TenuredSpace* bot_space = UseNewCode && space == SerialHeap::heap()->old_gen()->space() ? SerialHeap::heap()->old_gen()->space() : nullptr;
     // Clear table (only required for assertion in forwardee()).
     DEBUG_ONLY(clear(bottom, top);)
 
@@ -229,7 +241,7 @@ class SCCompacter {
       // Find first object that starts after this block and count
       // live words up to that object. This is how many words we
       // must fit into the current compaction space.
-      HeapWord* first_obj_after_block = first_object_in_block(current + words_per_block(), first_obj_this_block, top);
+      HeapWord* first_obj_after_block = first_object_in_block(current + words_per_block(), first_obj_this_block, top, bot_space);
       size_t live_in_block = _mark_bitmap.count_marked_words(first_obj_this_block, first_obj_after_block);
       while (live_in_block > pointer_delta(_spaces[_index]._space->end(),
                                            compact_top)) {
@@ -398,14 +410,16 @@ void SCCompacter::compact_space(uint idx) const {
   // Visit all live objects in the space.
   while (current < top) {
     assert(_mark_bitmap.is_marked(current), "must be marked");
-
-    // Copy the object.
-    size_t size = cast_to_oop(current)->size();
+    // Copy the whole chunk.
     HeapWord* compact_to = forwardee(current);
+
+    HeapWord* next_dead = _mark_bitmap.get_next_unmarked_addr(current, top);
+    size_t size = pointer_delta(next_dead, current);
+
     Copy::aligned_conjoint_words(current, compact_to, size);
 
     // Advance to next live object.
-    current = _mark_bitmap.get_next_marked_addr(current + size, top);
+    current = _mark_bitmap.get_next_marked_addr(next_dead, top);
   }
 
   // Reset top and unused memory
@@ -637,7 +651,7 @@ class SCKeepAliveClosure: public OopClosure {
 
 bool SerialCompressor::mark_object(oop obj) {
   HeapWord* addr = cast_from_oop<HeapWord*>(obj);
-  if (!_mark_bitmap.is_marked(addr)) {
+  if (_mark_bitmap.set_mark(addr)) {
     if (StringDedup::is_enabled() &&
         java_lang_String::is_instance(obj) &&
         SerialStringDedup::is_candidate_from_mark(obj)) {
@@ -648,7 +662,6 @@ bool SerialCompressor::mark_object(oop obj) {
     // which might include important class information.
     ContinuationGCSupport::transform_stack_chunk(obj);
 
-    _mark_bitmap.mark_range(addr, obj->size());
     return true;
   } else {
     return false;
@@ -674,9 +687,11 @@ void SerialCompressor::follow_object(oop obj) {
   assert(_mark_bitmap.is_marked(obj), "p must be marked");
   if (obj->is_objArray()) {
     follow_array((objArrayOop)obj);
+    _mark_bitmap.mark_range(cast_from_oop<HeapWord*>(obj), ((objArrayOop)obj)->size());
   } else {
     SCMarkAndPushClosure mark_and_push_closure(ClassLoaderData::_claim_stw_fullgc_mark, *this);
-    obj->oop_iterate(&mark_and_push_closure);
+    size_t size = obj->oop_iterate_size(&mark_and_push_closure);
+    _mark_bitmap.mark_range(cast_from_oop<HeapWord*>(obj), size);
   }
 }
 

src/hotspot/share/gc/shared/markBitMap.hpp

@@ -78,6 +78,7 @@ class MarkBitMap {
     return _bm.at(addr_to_offset(addr));
   }
 
+  inline bool set_mark(HeapWord* addr);
   // Return the address corresponding to the next marked bit at or after
   // "addr", and before "limit", if "limit" is non-null.  If there is no
   // such bit, returns "limit" if that is non-null, or else "endWord()".

src/hotspot/share/gc/shared/markBitMap.inline.hpp

@@ -33,6 +33,10 @@
 #include "utilities/align.hpp"
 #include "utilities/bitMap.inline.hpp"
 
+inline bool MarkBitMap::set_mark(HeapWord* addr) {
+  return _bm.maybe_set_bit(addr_to_offset(addr));
+}
+
 inline HeapWord* MarkBitMap::get_next_marked_addr(const HeapWord* const addr,
                                                   HeapWord* const limit) const {
   assert(limit != nullptr, "limit must not be null");

src/hotspot/share/utilities/bitMap.hpp

@@ -214,6 +214,7 @@ class BitMap {
   // memory_order must be memory_order_relaxed or memory_order_acquire.
   bool par_at(idx_t index, atomic_memory_order memory_order = memory_order_acquire) const;
 
+  inline bool maybe_set_bit(idx_t bit);
   // Set or clear the specified bit.
   inline void set_bit(idx_t bit);
   inline void clear_bit(idx_t bit);

src/hotspot/share/utilities/bitMap.inline.hpp

@@ -33,6 +33,14 @@
 #include "utilities/population_count.hpp"
 #include "utilities/powerOfTwo.hpp"
 
+inline bool BitMap::maybe_set_bit(idx_t bit) {
+  verify_index(bit);
+  bm_word_t old_value = *word_addr(bit);
+  bm_word_t new_value = old_value | bit_mask(bit);
+  *word_addr(bit) = new_value;
+  return new_value != old_value;
+}
+
 inline void BitMap::set_bit(idx_t bit) {
   verify_index(bit);
   *word_addr(bit) |= bit_mask(bit);