initial version

tschatzl · Jan 16, 2024 · 6f7e9b0 · 6f7e9b0
1 parent 2d7b94c
commit 6f7e9b0
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 13 deletions.
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
@@ -2444,6 +2444,13 @@ void G1CollectedHeap::prepare_for_mutator_after_young_collection() {
 
 void G1CollectedHeap::retire_tlabs() {
   ensure_parsability(true);
+  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next();) {
+    uint cached_pinned_region_idx = G1ThreadLocalData::last_pinned_region_idx(thread);
+    size_t pin_count = G1ThreadLocalData::set_new_last_pinned_region_idx(thread, 0, 0);
+    if (pin_count != 0) {
+      region_at(cached_pinned_region_idx)->add_pinned_object_count(pin_count);
+    }
+  }
 }
 
 void G1CollectedHeap::do_collection_pause_at_safepoint_helper() {

diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
@@ -266,15 +266,31 @@ inline void G1CollectedHeap::pin_object(JavaThread* thread, oop obj) {
   assert(obj != nullptr, "obj must not be null");
   assert(!is_gc_active(), "must not pin objects during a GC");
   assert(obj->is_typeArray(), "must be typeArray");
-  HeapRegion *r = heap_region_containing(obj);
-  r->increment_pinned_object_count();
+  HeapRegion* r = heap_region_containing(obj);
+
+  uint last_idx = G1ThreadLocalData::last_pinned_region_idx(thread);
+  if (last_idx == r->hrm_index()) {
+    G1ThreadLocalData::increment_pinned_region_count(thread);
+  } else {
+    // Flush old.
+    region_at(last_idx)->add_pinned_object_count(G1ThreadLocalData::pinned_region_count(thread));
+    G1ThreadLocalData::set_new_last_pinned_region_idx(thread, r->hrm_index(), (size_t)1);
+  }
 }
 
 inline void G1CollectedHeap::unpin_object(JavaThread* thread, oop obj) {
   assert(obj != nullptr, "obj must not be null");
   assert(!is_gc_active(), "must not unpin objects during a GC");
-  HeapRegion *r = heap_region_containing(obj);
-  r->decrement_pinned_object_count();
+  HeapRegion* r = heap_region_containing(obj);
+
+  uint last_idx = G1ThreadLocalData::last_pinned_region_idx(thread);
+  if (last_idx == r->hrm_index()) {
+    G1ThreadLocalData::decrement_pinned_region_count(thread);
+  } else {
+    // Flush old.
+    region_at(last_idx)->add_pinned_object_count(G1ThreadLocalData::pinned_region_count(thread));
+    G1ThreadLocalData::set_new_last_pinned_region_idx(thread, r->hrm_index(), (size_t)~0);
+  }
 }
 
 inline bool G1CollectedHeap::is_obj_dead(const oop obj) const {

diff --git a/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp b/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp
@@ -39,7 +39,9 @@ class G1ThreadLocalData {
 
   G1ThreadLocalData() :
       _satb_mark_queue(&G1BarrierSet::satb_mark_queue_set()),
-      _dirty_card_queue(&G1BarrierSet::dirty_card_queue_set()) {}
+      _dirty_card_queue(&G1BarrierSet::dirty_card_queue_set()),
+      _last_pinned_region_idx(0),
+      _pinned_region_count(0) {}
 
   static G1ThreadLocalData* data(Thread* thread) {
     assert(UseG1GC, "Sanity");
@@ -90,6 +92,33 @@ class G1ThreadLocalData {
   static ByteSize dirty_card_queue_buffer_offset() {
     return dirty_card_queue_offset() + G1DirtyCardQueue::byte_offset_of_buf();
   }
+
+  static uint last_pinned_region_idx(Thread* thread) {
+    return data(thread)->_last_pinned_region_idx;
+  }
+
+  static size_t pinned_region_count(Thread* thread) {
+    return data(thread)->_pinned_region_count;
+  }
+
+  static void increment_pinned_region_count(Thread* thread) {
+    data(thread)->_pinned_region_count++;
+  }
+
+  static void decrement_pinned_region_count(Thread* thread) {
+    data(thread)->_pinned_region_count--;
+  }
+
+  static size_t set_new_last_pinned_region_idx(Thread* thread, uint region_idx, size_t new_count) {
+    G1ThreadLocalData* d = data(thread);
+    size_t result = d->_pinned_region_count;
+    d->_last_pinned_region_idx = region_idx;
+    d->_pinned_region_count = new_count;
+    return result;
+  }
+
+  uint _last_pinned_region_idx;
+  size_t _pinned_region_count;
 };
 
 #endif // SHARE_GC_G1_G1THREADLOCALDATA_HPP
diff --git a/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp
@@ -63,6 +63,12 @@ class G1PreEvacuateCollectionSetBatchTask::JavaThreadRetireTLABAndFlushLogs : pu
 
       G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
       _refinement_stats += qset.concatenate_log_and_stats(thread);
+
+      uint cached_pinned_region_idx = G1ThreadLocalData::last_pinned_region_idx(thread);
+      size_t pin_count = G1ThreadLocalData::set_new_last_pinned_region_idx(thread, 0, 0);
+      if (pin_count != 0) {
+        G1CollectedHeap::heap()->region_at(cached_pinned_region_idx)->add_pinned_object_count(pin_count);
+      }
     }
   };
 

diff --git a/src/hotspot/share/gc/g1/heapRegion.hpp b/src/hotspot/share/gc/g1/heapRegion.hpp
@@ -302,8 +302,7 @@ class HeapRegion : public CHeapObj<mtGC> {
   static uint   LogOfHRGrainBytes;
   static uint   LogCardsPerRegion;
 
-  inline void increment_pinned_object_count();
-  inline void decrement_pinned_object_count();
+  inline void add_pinned_object_count(size_t value);
 
   static size_t GrainBytes;
   static size_t GrainWords;

diff --git a/src/hotspot/share/gc/g1/heapRegion.inline.hpp b/src/hotspot/share/gc/g1/heapRegion.inline.hpp
@@ -553,12 +553,8 @@ inline void HeapRegion::record_surv_words_in_group(size_t words_survived) {
   _surv_rate_group->record_surviving_words(age, words_survived);
 }
 
-inline void HeapRegion::increment_pinned_object_count() {
-  Atomic::add(&_pinned_object_count, (size_t)1, memory_order_relaxed);
-}
-
-inline void HeapRegion::decrement_pinned_object_count() {
-  Atomic::sub(&_pinned_object_count, (size_t)1, memory_order_relaxed);
+inline void HeapRegion::add_pinned_object_count(size_t value) {
+  Atomic::add(&_pinned_object_count, value, memory_order_relaxed);
 }
 
 #endif // SHARE_GC_G1_HEAPREGION_INLINE_HPP