From 6f7e9b004eaff3def19f9b12731a52ce8cb89d65 Mon Sep 17 00:00:00 2001 From: Thomas Schatzl Date: Mon, 15 Jan 2024 13:11:08 +0100 Subject: [PATCH] initial version --- src/hotspot/share/gc/g1/g1CollectedHeap.cpp | 7 +++++ .../share/gc/g1/g1CollectedHeap.inline.hpp | 24 +++++++++++--- src/hotspot/share/gc/g1/g1ThreadLocalData.hpp | 31 ++++++++++++++++++- .../share/gc/g1/g1YoungGCPreEvacuateTasks.cpp | 6 ++++ src/hotspot/share/gc/g1/heapRegion.hpp | 3 +- src/hotspot/share/gc/g1/heapRegion.inline.hpp | 8 ++--- 6 files changed, 66 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index e9032d915a212..a2c881a280435 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -2444,6 +2444,13 @@ void G1CollectedHeap::prepare_for_mutator_after_young_collection() { void G1CollectedHeap::retire_tlabs() { ensure_parsability(true); + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next();) { + uint cached_pinned_region_idx = G1ThreadLocalData::last_pinned_region_idx(thread); + size_t pin_count = G1ThreadLocalData::set_new_last_pinned_region_idx(thread, 0, 0); + if (pin_count != 0) { + region_at(cached_pinned_region_idx)->add_pinned_object_count(pin_count); + } + } } void G1CollectedHeap::do_collection_pause_at_safepoint_helper() { diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp index 7b81a93abe177..5970f19eb26d3 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp @@ -266,15 +266,31 @@ inline void G1CollectedHeap::pin_object(JavaThread* thread, oop obj) { assert(obj != nullptr, "obj must not be null"); assert(!is_gc_active(), "must not pin objects during a GC"); assert(obj->is_typeArray(), "must be typeArray"); - HeapRegion *r = heap_region_containing(obj); - r->increment_pinned_object_count(); + HeapRegion* r = heap_region_containing(obj); + + uint last_idx = G1ThreadLocalData::last_pinned_region_idx(thread); + if (last_idx == r->hrm_index()) { + G1ThreadLocalData::increment_pinned_region_count(thread); + } else { + // Flush old. + region_at(last_idx)->add_pinned_object_count(G1ThreadLocalData::pinned_region_count(thread)); + G1ThreadLocalData::set_new_last_pinned_region_idx(thread, r->hrm_index(), (size_t)1); + } } inline void G1CollectedHeap::unpin_object(JavaThread* thread, oop obj) { assert(obj != nullptr, "obj must not be null"); assert(!is_gc_active(), "must not unpin objects during a GC"); - HeapRegion *r = heap_region_containing(obj); - r->decrement_pinned_object_count(); + HeapRegion* r = heap_region_containing(obj); + + uint last_idx = G1ThreadLocalData::last_pinned_region_idx(thread); + if (last_idx == r->hrm_index()) { + G1ThreadLocalData::decrement_pinned_region_count(thread); + } else { + // Flush old. + region_at(last_idx)->add_pinned_object_count(G1ThreadLocalData::pinned_region_count(thread)); + G1ThreadLocalData::set_new_last_pinned_region_idx(thread, r->hrm_index(), (size_t)~0); + } } inline bool G1CollectedHeap::is_obj_dead(const oop obj) const { diff --git a/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp b/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp index e17a8c7960a4d..9fc8a2e76b69f 100644 --- a/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp +++ b/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp @@ -39,7 +39,9 @@ class G1ThreadLocalData { G1ThreadLocalData() : _satb_mark_queue(&G1BarrierSet::satb_mark_queue_set()), - _dirty_card_queue(&G1BarrierSet::dirty_card_queue_set()) {} + _dirty_card_queue(&G1BarrierSet::dirty_card_queue_set()), + _last_pinned_region_idx(0), + _pinned_region_count(0) {} static G1ThreadLocalData* data(Thread* thread) { assert(UseG1GC, "Sanity"); @@ -90,6 +92,33 @@ class G1ThreadLocalData { static ByteSize dirty_card_queue_buffer_offset() { return dirty_card_queue_offset() + G1DirtyCardQueue::byte_offset_of_buf(); } + + static uint last_pinned_region_idx(Thread* thread) { + return data(thread)->_last_pinned_region_idx; + } + + static size_t pinned_region_count(Thread* thread) { + return data(thread)->_pinned_region_count; + } + + static void increment_pinned_region_count(Thread* thread) { + data(thread)->_pinned_region_count++; + } + + static void decrement_pinned_region_count(Thread* thread) { + data(thread)->_pinned_region_count--; + } + + static size_t set_new_last_pinned_region_idx(Thread* thread, uint region_idx, size_t new_count) { + G1ThreadLocalData* d = data(thread); + size_t result = d->_pinned_region_count; + d->_last_pinned_region_idx = region_idx; + d->_pinned_region_count = new_count; + return result; + } + + uint _last_pinned_region_idx; + size_t _pinned_region_count; }; #endif // SHARE_GC_G1_G1THREADLOCALDATA_HPP diff --git a/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp index d3a6436e432f5..e4e6a675286dd 100644 --- a/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp +++ b/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp @@ -63,6 +63,12 @@ class G1PreEvacuateCollectionSetBatchTask::JavaThreadRetireTLABAndFlushLogs : pu G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set(); _refinement_stats += qset.concatenate_log_and_stats(thread); + + uint cached_pinned_region_idx = G1ThreadLocalData::last_pinned_region_idx(thread); + size_t pin_count = G1ThreadLocalData::set_new_last_pinned_region_idx(thread, 0, 0); + if (pin_count != 0) { + G1CollectedHeap::heap()->region_at(cached_pinned_region_idx)->add_pinned_object_count(pin_count); + } } }; diff --git a/src/hotspot/share/gc/g1/heapRegion.hpp b/src/hotspot/share/gc/g1/heapRegion.hpp index 725433215c444..f4b8836c8db44 100644 --- a/src/hotspot/share/gc/g1/heapRegion.hpp +++ b/src/hotspot/share/gc/g1/heapRegion.hpp @@ -302,8 +302,7 @@ class HeapRegion : public CHeapObj { static uint LogOfHRGrainBytes; static uint LogCardsPerRegion; - inline void increment_pinned_object_count(); - inline void decrement_pinned_object_count(); + inline void add_pinned_object_count(size_t value); static size_t GrainBytes; static size_t GrainWords; diff --git a/src/hotspot/share/gc/g1/heapRegion.inline.hpp b/src/hotspot/share/gc/g1/heapRegion.inline.hpp index 4a42b9221829c..6e458dc85716c 100644 --- a/src/hotspot/share/gc/g1/heapRegion.inline.hpp +++ b/src/hotspot/share/gc/g1/heapRegion.inline.hpp @@ -553,12 +553,8 @@ inline void HeapRegion::record_surv_words_in_group(size_t words_survived) { _surv_rate_group->record_surviving_words(age, words_survived); } -inline void HeapRegion::increment_pinned_object_count() { - Atomic::add(&_pinned_object_count, (size_t)1, memory_order_relaxed); -} - -inline void HeapRegion::decrement_pinned_object_count() { - Atomic::sub(&_pinned_object_count, (size_t)1, memory_order_relaxed); +inline void HeapRegion::add_pinned_object_count(size_t value) { + Atomic::add(&_pinned_object_count, value, memory_order_relaxed); } #endif // SHARE_GC_G1_HEAPREGION_INLINE_HPP