From b59b332c739016770f970d78c975fc574b9ca87f Mon Sep 17 00:00:00 2001 From: Thomas Schatzl Date: Mon, 15 Jan 2024 13:11:08 +0100 Subject: [PATCH] initial version count atomic instructions executed Refactoring Fix -1 casting refactoring, fix loosing pin counts on thread exit refactoring, additional verification fix compilation and issue with parameter evaluation ordering Some more checking Cleanup missing comma in copyright newline at end of file in g1RegionPinCache.cpp --- src/hotspot/share/gc/g1/g1CollectedHeap.cpp | 11 +++- src/hotspot/share/gc/g1/g1CollectedHeap.hpp | 6 +- .../share/gc/g1/g1CollectedHeap.inline.hpp | 34 ++++++++-- src/hotspot/share/gc/g1/g1ConcurrentMark.cpp | 2 +- src/hotspot/share/gc/g1/g1FullCollector.cpp | 3 +- src/hotspot/share/gc/g1/g1RegionPinCache.cpp | 36 +++++++++++ src/hotspot/share/gc/g1/g1RegionPinCache.hpp | 62 +++++++++++++++++++ src/hotspot/share/gc/g1/g1ThreadLocalData.hpp | 37 ++++++++++- src/hotspot/share/gc/g1/g1YoungCollector.cpp | 6 ++ .../share/gc/g1/g1YoungGCPreEvacuateTasks.cpp | 13 +++- src/hotspot/share/gc/g1/heapRegion.hpp | 3 +- src/hotspot/share/gc/g1/heapRegion.inline.hpp | 10 ++- 12 files changed, 201 insertions(+), 22 deletions(-) create mode 100644 src/hotspot/share/gc/g1/g1RegionPinCache.cpp create mode 100644 src/hotspot/share/gc/g1/g1RegionPinCache.hpp diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index 7a630c27f7963..07186ef856a52 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -2471,6 +2471,15 @@ void G1CollectedHeap::retire_tlabs() { ensure_parsability(true); } +void G1CollectedHeap::flush_region_pin_cache() { + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) { + Pair stats = G1ThreadLocalData::get_and_reset_pin_cache(thread); + if (stats.second != 0) { + region_at(stats.first)->add_pinned_object_count(stats.second); + } + } +} + void G1CollectedHeap::do_collection_pause_at_safepoint_helper() { ResourceMark rm; diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp index 58ac36a734f7f..4245fb1b0cbfb 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -775,6 +775,10 @@ class G1CollectedHeap : public CollectedHeap { void retire_tlabs(); + // Update all region's pin counts from the per-thread caches and resets them. + // Must be called before any decision based on pin counts. + void flush_region_pin_cache(); + void expand_heap_after_young_collection(); // Update object copying statistics. void record_obj_copy_mem_stats(); diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp index 7b81a93abe177..45f9fcbc116f1 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp @@ -1,5 +1,5 @@ -/* - * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved. + /* + * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -266,15 +266,37 @@ inline void G1CollectedHeap::pin_object(JavaThread* thread, oop obj) { assert(obj != nullptr, "obj must not be null"); assert(!is_gc_active(), "must not pin objects during a GC"); assert(obj->is_typeArray(), "must be typeArray"); - HeapRegion *r = heap_region_containing(obj); - r->increment_pinned_object_count(); + HeapRegion* r = heap_region_containing(obj); + uint obj_region_idx = r->hrm_index(); + + uint cache_region_idx = G1ThreadLocalData::cached_pinned_region_idx(thread); + if (cache_region_idx == obj_region_idx) { + G1ThreadLocalData::inc_cached_pin_count(thread); + } else { + // Flush old. + size_t cache_pin_count = G1ThreadLocalData::get_and_set_pin_cache(thread, obj_region_idx, (size_t)1); + if (cache_pin_count != 0) { + region_at(cache_region_idx)->add_pinned_object_count(cache_pin_count); + } + } } inline void G1CollectedHeap::unpin_object(JavaThread* thread, oop obj) { assert(obj != nullptr, "obj must not be null"); assert(!is_gc_active(), "must not unpin objects during a GC"); - HeapRegion *r = heap_region_containing(obj); - r->decrement_pinned_object_count(); + HeapRegion* r = heap_region_containing(obj); + uint obj_region_idx = r->hrm_index(); + + uint cache_region_idx = G1ThreadLocalData::cached_pinned_region_idx(thread); + if (cache_region_idx == obj_region_idx) { + G1ThreadLocalData::dec_cached_pin_count(thread); + } else { + // Flush old. + size_t cache_pin_count = G1ThreadLocalData::get_and_set_pin_cache(thread, obj_region_idx, ~(size_t)0); + if (cache_pin_count != 0) { + region_at(cache_region_idx)->add_pinned_object_count(cache_pin_count); + } + } } inline bool G1CollectedHeap::is_obj_dead(const oop obj) const { diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp index 533817f97246c..cb7f90b08477c 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp index 7f8b4221cab6b..430d6f327886b 100644 --- a/src/hotspot/share/gc/g1/g1FullCollector.cpp +++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -191,6 +191,7 @@ void G1FullCollector::prepare_collection() { _heap->gc_prologue(true); _heap->retire_tlabs(); + _heap->flush_region_pin_cache(); _heap->prepare_heap_for_full_collection(); PrepareRegionsClosure cl(this); diff --git a/src/hotspot/share/gc/g1/g1RegionPinCache.cpp b/src/hotspot/share/gc/g1/g1RegionPinCache.cpp new file mode 100644 index 0000000000000..4d55ef592820e --- /dev/null +++ b/src/hotspot/share/gc/g1/g1RegionPinCache.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/g1/g1RegionPinCache.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/heapRegion.inline.hpp" + +G1RegionPinCache::~G1RegionPinCache() { + if (_count != 0) { + G1CollectedHeap::heap()->region_at(_region_idx)->add_pinned_object_count(_count); + } + get_and_reset(); +} diff --git a/src/hotspot/share/gc/g1/g1RegionPinCache.hpp b/src/hotspot/share/gc/g1/g1RegionPinCache.hpp new file mode 100644 index 0000000000000..1ca8384fc555a --- /dev/null +++ b/src/hotspot/share/gc/g1/g1RegionPinCache.hpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_G1_G1REGIONPINCACHE_HPP +#define SHARE_GC_G1_G1REGIONPINCACHE_HPP + +#include "memory/allocation.hpp" +#include "utilities/pair.hpp" +#include "utilities/globalDefinitions.hpp" + +// Holds the pinned object count increment for the given region for a Java thread. +// I.e. the _count value may actually be negative temporarily if pinning operations +// were interleaved between two regions. +class G1RegionPinCache : public StackObj { + uint _region_idx; + size_t _count; + +public: + G1RegionPinCache() : _region_idx(0), _count(0) { } + ~G1RegionPinCache(); + + uint region_idx() const { return _region_idx; } + size_t count() const { return _count; } + + void inc_count() { ++_count; } + void dec_count() { --_count; } + + size_t get_and_set(uint new_region_idx, size_t new_count) { + size_t result = _count; + _region_idx = new_region_idx; + _count = new_count; + return result; + } + + // Gets current region and pin count and resets the values to defaults. + Pair get_and_reset() { + return Pair { _region_idx, get_and_set(0, 0) }; + } +}; + +#endif /* SHARE_GC_G1_G1REGIONPINCACHE_HPP */ diff --git a/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp b/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp index e17a8c7960a4d..ec065f3356707 100644 --- a/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp +++ b/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,6 +26,7 @@ #include "gc/g1/g1BarrierSet.hpp" #include "gc/g1/g1DirtyCardQueue.hpp" +#include "gc/g1/g1RegionPinCache.hpp" #include "gc/shared/gc_globals.hpp" #include "gc/shared/satbMarkQueue.hpp" #include "runtime/javaThread.hpp" @@ -37,9 +38,15 @@ class G1ThreadLocalData { SATBMarkQueue _satb_mark_queue; G1DirtyCardQueue _dirty_card_queue; + // Per-thread cache of pinned object count to reduce atomic operation traffic + // due to region pinning. Holds the last region where the mutator pinned an + // object and the number of pin operations since the last change of the region. + G1RegionPinCache _pin_cache; + G1ThreadLocalData() : _satb_mark_queue(&G1BarrierSet::satb_mark_queue_set()), - _dirty_card_queue(&G1BarrierSet::dirty_card_queue_set()) {} + _dirty_card_queue(&G1BarrierSet::dirty_card_queue_set()), + _pin_cache() {} static G1ThreadLocalData* data(Thread* thread) { assert(UseG1GC, "Sanity"); @@ -90,6 +97,32 @@ class G1ThreadLocalData { static ByteSize dirty_card_queue_buffer_offset() { return dirty_card_queue_offset() + G1DirtyCardQueue::byte_offset_of_buf(); } + + static uint cached_pinned_region_idx(Thread* thread) { + return data(thread)->_pin_cache.region_idx(); + } + +#ifdef ASSERT + static size_t cached_pin_count(Thread* thread) { + return data(thread)->_pin_cache.count(); + } +#endif + + static void inc_cached_pin_count(Thread* thread) { + data(thread)->_pin_cache.inc_count(); + } + + static void dec_cached_pin_count(Thread* thread) { + data(thread)->_pin_cache.dec_count(); + } + + static size_t get_and_set_pin_cache(Thread* thread, uint region_idx, size_t new_count) { + return data(thread)->_pin_cache.get_and_set(region_idx, new_count); + } + + static Pair get_and_reset_pin_cache(Thread* thread) { + return data(thread)->_pin_cache.get_and_reset(); + } }; #endif // SHARE_GC_G1_G1THREADLOCALDATA_HPP diff --git a/src/hotspot/share/gc/g1/g1YoungCollector.cpp b/src/hotspot/share/gc/g1/g1YoungCollector.cpp index b52bfdfbb0817..f7d024e77f734 100644 --- a/src/hotspot/share/gc/g1/g1YoungCollector.cpp +++ b/src/hotspot/share/gc/g1/g1YoungCollector.cpp @@ -1087,6 +1087,12 @@ void G1YoungCollector::collect() { pre_evacuate_collection_set(jtm.evacuation_info()); +#ifdef ASSERT + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) { + assert(G1ThreadLocalData::cached_pin_count(thread) == 0, "must be flushed"); + } +#endif + G1ParScanThreadStateSet per_thread_states(_g1h, workers()->active_workers(), collection_set(), diff --git a/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp index d3a6436e432f5..0265d966f3e67 100644 --- a/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp +++ b/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,6 +28,7 @@ #include "gc/g1/g1ConcurrentRefineStats.hpp" #include "gc/g1/g1DirtyCardQueue.hpp" #include "gc/g1/g1YoungGCPreEvacuateTasks.hpp" +#include "gc/g1/g1ThreadLocalData.hpp" #include "gc/shared/barrierSet.inline.hpp" #include "gc/shared/threadLocalAllocBuffer.inline.hpp" #include "memory/allocation.inline.hpp" @@ -57,12 +58,18 @@ class G1PreEvacuateCollectionSetBatchTask::JavaThreadRetireTLABAndFlushLogs : pu assert(thread->is_Java_thread(), "must be"); // Flushes deferred card marks, so must precede concatenating logs. BarrierSet::barrier_set()->make_parsable((JavaThread*)thread); + // Retire TLABs. if (UseTLAB) { thread->tlab().retire(&_tlab_stats); } - + // Concatenate logs. G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set(); _refinement_stats += qset.concatenate_log_and_stats(thread); + // Flush region pincount cache. + Pair region_pin_cache = G1ThreadLocalData::get_and_reset_pin_cache(thread); + if (region_pin_cache.second != 0) { + G1CollectedHeap::heap()->region_at(region_pin_cache.first)->add_pinned_object_count(region_pin_cache.second); + } } }; @@ -132,6 +139,8 @@ class G1PreEvacuateCollectionSetBatchTask::NonJavaThreadFlushLogs : public G1Abs void do_thread(Thread* thread) override { G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set(); _refinement_stats += qset.concatenate_log_and_stats(thread); + + assert(G1ThreadLocalData::cached_pin_count(thread) == 0, "NonJava thread has pinned Java objects"); } } _tc; diff --git a/src/hotspot/share/gc/g1/heapRegion.hpp b/src/hotspot/share/gc/g1/heapRegion.hpp index 725433215c444..f4b8836c8db44 100644 --- a/src/hotspot/share/gc/g1/heapRegion.hpp +++ b/src/hotspot/share/gc/g1/heapRegion.hpp @@ -302,8 +302,7 @@ class HeapRegion : public CHeapObj { static uint LogOfHRGrainBytes; static uint LogCardsPerRegion; - inline void increment_pinned_object_count(); - inline void decrement_pinned_object_count(); + inline void add_pinned_object_count(size_t value); static size_t GrainBytes; static size_t GrainWords; diff --git a/src/hotspot/share/gc/g1/heapRegion.inline.hpp b/src/hotspot/share/gc/g1/heapRegion.inline.hpp index 4a42b9221829c..608cc2025a2a8 100644 --- a/src/hotspot/share/gc/g1/heapRegion.inline.hpp +++ b/src/hotspot/share/gc/g1/heapRegion.inline.hpp @@ -553,12 +553,10 @@ inline void HeapRegion::record_surv_words_in_group(size_t words_survived) { _surv_rate_group->record_surviving_words(age, words_survived); } -inline void HeapRegion::increment_pinned_object_count() { - Atomic::add(&_pinned_object_count, (size_t)1, memory_order_relaxed); -} - -inline void HeapRegion::decrement_pinned_object_count() { - Atomic::sub(&_pinned_object_count, (size_t)1, memory_order_relaxed); +inline void HeapRegion::add_pinned_object_count(size_t value) { + assert(value != 0, "wasted effort"); + assert(!is_free(), "trying to pin free region %u, adding %zu", hrm_index(), value); + Atomic::add(&_pinned_object_count, value, memory_order_relaxed); } #endif // SHARE_GC_G1_HEAPREGION_INLINE_HPP