initial version

count atomic instructions executed

Refactoring

Fix -1 casting

refactoring, fix loosing pin counts on thread exit

refactoring, additional verification

fix compilation and issue with parameter evaluation ordering

Some more checking

Cleanup

missing comma in copyright

newline at end of file in g1RegionPinCache.cpp

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2471,6 +2471,15 @@ void G1CollectedHeap::retire_tlabs() {
   ensure_parsability(true);
 }
 
+void G1CollectedHeap::flush_region_pin_cache() {
+  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) {
+    Pair<uint, size_t> stats = G1ThreadLocalData::get_and_reset_pin_cache(thread);
+    if (stats.second != 0) {
+      region_at(stats.first)->add_pinned_object_count(stats.second);
+    }
+  }
+}
+
 void G1CollectedHeap::do_collection_pause_at_safepoint_helper() {
   ResourceMark rm;
 

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -775,6 +775,10 @@ class G1CollectedHeap : public CollectedHeap {
 
   void retire_tlabs();
 
+  // Update all region's pin counts from the per-thread caches and resets them.
+  // Must be called before any decision based on pin counts.
+  void flush_region_pin_cache();
+
   void expand_heap_after_young_collection();
   // Update object copying statistics.
   void record_obj_copy_mem_stats();

src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp

@@ -1,5 +1,5 @@
-/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ /*
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -266,15 +266,37 @@ inline void G1CollectedHeap::pin_object(JavaThread* thread, oop obj) {
   assert(obj != nullptr, "obj must not be null");
   assert(!is_gc_active(), "must not pin objects during a GC");
   assert(obj->is_typeArray(), "must be typeArray");
-  HeapRegion *r = heap_region_containing(obj);
-  r->increment_pinned_object_count();
+  HeapRegion* r = heap_region_containing(obj);
+  uint obj_region_idx = r->hrm_index();
+
+  uint cache_region_idx = G1ThreadLocalData::cached_pinned_region_idx(thread);
+  if (cache_region_idx == obj_region_idx) {
+    G1ThreadLocalData::inc_cached_pin_count(thread);
+  } else {
+    // Flush old.
+    size_t cache_pin_count = G1ThreadLocalData::get_and_set_pin_cache(thread, obj_region_idx, (size_t)1);
+    if (cache_pin_count != 0) {
+      region_at(cache_region_idx)->add_pinned_object_count(cache_pin_count);
+    }
+  }
 }
 
 inline void G1CollectedHeap::unpin_object(JavaThread* thread, oop obj) {
   assert(obj != nullptr, "obj must not be null");
   assert(!is_gc_active(), "must not unpin objects during a GC");
-  HeapRegion *r = heap_region_containing(obj);
-  r->decrement_pinned_object_count();
+  HeapRegion* r = heap_region_containing(obj);
+  uint obj_region_idx = r->hrm_index();
+
+  uint cache_region_idx = G1ThreadLocalData::cached_pinned_region_idx(thread);
+  if (cache_region_idx == obj_region_idx) {
+    G1ThreadLocalData::dec_cached_pin_count(thread);
+  } else {
+    // Flush old.
+    size_t cache_pin_count = G1ThreadLocalData::get_and_set_pin_cache(thread, obj_region_idx, ~(size_t)0);
+    if (cache_pin_count != 0) {
+      region_at(cache_region_idx)->add_pinned_object_count(cache_pin_count);
+    }
+  }
 }
 
 inline bool G1CollectedHeap::is_obj_dead(const oop obj) const {

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -191,6 +191,7 @@ void G1FullCollector::prepare_collection() {
 
   _heap->gc_prologue(true);
   _heap->retire_tlabs();
+  _heap->flush_region_pin_cache();
   _heap->prepare_heap_for_full_collection();
 
   PrepareRegionsClosure cl(this);

src/hotspot/share/gc/g1/g1RegionPinCache.cpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/g1/g1RegionPinCache.hpp"
+#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/heapRegion.inline.hpp"
+
+G1RegionPinCache::~G1RegionPinCache() {
+  if (_count != 0) {
+    G1CollectedHeap::heap()->region_at(_region_idx)->add_pinned_object_count(_count);
+  }
+  get_and_reset();
+}

src/hotspot/share/gc/g1/g1RegionPinCache.hpp

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1REGIONPINCACHE_HPP
+#define SHARE_GC_G1_G1REGIONPINCACHE_HPP
+
+#include "memory/allocation.hpp"
+#include "utilities/pair.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// Holds the pinned object count increment for the given region for a Java thread.
+// I.e. the _count value may actually be negative temporarily if pinning operations
+// were interleaved between two regions.
+class G1RegionPinCache : public StackObj {
+  uint _region_idx;
+  size_t _count;
+
+public:
+  G1RegionPinCache() : _region_idx(0), _count(0) { }
+  ~G1RegionPinCache();
+
+  uint region_idx() const { return _region_idx; }
+  size_t count() const { return _count; }
+
+  void inc_count() { ++_count; }
+  void dec_count() { --_count; }
+
+  size_t get_and_set(uint new_region_idx, size_t new_count) {
+    size_t result = _count;
+    _region_idx = new_region_idx;
+    _count = new_count;
+    return result;
+  }
+
+  // Gets current region and pin count and resets the values to defaults.
+  Pair<uint, size_t> get_and_reset() {
+    return Pair<uint, size_t> { _region_idx, get_and_set(0, 0) };
+  }
+};
+
+#endif /* SHARE_GC_G1_G1REGIONPINCACHE_HPP */

src/hotspot/share/gc/g1/g1ThreadLocalData.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 
 #include "gc/g1/g1BarrierSet.hpp"
 #include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1RegionPinCache.hpp"
 #include "gc/shared/gc_globals.hpp"
 #include "gc/shared/satbMarkQueue.hpp"
 #include "runtime/javaThread.hpp"
@@ -37,9 +38,15 @@ class G1ThreadLocalData {
   SATBMarkQueue _satb_mark_queue;
   G1DirtyCardQueue _dirty_card_queue;
 
+  // Per-thread cache of pinned object count to reduce atomic operation traffic
+  // due to region pinning. Holds the last region where the mutator pinned an
+  // object and the number of pin operations since the last change of the region.
+  G1RegionPinCache _pin_cache;
+
   G1ThreadLocalData() :
       _satb_mark_queue(&G1BarrierSet::satb_mark_queue_set()),
-      _dirty_card_queue(&G1BarrierSet::dirty_card_queue_set()) {}
+      _dirty_card_queue(&G1BarrierSet::dirty_card_queue_set()),
+      _pin_cache() {}
 
   static G1ThreadLocalData* data(Thread* thread) {
     assert(UseG1GC, "Sanity");
@@ -90,6 +97,32 @@ class G1ThreadLocalData {
   static ByteSize dirty_card_queue_buffer_offset() {
     return dirty_card_queue_offset() + G1DirtyCardQueue::byte_offset_of_buf();
   }
+
+  static uint cached_pinned_region_idx(Thread* thread) {
+    return data(thread)->_pin_cache.region_idx();
+  }
+
+#ifdef ASSERT
+  static size_t cached_pin_count(Thread* thread) {
+    return data(thread)->_pin_cache.count();
+  }
+#endif
+
+  static void inc_cached_pin_count(Thread* thread) {
+    data(thread)->_pin_cache.inc_count();
+  }
+
+  static void dec_cached_pin_count(Thread* thread) {
+    data(thread)->_pin_cache.dec_count();
+  }
+
+  static size_t get_and_set_pin_cache(Thread* thread, uint region_idx, size_t new_count) {
+    return data(thread)->_pin_cache.get_and_set(region_idx, new_count);
+  }
+
+  static Pair<uint, size_t> get_and_reset_pin_cache(Thread* thread) {
+    return data(thread)->_pin_cache.get_and_reset();
+  }
 };
 
 #endif // SHARE_GC_G1_G1THREADLOCALDATA_HPP

src/hotspot/share/gc/g1/g1YoungCollector.cpp

@@ -1087,6 +1087,12 @@ void G1YoungCollector::collect() {
 
     pre_evacuate_collection_set(jtm.evacuation_info());
 
+#ifdef ASSERT
+    for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) {
+      assert(G1ThreadLocalData::cached_pin_count(thread) == 0, "must be flushed");
+    }
+#endif
+
     G1ParScanThreadStateSet per_thread_states(_g1h,
                                               workers()->active_workers(),
                                               collection_set(),

src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
 #include "gc/g1/g1ConcurrentRefineStats.hpp"
 #include "gc/g1/g1DirtyCardQueue.hpp"
 #include "gc/g1/g1YoungGCPreEvacuateTasks.hpp"
+#include "gc/g1/g1ThreadLocalData.hpp"
 #include "gc/shared/barrierSet.inline.hpp"
 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
 #include "memory/allocation.inline.hpp"
@@ -57,12 +58,18 @@ class G1PreEvacuateCollectionSetBatchTask::JavaThreadRetireTLABAndFlushLogs : pu
       assert(thread->is_Java_thread(), "must be");
       // Flushes deferred card marks, so must precede concatenating logs.
       BarrierSet::barrier_set()->make_parsable((JavaThread*)thread);
+      // Retire TLABs.
       if (UseTLAB) {
         thread->tlab().retire(&_tlab_stats);
       }
-
+      // Concatenate logs.
       G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
       _refinement_stats += qset.concatenate_log_and_stats(thread);
+      // Flush region pincount cache.
+      Pair<uint, size_t> region_pin_cache = G1ThreadLocalData::get_and_reset_pin_cache(thread);
+      if (region_pin_cache.second != 0) {
+        G1CollectedHeap::heap()->region_at(region_pin_cache.first)->add_pinned_object_count(region_pin_cache.second);
+      }
     }
   };
 
@@ -132,6 +139,8 @@ class G1PreEvacuateCollectionSetBatchTask::NonJavaThreadFlushLogs : public G1Abs
     void do_thread(Thread* thread) override {
       G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
       _refinement_stats += qset.concatenate_log_and_stats(thread);
+
+      assert(G1ThreadLocalData::cached_pin_count(thread) == 0, "NonJava thread has pinned Java objects");
     }
   } _tc;
 

src/hotspot/share/gc/g1/heapRegion.hpp

@@ -302,8 +302,7 @@ class HeapRegion : public CHeapObj<mtGC> {
   static uint   LogOfHRGrainBytes;
   static uint   LogCardsPerRegion;
 
-  inline void increment_pinned_object_count();
-  inline void decrement_pinned_object_count();
+  inline void add_pinned_object_count(size_t value);
 
   static size_t GrainBytes;
   static size_t GrainWords;

src/hotspot/share/gc/g1/heapRegion.inline.hpp

@@ -553,12 +553,10 @@ inline void HeapRegion::record_surv_words_in_group(size_t words_survived) {
   _surv_rate_group->record_surviving_words(age, words_survived);
 }
 
-inline void HeapRegion::increment_pinned_object_count() {
-  Atomic::add(&_pinned_object_count, (size_t)1, memory_order_relaxed);
-}
-
-inline void HeapRegion::decrement_pinned_object_count() {
-  Atomic::sub(&_pinned_object_count, (size_t)1, memory_order_relaxed);
+inline void HeapRegion::add_pinned_object_count(size_t value) {
+  assert(value != 0, "wasted effort");
+  assert(!is_free(), "trying to pin free region %u, adding %zu", hrm_index(), value);
+  Atomic::add(&_pinned_object_count, value, memory_order_relaxed);
 }
 
 #endif // SHARE_GC_G1_HEAPREGION_INLINE_HPP