Bug 1836085 - Set nursery position to the end of the chunk when minor GC is requested r=jandem

This allows us to fold the check for this into the space check for nursery
allocations.

This change also means we don't trigger requested major GC on nursery
allocations. We still trigger this via interrupts.

This also separates the interrupt bits for minor and major GCs.

Differential Revision: https://phabricator.services.mozilla.com/D179789

Author: jonco3

js/src/gc/Allocator.cpp

@@ -120,14 +120,18 @@ void* GCRuntime::tryNewNurseryCell(JSContext* cx, size_t thingSize,
   MOZ_ASSERT(!cx->zone()->isAtomsZone());
   MOZ_ASSERT(cx->zone()->allocKindInNursery(kind));
 
-  void* ptr = cx->nursery().allocateCell(site, thingSize, kind);
+  Nursery& nursery = cx->nursery();
+  void* ptr = nursery.allocateCell(site, thingSize, kind);
   if (ptr) {
     return ptr;
   }
 
   if constexpr (allowGC) {
     if (!cx->suppressGC) {
-      cx->runtime()->gc.minorGC(JS::GCReason::OUT_OF_NURSERY);
+      JS::GCReason reason = nursery.minorGCRequested()
+                                ? nursery.minorGCTriggerReason()
+                                : JS::GCReason::OUT_OF_NURSERY;
+      cx->runtime()->gc.minorGC(reason);
 
       // Exceeding gcMaxBytes while tenuring can disable the Nursery.
       if (cx->zone()->allocKindInNursery(kind)) {
@@ -164,6 +168,10 @@ template <AllowGC allowGC>
 /* static */
 void* GCRuntime::tryNewTenuredThing(JSContext* cx, AllocKind kind,
                                     size_t thingSize) {
+  if constexpr (allowGC) {
+    gcIfNeededAtAllocation(cx);
+  }
+
   // Bump allocate in the arena's current free-list span.
   Zone* zone = cx->zone();
   void* ptr = zone->arenas.freeLists().allocate(kind);
@@ -219,23 +227,30 @@ void GCRuntime::attemptLastDitchGC(JSContext* cx) {
   lastLastDitchTime = mozilla::TimeStamp::Now();
 }
 
+#ifdef DEBUG
+static bool IsAtomsZoneKind(AllocKind kind) {
+  return kind == AllocKind::ATOM || kind == AllocKind::FAT_INLINE_ATOM ||
+         kind == AllocKind::SYMBOL;
+}
+#endif
+
 template <AllowGC allowGC>
 bool GCRuntime::checkAllocatorState(JSContext* cx, AllocKind kind) {
   MOZ_ASSERT_IF(cx->zone()->isAtomsZone(),
-                kind == AllocKind::ATOM || kind == AllocKind::FAT_INLINE_ATOM ||
-                    kind == AllocKind::SYMBOL || kind == AllocKind::JITCODE ||
-                    kind == AllocKind::SCOPE);
-  MOZ_ASSERT_IF(!cx->zone()->isAtomsZone(),
-                kind != AllocKind::ATOM && kind != AllocKind::FAT_INLINE_ATOM);
-  MOZ_ASSERT(!JS::RuntimeHeapIsBusy());
+                IsAtomsZoneKind(kind) || kind == AllocKind::JITCODE);
+  MOZ_ASSERT_IF(!cx->zone()->isAtomsZone(), !IsAtomsZoneKind(kind));
 
   if constexpr (allowGC) {
     // Crash if we could perform a GC action when it is not safe.
     if (!cx->suppressGC) {
       cx->verifyIsSafeToGC();
-    }
 
-    gcIfNeededAtAllocation(cx);
+#ifdef JS_GC_ZEAL
+      if (needZealousGC()) {
+        runDebugGC();
+      }
+#endif
+    }
   }
 
   // For testing out of memory conditions.
@@ -251,17 +266,12 @@ bool GCRuntime::checkAllocatorState(JSContext* cx, AllocKind kind) {
   return true;
 }
 
+/* static */
 inline void GCRuntime::gcIfNeededAtAllocation(JSContext* cx) {
-#ifdef JS_GC_ZEAL
-  if (needZealousGC()) {
-    runDebugGC();
-  }
-#endif
-
   // Invoking the interrupt callback can fail and we can't usefully
   // handle that here. Just check in case we need to collect instead.
-  if (cx->hasAnyPendingInterrupt()) {
-    gcIfRequested();
+  if (cx->hasPendingInterrupt(InterruptReason::MajorGC)) {
+    cx->runtime()->gc.gcIfRequested();
   }
 }
 

js/src/gc/GC.cpp

@@ -1650,18 +1650,7 @@ void GCRuntime::requestMajorGC(JS::GCReason reason) {
   }
 
   majorGCTriggerReason = reason;
-  rt->mainContextFromAnyThread()->requestInterrupt(InterruptReason::GC);
-}
-
-void Nursery::requestMinorGC(JS::GCReason reason) const {
-  MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime()));
-
-  if (minorGCRequested()) {
-    return;
-  }
-
-  minorGCTriggerReason_ = reason;
-  runtime()->mainContextFromOwnThread()->requestInterrupt(InterruptReason::GC);
+  rt->mainContextFromAnyThread()->requestInterrupt(InterruptReason::MajorGC);
 }
 
 bool GCRuntime::triggerGC(JS::GCReason reason) {
@@ -4596,7 +4585,6 @@ void GCRuntime::collectNursery(JS::GCOptions options, JS::GCReason reason,
 
   gcstats::AutoPhase ap(stats(), phase);
 
-  nursery().clearMinorGCRequest();
   nursery().collect(options, reason);
   MOZ_ASSERT(nursery().isEmpty());
 

js/src/gc/GCRuntime.h

@@ -668,7 +668,7 @@ class GCRuntime {
                        const AutoLockGC& lock);
 
   // Allocator internals
-  void gcIfNeededAtAllocation(JSContext* cx);
+  static void gcIfNeededAtAllocation(JSContext* cx);
   static void* refillFreeList(JSContext* cx, AllocKind thingKind);
   void attemptLastDitchGC(JSContext* cx);
 #ifdef DEBUG

js/src/gc/Nursery.cpp

@@ -229,6 +229,7 @@ js::Nursery::Nursery(GCRuntime* gc)
       reportPretenuring_(false),
       reportPretenuringThreshold_(0),
       minorGCTriggerReason_(JS::GCReason::NO_REASON),
+      prevPosition_(0),
       hasRecentGrowthData(false),
       smoothedTargetSize(0.0) {
   const char* env = getenv("MOZ_NURSERY_STRINGS");
@@ -539,8 +540,8 @@ inline void* js::Nursery::allocate(size_t size) {
   MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime()));
   MOZ_ASSERT_IF(currentChunk_ == currentStartChunk_,
                 position() >= currentStartPosition_);
-  MOZ_ASSERT(position() % CellAlignBytes == 0);
   MOZ_ASSERT(size % CellAlignBytes == 0);
+  MOZ_ASSERT(position() % CellAlignBytes == 0);
 
   if (MOZ_UNLIKELY(currentEnd() < position() + size)) {
     return moveToNextChunkAndAllocate(size);
@@ -556,6 +557,12 @@ inline void* js::Nursery::allocate(size_t size) {
 }
 
 void* Nursery::moveToNextChunkAndAllocate(size_t size) {
+  if (minorGCRequested()) {
+    // If a minor GC was requested then fail the allocation. The collection is
+    // then run in GCRuntime::tryNewNurseryCell.
+    return nullptr;
+  }
+
   MOZ_ASSERT(currentEnd() < position() + size);
 
   unsigned chunkno = currentChunk_ + 1;
@@ -1124,6 +1131,15 @@ void js::Nursery::collect(JS::GCOptions options, JS::GCReason reason) {
   JSRuntime* rt = runtime();
   MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC);
 
+  if (minorGCRequested()) {
+    MOZ_ASSERT(position_ == chunk(currentChunk_).end());
+    position_ = prevPosition_;
+    prevPosition_ = 0;
+    minorGCTriggerReason_ = JS::GCReason::NO_REASON;
+    rt->mainContextFromOwnThread()->clearPendingInterrupt(
+        InterruptReason::MinorGC);
+  }
+
   if (!isEnabled() || isEmpty()) {
     // Our barriers are not always exact, and there may be entries in the
     // storebuffer even when the nursery is disabled or empty. It's not safe
@@ -1584,6 +1600,25 @@ bool js::Nursery::registerMallocedBuffer(void* buffer, size_t nbytes) {
   return true;
 }
 
+void Nursery::requestMinorGC(JS::GCReason reason) {
+  MOZ_ASSERT(reason != JS::GCReason::NO_REASON);
+  MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime()));
+  MOZ_ASSERT(isEnabled());
+
+  if (minorGCRequested()) {
+    return;
+  }
+
+  // Set position to end of chunk to block further allocation.
+  MOZ_ASSERT(prevPosition_ == 0);
+  prevPosition_ = position_;
+  position_ = chunk(currentChunk_).end();
+
+  minorGCTriggerReason_ = reason;
+  runtime()->mainContextFromOwnThread()->requestInterrupt(
+      InterruptReason::MinorGC);
+}
+
 size_t Nursery::sizeOfMallocedBuffers(
     mozilla::MallocSizeOf mallocSizeOf) const {
   size_t total = 0;

js/src/gc/Nursery.h

@@ -313,15 +313,12 @@ class alignas(TypicalCacheLineSize) Nursery {
     return pretenuringNursery.addressOfAllocatedSites();
   }
 
-  void requestMinorGC(JS::GCReason reason) const;
+  void requestMinorGC(JS::GCReason reason);
 
   bool minorGCRequested() const {
     return minorGCTriggerReason_ != JS::GCReason::NO_REASON;
   }
   JS::GCReason minorGCTriggerReason() const { return minorGCTriggerReason_; }
-  void clearMinorGCRequest() {
-    minorGCTriggerReason_ = JS::GCReason::NO_REASON;
-  }
 
   bool shouldCollect() const;
   bool isNearlyFull() const;
@@ -419,10 +416,11 @@ class alignas(TypicalCacheLineSize) Nursery {
   bool reportPretenuring_;
   size_t reportPretenuringThreshold_;
 
-  // Whether and why a collection of this nursery has been requested. This is
-  // mutable as it is set by the store buffer, which otherwise cannot modify
-  // anything in the nursery.
-  mutable JS::GCReason minorGCTriggerReason_;
+  // Whether and why a collection of this nursery has been requested. When this
+  // happens |prevPosition_| is set to the current position and |position_| set
+  // to the end of the chunk to force the next allocation to fail.
+  JS::GCReason minorGCTriggerReason_;
+  uintptr_t prevPosition_;
 
   // Profiling data.
 

js/src/gc/StoreBuffer.cpp

@@ -77,7 +77,7 @@ void StoreBuffer::GenericBuffer::trace(JSTracer* trc) {
   }
 }
 
-StoreBuffer::StoreBuffer(JSRuntime* rt, const Nursery& nursery)
+StoreBuffer::StoreBuffer(JSRuntime* rt, Nursery& nursery)
     : lock_(mutexid::StoreBuffer),
       bufferVal(this, JS::GCReason::FULL_VALUE_BUFFER),
       bufStrCell(this, JS::GCReason::FULL_CELL_PTR_STR_BUFFER),

js/src/gc/StoreBuffer.h

@@ -439,7 +439,7 @@ class StoreBuffer {
   GenericBuffer bufferGeneric;
 
   JSRuntime* runtime_;
-  const Nursery& nursery_;
+  Nursery& nursery_;
 
   bool aboutToOverflow_;
   bool enabled_;
@@ -453,7 +453,7 @@ class StoreBuffer {
   bool markingNondeduplicatable;
 #endif
 
-  explicit StoreBuffer(JSRuntime* rt, const Nursery& nursery);
+  explicit StoreBuffer(JSRuntime* rt, Nursery& nursery);
   [[nodiscard]] bool enable();
 
   void disable();

js/src/vm/JSContext.h

@@ -141,10 +141,11 @@ bool CurrentThreadIsParseThread();
 #endif
 
 enum class InterruptReason : uint32_t {
-  GC = 1 << 0,
-  AttachIonCompilations = 1 << 1,
-  CallbackUrgent = 1 << 2,
-  CallbackCanWait = 1 << 3,
+  MinorGC = 1 << 0,
+  MajorGC = 1 << 1,
+  AttachIonCompilations = 1 << 2,
+  CallbackUrgent = 1 << 3,
+  CallbackCanWait = 1 << 4,
 };
 
 enum class ShouldCaptureStack { Maybe, Always };
@@ -854,6 +855,7 @@ struct JS_PUBLIC_API JSContext : public JS::RootingContext,
   bool hasPendingInterrupt(js::InterruptReason reason) const {
     return interruptBits_ & uint32_t(reason);
   }
+  void clearPendingInterrupt(js::InterruptReason reason);
 
   // For JIT use. Points to the inlined ICScript for a baseline script
   // being invoked as part of a trial inlining.  Contains nullptr at

js/src/vm/Runtime.cpp

@@ -488,6 +488,11 @@ bool JSContext::handleInterrupt() {
   return true;
 }
 
+void JSContext::clearPendingInterrupt(js::InterruptReason reason) {
+  // Interrupt bit have already been cleared.
+  interruptBits_ &= ~uint32_t(reason);
+}
+
 bool JSRuntime::setDefaultLocale(const char* locale) {
   if (!locale) {
     return false;