Gencopy LOS nursery allocation (#380)

Author: qinsoon

src/plan/barriers.rs

@@ -37,6 +37,8 @@ impl Barrier for NoBarrier {
 pub struct ObjectRememberingBarrier<E: ProcessEdgesWork> {
     mmtk: &'static MMTK<E::VM>,
     modbuf: Vec<ObjectReference>,
+    /// The metadata used for log bit. Though this allows taking an arbitrary metadata spec,
+    /// for this field, 0 means logged, and 1 means unlogged (the same as the vm::object_model::VMGlobalLogBitSpec).
     meta: MetadataSpec,
 }
 
@@ -52,6 +54,7 @@ impl<E: ProcessEdgesWork> ObjectRememberingBarrier<E> {
 
     #[inline(always)]
     fn enqueue_node<VM: VMBinding>(&mut self, obj: ObjectReference) {
+        // If the objecct is unlogged, log it and push it to mod buffer
         if compare_exchange_metadata::<VM>(
             &self.meta,
             obj,

src/plan/gencopy/gc_work.rs

@@ -7,7 +7,6 @@ use crate::policy::space::Space;
 use crate::scheduler::gc_work::*;
 use crate::scheduler::WorkerLocal;
 use crate::util::alloc::{Allocator, BumpAllocator};
-use crate::util::metadata::store_metadata;
 use crate::util::object_forwarding;
 use crate::util::opaque_pointer::*;
 use crate::util::{Address, ObjectReference};
@@ -57,13 +56,7 @@ impl<VM: VMBinding> CopyContext for GenCopyCopyContext<VM> {
     ) {
         object_forwarding::clear_forwarding_bits::<VM>(obj);
         if !super::NO_SLOW && super::ACTIVE_BARRIER == BarrierSelector::ObjectBarrier {
-            store_metadata::<VM>(
-                &VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC,
-                obj,
-                0b1,
-                None,
-                Some(Ordering::SeqCst),
-            );
+            VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::<VM>(obj, Ordering::SeqCst);
         }
     }
 }
@@ -120,6 +113,14 @@ impl<VM: VMBinding> ProcessEdgesWork for GenCopyNurseryProcessEdges<VM> {
                     unsafe { self.worker().local::<GenCopyCopyContext<VM>>() },
                 );
         }
+        // We may alloc large object into LOS as nursery objects. Trace them here.
+        if self.gencopy().common.get_los().in_space(object) {
+            return self
+                .gencopy()
+                .common
+                .get_los()
+                .trace_object::<Self>(self, object);
+        }
         debug_assert!(!self.gencopy().fromspace().in_space(object));
         debug_assert!(self.gencopy().tospace().in_space(object));
         object

src/plan/gencopy/global.rs

@@ -48,15 +48,12 @@ pub const GENCOPY_CONSTRAINTS: PlanConstraints = PlanConstraints {
     gc_header_bits: 2,
     gc_header_words: 0,
     num_specialized_scans: 1,
+    needs_log_bit: true,
     barrier: super::ACTIVE_BARRIER,
-    // TODO: We should use MAX_NON_LOS_ALLOC_BYTES_COPYING_PLAN which will allocate
-    // large objects directly to LOS. However, there are bugs in gencopy that prevents us doing it.
-    // I will do a separate PR to fix it.
-    // max_non_los_default_alloc_bytes: crate::util::rust_util::min_of_usize(
-    //     crate::plan::plan_constraints::MAX_NON_LOS_ALLOC_BYTES_COPYING_PLAN,
-    //     crate::util::options::NURSERY_SIZE,
-    // ),
-    max_non_los_default_alloc_bytes: crate::util::options::NURSERY_SIZE,
+    max_non_los_default_alloc_bytes: crate::util::rust_util::min_of_usize(
+        crate::plan::plan_constraints::MAX_NON_LOS_ALLOC_BYTES_COPYING_PLAN,
+        crate::util::options::NURSERY_SIZE,
+    ),
     ..PlanConstraints::default()
 };
 
@@ -153,9 +150,10 @@ impl<VM: VMBinding> Plan for GenCopy<VM> {
     }
 
     fn prepare(&mut self, tls: VMWorkerThread) {
-        self.common.prepare(tls, true);
+        let full_heap = !self.is_current_gc_nursery();
+        self.common.prepare(tls, full_heap);
         self.nursery.prepare(true);
-        if !self.is_current_gc_nursery() {
+        if full_heap {
             self.hi
                 .store(!self.hi.load(Ordering::SeqCst), Ordering::SeqCst); // flip the semi-spaces
         }
@@ -165,9 +163,10 @@ impl<VM: VMBinding> Plan for GenCopy<VM> {
     }
 
     fn release(&mut self, tls: VMWorkerThread) {
-        self.common.release(tls, true);
+        let full_heap = !self.is_current_gc_nursery();
+        self.common.release(tls, full_heap);
         self.nursery.release();
-        if !self.is_current_gc_nursery() {
+        if full_heap {
             self.fromspace().release();
         }
 

src/plan/plan_constraints.rs

@@ -19,6 +19,8 @@ pub struct PlanConstraints {
     /// Size (in bytes) beyond which copied objects must be copied to the LOS.
     /// This depends on the copy allocator.
     pub max_non_los_copy_bytes: usize,
+    /// Does this plan use the log bit? See vm::ObjectModel::GLOBAL_LOG_BIT_SPEC.
+    pub needs_log_bit: bool,
     /// Some plans may allow benign race for testing mark bit, and this will lead to trace the same edges
     /// multiple times. If a plan allows tracing duplicate edges, we will not run duplicate edge check
     /// in extreme_assertions.
@@ -45,6 +47,7 @@ impl PlanConstraints {
             generate_gc_trace: false,
             may_trace_duplicate_edges: false,
             needs_forward_after_liveness: false,
+            needs_log_bit: false,
             barrier: BarrierSelector::NoBarrier,
         }
     }

src/policy/copyspace.rs

@@ -96,6 +96,7 @@ impl<VM: VMBinding> CopySpace<VM> {
                 name,
                 movable: true,
                 immortal: false,
+                needs_log_bit: false,
                 zeroed,
                 vmrequest,
                 side_metadata_specs: SideMetadataContext {

src/policy/immortalspace.rs

@@ -69,6 +69,10 @@ impl<VM: VMBinding> SFT for ImmortalSpace<VM> {
             None,
             Some(Ordering::SeqCst),
         );
+
+        if self.common.needs_log_bit {
+            VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::<VM>(object, Ordering::SeqCst);
+        }
     }
 }
 
@@ -104,13 +108,14 @@ impl<VM: VMBinding> ImmortalSpace<VM> {
         vm_map: &'static VMMap,
         mmapper: &'static Mmapper,
         heap: &mut HeapMeta,
-        _constraints: &'static PlanConstraints,
+        constraints: &'static PlanConstraints,
     ) -> Self {
         let common = CommonSpace::new(
             SpaceOptions {
                 name,
                 movable: false,
                 immortal: true,
+                needs_log_bit: constraints.needs_log_bit,
                 zeroed,
                 vmrequest,
                 side_metadata_specs: SideMetadataContext {

src/policy/largeobjectspace.rs

@@ -69,6 +69,10 @@ impl<VM: VMBinding> SFT for LargeObjectSpace<VM> {
             Some(Ordering::SeqCst),
         );
 
+        if self.common.needs_log_bit {
+            VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::<VM>(object, Ordering::SeqCst);
+        }
+
         let cell = VM::VMObjectModel::object_start_ref(object);
         self.treadmill.add_to_treadmill(cell, alloc);
     }
@@ -84,7 +88,9 @@ impl<VM: VMBinding> Space<VM> for LargeObjectSpace<VM> {
     fn get_page_resource(&self) -> &dyn PageResource<VM> {
         &self.pr
     }
-    fn init(&mut self, _vm_map: &'static VMMap) {}
+    fn init(&mut self, _vm_map: &'static VMMap) {
+        self.common().init(self.as_space());
+    }
 
     fn common(&self) -> &CommonSpace<VM> {
         &self.common
@@ -105,7 +111,7 @@ impl<VM: VMBinding> LargeObjectSpace<VM> {
         vm_map: &'static VMMap,
         mmapper: &'static Mmapper,
         heap: &mut HeapMeta,
-        _constraints: &'static PlanConstraints,
+        constraints: &'static PlanConstraints,
         protect_memory_on_release: bool,
     ) -> Self {
         let common = CommonSpace::new(
@@ -114,6 +120,7 @@ impl<VM: VMBinding> LargeObjectSpace<VM> {
                 movable: false,
                 immortal: false,
                 zeroed,
+                needs_log_bit: constraints.needs_log_bit,
                 vmrequest,
                 side_metadata_specs: SideMetadataContext {
                     global: global_side_metadata_specs,

src/policy/space.rs

@@ -482,6 +482,10 @@ pub struct CommonSpace<VM: VMBinding> {
 
     pub metadata: SideMetadataContext,
 
+    /// This field equals to needs_log_bit in the plan constraints.
+    // TODO: This should be a constant for performance.
+    pub needs_log_bit: bool,
+
     p: PhantomData<VM>,
 }
 
@@ -490,6 +494,7 @@ pub struct SpaceOptions {
     pub movable: bool,
     pub immortal: bool,
     pub zeroed: bool,
+    pub needs_log_bit: bool,
     pub vmrequest: VMRequest,
     pub side_metadata_specs: SideMetadataContext,
 }
@@ -517,6 +522,7 @@ impl<VM: VMBinding> CommonSpace<VM> {
             head_discontiguous_region: unsafe { Address::zero() },
             vm_map,
             mmapper,
+            needs_log_bit: opt.needs_log_bit,
             metadata: opt.side_metadata_specs,
             p: PhantomData,
         };

src/util/metadata/log_bit.rs

@@ -0,0 +1,12 @@
+use crate::util::metadata::*;
+use crate::util::ObjectReference;
+use crate::vm::VMBinding;
+use crate::vm::VMGlobalLogBitSpec;
+use std::sync::atomic::Ordering;
+
+impl VMGlobalLogBitSpec {
+    /// Mark the log bit as unlogged (1 means unlogged)
+    pub fn mark_as_unlogged<VM: VMBinding>(&self, object: ObjectReference, order: Ordering) {
+        store_metadata::<VM>(&self, object, 1, None, Some(order))
+    }
+}

src/util/metadata/mod.rs

@@ -222,4 +222,6 @@ mod global;
 pub mod header_metadata;
 pub mod side_metadata;
 
+pub(crate) mod log_bit;
+
 pub use global::*;

src/util/mod.rs

@@ -44,6 +44,8 @@ pub mod metadata;
 pub(crate) mod object_forwarding;
 /// MMTk command line options.
 pub(crate) mod options;
+/// Utilities funcitons for Rust
+pub(crate) mod rust_util;
 /// Sanity checker for GC.
 #[cfg(feature = "sanity")]
 pub(crate) mod sanity;

src/util/rust_util.rs

@@ -0,0 +1,8 @@
+/// Const funciton for min value of two usize numbers.
+pub const fn min_of_usize(a: usize, b: usize) -> usize {
+    if a > b {
+        b
+    } else {
+        a
+    }
+}

src/util/treadmill.rs

@@ -4,14 +4,24 @@ use std::sync::Mutex;
 
 use crate::util::Address;
 
-#[derive(Debug)]
 pub struct TreadMill {
     from_space: Mutex<HashSet<Address>>,
     to_space: Mutex<HashSet<Address>>,
     collect_nursery: Mutex<HashSet<Address>>,
     alloc_nursery: Mutex<HashSet<Address>>,
 }
 
+impl std::fmt::Debug for TreadMill {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TreadMill")
+            .field("from", &self.from_space.lock().unwrap())
+            .field("to", &self.to_space.lock().unwrap())
+            .field("collect_nursery", &self.collect_nursery.lock().unwrap())
+            .field("alloc_nursery", &self.alloc_nursery.lock().unwrap())
+            .finish()
+    }
+}
+
 impl TreadMill {
     pub fn new() -> Self {
         TreadMill {

src/vm/object_model.rs

@@ -70,33 +70,11 @@ pub trait ObjectModel<VM: VMBinding> {
     // Any side metadata offset calculation must consider these to prevent overlaps. A binding should start their
     // side metadata from GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS or LOCAL_SIDE_METADATA_VM_BASE_ADDRESS.
 
-    // --------------------------------------------------
-    //
-    // Global Metadata
-    //
-    // MMTk reserved Global side metadata offsets:
-    //
-    //  1 - MarkSweep Active Chunk byte:
-    //      - Offset `GLOBAL_SIDE_METADATA_BASE_ADDRESS`
-    //
-    // --------------------------------------------------
-
     /// The metadata specification of the global log bit. 1 bit.
+    /// Note that for this bit, 0 represents logged (default), and 1 represents unlogged.
+    /// This bit is also referred to as unlogged bit in Java MMTk for this reason.
     const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec;
 
-    // --------------------------------------------------
-    // PolicySpecific Metadata
-    //
-    // MMTk reserved PolicySpecific side metadata offsets:
-    //
-    //  1 - MarkSweep Alloc bit:
-    //      - Offset `0x0` on 32-bits
-    //      - Offset `LOCAL_SIDE_METADATA_BASE_ADDRESS` on 64-bits
-    //  2 - MarkSweep Active Page byte:
-    //      - Offset `Alloc bit`.offset + `Alloc bit`.metadata_address_range_size()
-    //
-    // --------------------------------------------------
-
     /// The metadata specification for the forwarding pointer, used by copying plans. Word size.
     const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec;
     /// The metadata specification for the forwarding status bits, used by copying plans. 2 bits.