Add Immix GC (#382)

Author: wenyuzhao

src/lib.rs

@@ -14,6 +14,9 @@
 #![feature(trait_alias)]
 #![feature(const_panic)]
 #![feature(const_fn_union)]
+#![feature(step_trait)]
+#![feature(step_trait_ext)]
+#![feature(const_generics)]
 // TODO: We should fix missing docs for public items and turn this on (Issue #309).
 // #![deny(missing_docs)]
 

src/mmtk.rs

@@ -53,7 +53,13 @@ impl<VM: VMBinding> MMTK<VM> {
     pub fn new() -> Self {
         let scheduler = MMTkScheduler::new();
         let options = Arc::new(UnsafeOptionsWrapper::new(Options::default()));
-        let plan = crate::plan::create_plan(options.plan, &VM_MAP, &MMAPPER, options.clone());
+        let plan = crate::plan::create_plan(
+            options.plan,
+            &VM_MAP,
+            &MMAPPER,
+            options.clone(),
+            scheduler.clone(),
+        );
         MMTK {
             plan,
             reference_processors: ReferenceProcessors::new(),

src/plan/global.rs

@@ -126,6 +126,7 @@ pub fn create_mutator<VM: VMBinding>(
         PlanSelector::MarkSweep => {
             crate::plan::marksweep::mutator::create_ms_mutator(tls, &*mmtk.plan)
         }
+        PlanSelector::Immix => crate::plan::immix::mutator::create_immix_mutator(tls, &*mmtk.plan),
         PlanSelector::PageProtect => {
             crate::plan::pageprotect::mutator::create_pp_mutator(tls, &*mmtk.plan)
         }
@@ -137,6 +138,7 @@ pub fn create_plan<VM: VMBinding>(
     vm_map: &'static VMMap,
     mmapper: &'static Mmapper,
     options: Arc<UnsafeOptionsWrapper>,
+    scheduler: Arc<MMTkScheduler<VM>>,
 ) -> Box<dyn Plan<VM = VM>> {
     match plan {
         PlanSelector::NoGC => Box::new(crate::plan::nogc::NoGC::new(vm_map, mmapper, options)),
@@ -149,6 +151,9 @@ pub fn create_plan<VM: VMBinding>(
         PlanSelector::MarkSweep => Box::new(crate::plan::marksweep::MarkSweep::new(
             vm_map, mmapper, options,
         )),
+        PlanSelector::Immix => Box::new(crate::plan::immix::Immix::new(
+            vm_map, mmapper, options, scheduler,
+        )),
         PlanSelector::PageProtect => Box::new(crate::plan::pageprotect::PageProtect::new(
             vm_map, mmapper, options,
         )),
@@ -649,7 +654,7 @@ impl<VM: VMBinding> BasePlan<VM> {
         *self.gc_status.lock().unwrap() == GcStatus::GcProper
     }
 
-    fn is_user_triggered_collection(&self) -> bool {
+    pub fn is_user_triggered_collection(&self) -> bool {
         self.user_triggered_collection.load(Ordering::Relaxed)
     }
 

src/plan/immix/gc_work.rs

@@ -0,0 +1,195 @@
+use super::global::Immix;
+use crate::plan::PlanConstraints;
+use crate::policy::immix::ScanObjectsAndMarkLines;
+use crate::policy::space::Space;
+use crate::scheduler::gc_work::*;
+use crate::scheduler::{WorkBucketStage, WorkerLocal};
+use crate::util::alloc::{Allocator, ImmixAllocator};
+use crate::util::object_forwarding;
+use crate::util::{Address, ObjectReference};
+use crate::vm::VMBinding;
+use crate::MMTK;
+use crate::{
+    plan::CopyContext,
+    util::opaque_pointer::{VMThread, VMWorkerThread},
+};
+use std::{
+    mem,
+    ops::{Deref, DerefMut},
+};
+
+/// Immix copy allocator
+pub struct ImmixCopyContext<VM: VMBinding> {
+    immix: ImmixAllocator<VM>,
+}
+
+impl<VM: VMBinding> CopyContext for ImmixCopyContext<VM> {
+    type VM = VM;
+
+    fn constraints(&self) -> &'static PlanConstraints {
+        &super::global::IMMIX_CONSTRAINTS
+    }
+    fn init(&mut self, tls: VMWorkerThread) {
+        self.immix.tls = tls.0;
+    }
+    fn prepare(&mut self) {
+        self.immix.reset()
+    }
+    fn release(&mut self) {
+        self.immix.reset()
+    }
+    #[inline(always)]
+    fn alloc_copy(
+        &mut self,
+        _original: ObjectReference,
+        bytes: usize,
+        align: usize,
+        offset: isize,
+        _semantics: crate::AllocationSemantics,
+    ) -> Address {
+        self.immix.alloc(bytes, align, offset)
+    }
+    #[inline(always)]
+    fn post_copy(
+        &mut self,
+        obj: ObjectReference,
+        _tib: Address,
+        _bytes: usize,
+        _semantics: crate::AllocationSemantics,
+    ) {
+        object_forwarding::clear_forwarding_bits::<VM>(obj);
+    }
+}
+
+impl<VM: VMBinding> ImmixCopyContext<VM> {
+    pub fn new(mmtk: &'static MMTK<VM>) -> Self {
+        Self {
+            immix: ImmixAllocator::new(
+                VMThread::UNINITIALIZED,
+                Some(&mmtk.plan.downcast_ref::<Immix<VM>>().unwrap().immix_space),
+                &*mmtk.plan,
+                true,
+            ),
+        }
+    }
+}
+
+impl<VM: VMBinding> WorkerLocal for ImmixCopyContext<VM> {
+    fn init(&mut self, tls: VMWorkerThread) {
+        CopyContext::init(self, tls);
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub(super) enum TraceKind {
+    Fast,
+    Defrag,
+}
+
+pub(super) struct ImmixProcessEdges<VM: VMBinding, const KIND: TraceKind> {
+    // Use a static ref to the specific plan to avoid overhead from dynamic dispatch or
+    // downcast for each traced object.
+    plan: &'static Immix<VM>,
+    base: ProcessEdgesBase<Self>,
+    mmtk: &'static MMTK<VM>,
+}
+
+impl<VM: VMBinding, const KIND: TraceKind> ImmixProcessEdges<VM, KIND> {
+    fn immix(&self) -> &'static Immix<VM> {
+        self.plan
+    }
+
+    #[inline(always)]
+    fn fast_trace_object(&mut self, object: ObjectReference) -> ObjectReference {
+        if object.is_null() {
+            return object;
+        }
+        if self.immix().immix_space.in_space(object) {
+            self.immix().immix_space.fast_trace_object(self, object)
+        } else {
+            self.immix()
+                .common
+                .trace_object::<Self, ImmixCopyContext<VM>>(self, object)
+        }
+    }
+
+    /// Trace objects without evacuation.
+    #[inline(always)]
+    fn fast_process_edge(&mut self, slot: Address) {
+        let object = unsafe { slot.load::<ObjectReference>() };
+        self.fast_trace_object(object);
+    }
+}
+
+impl<VM: VMBinding, const KIND: TraceKind> ProcessEdgesWork for ImmixProcessEdges<VM, KIND> {
+    type VM = VM;
+    const OVERWRITE_REFERENCE: bool = crate::policy::immix::DEFRAG;
+
+    fn new(edges: Vec<Address>, _roots: bool, mmtk: &'static MMTK<VM>) -> Self {
+        let base = ProcessEdgesBase::new(edges, mmtk);
+        let plan = base.plan().downcast_ref::<Immix<VM>>().unwrap();
+        Self { plan, base, mmtk }
+    }
+
+    #[cold]
+    fn flush(&mut self) {
+        let mut new_nodes = vec![];
+        mem::swap(&mut new_nodes, &mut self.nodes);
+        let scan_objects_work =
+            ScanObjectsAndMarkLines::<Self>::new(new_nodes, false, &self.immix().immix_space);
+        if Self::SCAN_OBJECTS_IMMEDIATELY {
+            self.worker().do_work(scan_objects_work);
+        } else {
+            self.mmtk.scheduler.work_buckets[WorkBucketStage::Closure].add(scan_objects_work);
+        }
+    }
+
+    /// Trace  and evacuate objects.
+    #[inline(always)]
+    fn trace_object(&mut self, object: ObjectReference) -> ObjectReference {
+        if object.is_null() {
+            return object;
+        }
+        if self.immix().immix_space.in_space(object) {
+            self.immix().immix_space.trace_object(
+                self,
+                object,
+                super::global::ALLOC_IMMIX,
+                unsafe { self.worker().local::<ImmixCopyContext<VM>>() },
+            )
+        } else {
+            self.immix()
+                .common
+                .trace_object::<Self, ImmixCopyContext<VM>>(self, object)
+        }
+    }
+
+    #[inline]
+    fn process_edges(&mut self) {
+        if KIND == TraceKind::Fast {
+            for i in 0..self.edges.len() {
+                // Use fast_process_edge since we don't need to forward any objects.
+                self.fast_process_edge(self.edges[i])
+            }
+        } else {
+            for i in 0..self.edges.len() {
+                self.process_edge(self.edges[i])
+            }
+        }
+    }
+}
+
+impl<VM: VMBinding, const KIND: TraceKind> Deref for ImmixProcessEdges<VM, KIND> {
+    type Target = ProcessEdgesBase<Self>;
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        &self.base
+    }
+}
+
+impl<VM: VMBinding, const KIND: TraceKind> DerefMut for ImmixProcessEdges<VM, KIND> {
+    #[inline]
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.base
+    }
+}