11use crate :: util:: address:: Address ;
2- use crate :: util:: constants:: DEFAULT_STRESS_FACTOR ;
3-
42use std:: sync:: atomic:: Ordering ;
53
64use crate :: plan:: Plan ;
@@ -110,6 +108,10 @@ pub trait Allocator<VM: VMBinding>: Downcast {
110108 fn get_space ( & self ) -> & ' static dyn Space < VM > ;
111109 fn get_plan ( & self ) -> & ' static dyn Plan < VM = VM > ;
112110
111+ /// Does this allocator do thread local allocation? If an allocator does not do thread local allocation,
112+ /// each allocation will go to slowpath and will have a check for GC polls.
113+ fn does_thread_local_allocation ( & self ) -> bool ;
114+
113115 fn alloc ( & mut self , size : usize , align : usize , offset : isize ) -> Address ;
114116
115117 #[ inline( never) ]
@@ -121,17 +123,31 @@ pub trait Allocator<VM: VMBinding>: Downcast {
121123 fn alloc_slow_inline ( & mut self , size : usize , align : usize , offset : isize ) -> Address {
122124 let tls = self . get_tls ( ) ;
123125 let plan = self . get_plan ( ) . base ( ) ;
124- let stress_test = plan . options . stress_factor != DEFAULT_STRESS_FACTOR
125- || plan. options . analysis_factor != DEFAULT_STRESS_FACTOR ;
126+ let is_mutator = VM :: VMActivePlan :: is_mutator ( tls ) ;
127+ let stress_test = plan. is_stress_test_gc_enabled ( ) ;
126128
127129 // Information about the previous collection.
128130 let mut emergency_collection = false ;
129131 let mut previous_result_zero = false ;
130132 loop {
131133 // Try to allocate using the slow path
132- let result = self . alloc_slow_once ( size, align, offset) ;
134+ let result = if is_mutator && stress_test {
135+ // If we are doing stress GC, we invoke the special allow_slow_once call.
136+ // allow_slow_once_stress_test() should make sure that every allocation goes
137+ // to the slowpath (here) so we can check the allocation bytes and decide
138+ // if we need to do a stress GC.
133139
134- if !VM :: VMActivePlan :: is_mutator ( tls) {
140+ // If we should do a stress GC now, we tell the alloc_slow_once_stress_test()
141+ // so they would avoid try any thread local allocation, and directly call
142+ // global acquire and do a poll.
143+ let need_poll = is_mutator && plan. should_do_stress_gc ( ) ;
144+ self . alloc_slow_once_stress_test ( size, align, offset, need_poll)
145+ } else {
146+ // If we are not doing stress GC, just call the normal alloc_slow_once().
147+ self . alloc_slow_once ( size, align, offset)
148+ } ;
149+
150+ if !is_mutator {
135151 debug_assert ! ( !result. is_zero( ) ) ;
136152 return result;
137153 }
@@ -150,7 +166,19 @@ pub trait Allocator<VM: VMBinding>: Downcast {
150166 // called by acquire(). In order to not double count the allocation, we only
151167 // update allocation bytes if the previous result wasn't 0x0.
152168 if stress_test && self . get_plan ( ) . is_initialized ( ) && !previous_result_zero {
153- plan. increase_allocation_bytes_by ( size) ;
169+ let _allocation_bytes = plan. increase_allocation_bytes_by ( size) ;
170+
171+ // This is the allocation hook for the analysis trait. If you want to call
172+ // an analysis counter specific allocation hook, then here is the place to do so
173+ #[ cfg( feature = "analysis" ) ]
174+ if _allocation_bytes > plan. options . analysis_factor {
175+ trace ! (
176+ "Analysis: allocation_bytes = {} more than analysis_factor = {}" ,
177+ _allocation_bytes,
178+ plan. options. analysis_factor
179+ ) ;
180+ plan. analysis_manager . alloc_hook ( size, align, offset) ;
181+ }
154182 }
155183
156184 return result;
@@ -196,7 +224,47 @@ pub trait Allocator<VM: VMBinding>: Downcast {
196224 }
197225 }
198226
227+ /// Single slow path allocation attempt. This is called by allocSlow.
199228fn alloc_slow_once ( & mut self , size : usize , align : usize , offset : isize ) -> Address ;
229+
230+ /// Single slowpath allocation attempt for stress test. When the stress factor is set (e.g. to N),
231+ /// we would expect for every N bytes allocated, we will trigger a stress GC.
232+ /// However, for allocators that do thread local allocation, they may allocate from their thread local buffer
233+ /// which does not have a GC poll check, and they may even allocate with the JIT generated allocation
234+ /// fastpath which is unaware of stress test GC. For both cases, we are not able to guarantee
235+ /// a stress GC is triggered every N bytes. To solve this, when the stress factor is set, we
236+ /// will call this method instead of the normal alloc_slow_once(). We expect the implementation of this slow allocation
237+ /// will trick the fastpath so every allocation will fail in the fastpath, jump to the slow path and eventually
238+ /// call this method again for the actual allocation.
239+ ///
240+ /// The actual implementation about how to trick the fastpath may vary. For example, our bump pointer allocator will
241+ /// set the thread local buffer limit to the buffer size instead of the buffer end address. In this case, every fastpath
242+ /// check (cursor + size < limit) will fail, and jump to this slowpath. In the slowpath, we still allocate from the thread
243+ /// local buffer, and recompute the limit (remaining buffer size).
244+ ///
245+ /// If an allocator does not do thread local allocation (which returns false for does_thread_local_allocation()), it does
246+ /// not need to override this method. The default implementation will simply call allow_slow_once() and it will work fine
247+ /// for allocators that do not have thread local allocation.
248+ ///
249+ /// Arguments:
250+ /// * `size`: the allocation size in bytes.
251+ /// * `align`: the required alignment in bytes.
252+ /// * `offset` the required offset in bytes.
253+ /// * `need_poll`: if this is true, the implementation must poll for a GC, rather than attempting to allocate from the local buffer.
254+ fn alloc_slow_once_stress_test (
255+ & mut self ,
256+ size : usize ,
257+ align : usize ,
258+ offset : isize ,
259+ need_poll : bool ,
260+ ) -> Address {
261+ // If an allocator does thread local allocation but does not override this method to provide a correct implementation,
262+ // we will log a warning.
263+ if self . does_thread_local_allocation ( ) && need_poll {
264+ warn ! ( "{} does not support stress GC (An allocator that does thread local allocation needs to implement allow_slow_once_stress_test())." , std:: any:: type_name:: <Self >( ) ) ;
265+ }
266+ self . alloc_slow_once ( size, align, offset)
267+ }
200268}
201269
202270impl_downcast ! ( Allocator <VM > where VM : VMBinding ) ;
0 commit comments