Skip to content

Commit 06d4b2c

Browse files
Keep Ruby's GC disable state across a forced GC.start
mmtk_handle_user_collection_request enabled collection in mmtk-core when GC was disabled, ran the GC, then disabled collection again and panicked if that failed. Another Ractor could request a pause after the GC ended and before the re-disable, so the process aborted with "failed to re-disable GC after GC". Ractor.new disables GC while it allocates the child's thread, so test_ractor_nt_switch.rb hit this. The same window could also leave GC disabled for good: if the disabling Ractor enabled GC during the window, the re-disable left Disabled(1) in mmtk-core with no owner. In this commit we keep the disable state in the binding instead of in mmtk-core. GcDisableState holds the internal flag (rb_gc_impl_gc_disable), the user flag (GC.disable) and the forced GC.start requests in flight, and mirrors them into mmtk-core as at most one disable_collection call. A forced GC.start enables collection only until a pause ends. resume_mutators applies the state again after mmtk-core sets NotInGC and before mutators resume, so no mutator can request a pause in between.
1 parent 76a2123 commit 06d4b2c

7 files changed

Lines changed: 241 additions & 47 deletions

File tree

‎gc/mmtk/mmtk.c‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
#endif
2323

2424
struct objspace {
25-
bool user_gc_disabled;
2625
bool measure_gc_time;
2726
bool gc_stress;
2827

@@ -837,23 +836,13 @@ rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc)
837836
bool
838837
rb_gc_impl_user_gc_disabled_set(void *objspace_ptr, bool disable)
839838
{
840-
struct objspace *objspace = objspace_ptr;
841-
const bool was = objspace->user_gc_disabled;
842-
objspace->user_gc_disabled = disable;
843-
844-
if (was != disable) {
845-
mmtk_set_gc_enabled(!disable);
846-
objspace->user_gc_disabled = disable;
847-
}
848-
849-
return was;
839+
return mmtk_set_user_gc_disabled(disable);
850840
}
851841

852842
bool
853843
rb_gc_impl_user_gc_disabled_p(void *objspace_ptr)
854844
{
855-
struct objspace *objspace = objspace_ptr;
856-
return objspace->user_gc_disabled;
845+
return mmtk_user_gc_disabled_p();
857846
}
858847

859848
bool

‎gc/mmtk/mmtk.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,10 @@ void mmtk_set_gc_enabled(bool enable);
8888

8989
bool mmtk_gc_enabled_p(void);
9090

91+
bool mmtk_set_user_gc_disabled(bool disable);
92+
93+
bool mmtk_user_gc_disabled_p(void);
94+
9195
size_t mmtk_max_non_los_default_alloc_bytes(void);
9296

9397
MMTk_Address mmtk_alloc(MMTk_Mutator *mutator,

‎gc/mmtk/src/api.rs‎

Lines changed: 17 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,8 @@ pub extern "C" fn mmtk_get_vo_bit_base_addr() -> usize {
249249

250250
#[unsafe(no_mangle)]
251251
pub extern "C" fn mmtk_initialize_collection(tls: VMThread) {
252-
memory_manager::initialize_collection(mmtk(), tls)
252+
memory_manager::initialize_collection(mmtk(), tls);
253+
crate::binding().gc_disable.on_collection_initialized();
253254
}
254255

255256
#[unsafe(no_mangle)]
@@ -291,47 +292,29 @@ pub extern "C" fn mmtk_handle_user_collection_request(
291292
force: bool,
292293
exhaustive: bool,
293294
) {
294-
// GC.start must run even when GC is disabled. However, GC.stress=true does
295-
// not run when GC is disabled. force is true when GC.start and false when
296-
// GC.stress=true.
297-
// TODO: This is not Ractor-safe. Another Ractor could disable GC, which
298-
// would prevent the GC from running. Another Ractor could also enable
299-
// GC, which would be lost.
300-
let gc_was_disabled = force && !crate::mmtk().is_collection_enabled();
301-
if gc_was_disabled {
302-
crate::mmtk().enable_collection();
303-
}
304-
305-
crate::mmtk().handle_user_collection_request(tls, force, exhaustive);
306-
307-
if gc_was_disabled {
308-
crate::mmtk()
309-
.disable_collection()
310-
.unwrap_or_else(|_| panic!("failed to re-disable GC after GC"));
311-
}
295+
crate::binding()
296+
.gc_disable
297+
.handle_user_collection_request(tls, force, exhaustive);
312298
}
313299

314300
#[unsafe(no_mangle)]
315301
pub extern "C" fn mmtk_set_gc_enabled(enable: bool) {
316-
if enable {
317-
crate::mmtk().enable_collection();
318-
} else if crate::mmtk().is_collection_enabled() {
319-
// We must not call disable_collection if GC is already disabled because
320-
// MMTk keeps track of the depth of disable_collection. This would require
321-
// enable_collection calls equal to the number of disable_collection calls
322-
// before GC is enabled.
323-
324-
// We cannot panic on failing to disable GC because
325-
// DURING_GC_COULD_MALLOC_REGION_START disables GC during GC. It's
326-
// fine to ignore the failure because MMTk's malloc does not trigger
327-
// GC.
328-
let _ = crate::mmtk().disable_collection();
329-
}
302+
crate::binding().gc_disable.set_internal_disabled(!enable);
330303
}
331304

332305
#[unsafe(no_mangle)]
333306
pub extern "C" fn mmtk_gc_enabled_p() -> bool {
334-
crate::mmtk().is_collection_enabled()
307+
!crate::binding().gc_disable.internal_disabled()
308+
}
309+
310+
#[unsafe(no_mangle)]
311+
pub extern "C" fn mmtk_set_user_gc_disabled(disable: bool) -> bool {
312+
crate::binding().gc_disable.set_user_disabled(disable)
313+
}
314+
315+
#[unsafe(no_mangle)]
316+
pub extern "C" fn mmtk_user_gc_disabled_p() -> bool {
317+
crate::binding().gc_disable.user_disabled()
335318
}
336319

337320
// =============== Object allocation ===============

‎gc/mmtk/src/binding.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use mmtk::util::ObjectReference;
88

99
use crate::Ruby;
1010
use crate::abi::RubyBindingOptions;
11+
use crate::gc_disable::GcDisableState;
1112
use crate::pinning_registry::PinningRegistry;
1213
use crate::weak_proc::WeakProcessor;
1314

@@ -35,6 +36,7 @@ pub struct RubyBinding {
3536
pub pinning_registry: PinningRegistry,
3637
pub gc_thread_join_handles: Mutex<Vec<JoinHandle<()>>>,
3738
pub wb_unprotected_objects: Mutex<HashSet<ObjectReference>>,
39+
pub gc_disable: GcDisableState,
3840
}
3941

4042
unsafe impl Sync for RubyBinding {}
@@ -54,6 +56,7 @@ impl RubyBinding {
5456
pinning_registry: PinningRegistry::new(),
5557
gc_thread_join_handles: Default::default(),
5658
wb_unprotected_objects: Default::default(),
59+
gc_disable: Default::default(),
5760
}
5861
}
5962

‎gc/mmtk/src/collection.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ impl Collection<Ruby> for VMCollection {
4747
}
4848

4949
fn resume_mutators(_tls: VMWorkerThread) {
50+
crate::binding().gc_disable.on_pause_end();
51+
5052
let current_gc_may_move = CURRENT_GC_MAY_MOVE.load(Ordering::Relaxed);
5153

5254
if current_gc_may_move {

‎gc/mmtk/src/gc_disable.rs‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
2+
3+
use mmtk::util::VMMutatorThread;
4+
5+
/// Ruby's GC-disable state, owned by the binding instead of mmtk-core.
6+
///
7+
/// mmtk-core cannot run a GC while collection is disabled, and every pause ends in `NotInGC`.
8+
/// A forced `GC.start` therefore has to enable collection in mmtk-core, and mmtk-core alone
9+
/// would lose the disabled state across that GC. The binding keeps the state here and mirrors
10+
/// it into mmtk-core, where it holds at most one `disable_collection` once all callers return.
11+
///
12+
/// CRuby's fatal-signal handlers call `rb_gc_impl_gc_disable` and `rb_gc_impl_gc_enable`, and a
13+
/// handler can interrupt a thread anywhere in this module. The state is therefore an atomic word.
14+
/// `apply` runs one caller at a time. A caller that read an old state corrects mmtk-core before
15+
/// it returns, so a stale step never undoes the result of an `apply` that already returned. A
16+
/// handler that interrupts the applying thread does not wait for it, because it would wait
17+
/// forever.
18+
#[derive(Default)]
19+
pub struct GcDisableState {
20+
state: AtomicU64,
21+
/// The thread that runs `apply`, as returned by `current_thread_id`, or 0.
22+
applier: AtomicUsize,
23+
}
24+
25+
/// A nonzero value that is unique to the calling thread. It must be safe to call in a signal
26+
/// handler. A Rust thread-local is not: this library is loaded with `dlopen`, so the first access
27+
/// on a thread can allocate in `__tls_get_addr`. `pthread_self` only reads the thread pointer.
28+
fn current_thread_id() -> usize {
29+
unsafe { libc::pthread_self() as usize }
30+
}
31+
32+
#[derive(Clone, Copy)]
33+
struct State {
34+
/// Set by `rb_gc_impl_gc_disable` and cleared by `rb_gc_impl_gc_enable`.
35+
internal_disabled: bool,
36+
/// Set and cleared by `GC.disable` and `GC.enable`.
37+
user_disabled: bool,
38+
/// Forced `GC.start` calls that enabled collection and wait for a pause.
39+
forced_gc_requests: u32,
40+
/// Incremented when a pause ends. A pause serves every forced request made before it ends.
41+
pause_epoch: u32,
42+
}
43+
44+
impl State {
45+
const INTERNAL_DISABLED: u64 = 1 << 0;
46+
const USER_DISABLED: u64 = 1 << 1;
47+
const FORCED_SHIFT: u32 = 2;
48+
const FORCED_MAX: u32 = (1 << 30) - 1;
49+
const EPOCH_SHIFT: u32 = 32;
50+
51+
fn decode(bits: u64) -> Self {
52+
Self {
53+
internal_disabled: bits & Self::INTERNAL_DISABLED != 0,
54+
user_disabled: bits & Self::USER_DISABLED != 0,
55+
forced_gc_requests: (bits >> Self::FORCED_SHIFT) as u32 & Self::FORCED_MAX,
56+
pause_epoch: (bits >> Self::EPOCH_SHIFT) as u32,
57+
}
58+
}
59+
60+
fn encode(self) -> u64 {
61+
debug_assert!(self.forced_gc_requests <= Self::FORCED_MAX);
62+
(self.internal_disabled as u64)
63+
| (self.user_disabled as u64) << 1
64+
| (self.forced_gc_requests as u64) << Self::FORCED_SHIFT
65+
| (self.pause_epoch as u64) << Self::EPOCH_SHIFT
66+
}
67+
68+
fn want_disabled(self) -> bool {
69+
(self.internal_disabled || self.user_disabled) && self.forced_gc_requests == 0
70+
}
71+
}
72+
73+
impl GcDisableState {
74+
fn load(&self) -> State {
75+
State::decode(self.state.load(Ordering::SeqCst))
76+
}
77+
78+
/// Returns the state before the update.
79+
fn update(&self, f: impl Fn(&mut State)) -> State {
80+
let bits = self
81+
.state
82+
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |bits| {
83+
let mut state = State::decode(bits);
84+
f(&mut state);
85+
Some(state.encode())
86+
})
87+
.unwrap();
88+
State::decode(bits)
89+
}
90+
91+
fn apply(&self) {
92+
let me = current_thread_id();
93+
94+
// A signal handler interrupted this thread in `apply`. CRuby's handlers never return to
95+
// the interrupted code: they abort or longjmp. Apply the state and release `applier`,
96+
// so that a longjmp does not leave it held.
97+
if self.applier.load(Ordering::SeqCst) == me {
98+
self.mirror_into_mmtk();
99+
let _ = self
100+
.applier
101+
.compare_exchange(me, 0, Ordering::SeqCst, Ordering::SeqCst);
102+
return;
103+
}
104+
105+
while self
106+
.applier
107+
.compare_exchange_weak(0, me, Ordering::SeqCst, Ordering::SeqCst)
108+
.is_err()
109+
{
110+
std::thread::yield_now();
111+
}
112+
self.mirror_into_mmtk();
113+
let _ = self
114+
.applier
115+
.compare_exchange(me, 0, Ordering::SeqCst, Ordering::SeqCst);
116+
}
117+
118+
/// Brings mmtk-core to the state that `want_disabled` asks for. The binding is the only
119+
/// caller of `disable_collection` and `enable_collection`, so mmtk-core's status is the
120+
/// mirror.
121+
fn mirror_into_mmtk(&self) {
122+
let mmtk = crate::mmtk();
123+
loop {
124+
let want_disabled = self.load().want_disabled();
125+
let disabled = !mmtk.is_collection_enabled();
126+
if want_disabled == disabled {
127+
return;
128+
}
129+
130+
if want_disabled {
131+
match mmtk.disable_collection() {
132+
Ok(true) => {}
133+
// Only if a signal handler ran `apply` on this thread and then returned.
134+
// Remove the extra nesting level.
135+
Ok(false) => {
136+
mmtk.enable_collection();
137+
}
138+
// A pause is requested or running, or collection is not initialized. The end
139+
// of that pause, or the initialization of collection, applies the state again.
140+
Err(_) => return,
141+
}
142+
} else {
143+
mmtk.enable_collection();
144+
}
145+
}
146+
}
147+
148+
pub fn set_internal_disabled(&self, disabled: bool) {
149+
self.update(|state| state.internal_disabled = disabled);
150+
self.apply();
151+
}
152+
153+
pub fn internal_disabled(&self) -> bool {
154+
self.load().internal_disabled
155+
}
156+
157+
/// Returns the previous value.
158+
pub fn set_user_disabled(&self, disabled: bool) -> bool {
159+
let was = self
160+
.update(|state| state.user_disabled = disabled)
161+
.user_disabled;
162+
self.apply();
163+
was
164+
}
165+
166+
pub fn user_disabled(&self) -> bool {
167+
self.load().user_disabled
168+
}
169+
170+
/// `GC.start` (`force`) runs even when GC is disabled. `GC.stress` does not.
171+
pub fn handle_user_collection_request(
172+
&self,
173+
tls: VMMutatorThread,
174+
force: bool,
175+
exhaustive: bool,
176+
) {
177+
let mmtk = crate::mmtk();
178+
179+
if !force {
180+
mmtk.handle_user_collection_request(tls, false, exhaustive);
181+
return;
182+
}
183+
184+
let epoch = self
185+
.update(|state| state.forced_gc_requests += 1)
186+
.pause_epoch;
187+
188+
self.apply();
189+
mmtk.handle_user_collection_request(tls, true, exhaustive);
190+
191+
self.update(|state| {
192+
if state.pause_epoch == epoch {
193+
state.forced_gc_requests -= 1;
194+
}
195+
});
196+
self.apply();
197+
}
198+
199+
/// Called after mmtk-core sets `NotInGC` at the end of a pause and before mutators resume,
200+
/// so no mutator can request another pause before the disabled state is applied.
201+
pub fn on_pause_end(&self) {
202+
self.update(|state| {
203+
state.forced_gc_requests = 0;
204+
state.pause_epoch = state.pause_epoch.wrapping_add(1);
205+
});
206+
self.apply();
207+
}
208+
209+
pub fn on_collection_initialized(&self) {
210+
self.apply();
211+
}
212+
}

‎gc/mmtk/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ pub mod active_plan;
2626
pub mod api;
2727
pub mod binding;
2828
pub mod collection;
29+
pub mod gc_disable;
2930
pub mod heap;
3031
pub mod object_model;
3132
pub mod pinning_registry;

0 commit comments

Comments
 (0)