vasi_sync/
sync.rs

1//! Synchronization primitives that are modeled in loom
2//!
3//! This module provides some very low-level primitives, such as atomics,
4//! and futex. When testing under loom they model the corresponding operation
5//! in loom instead of executing it natively.
6
7// Use std sync primitives, or loom equivalents
8#[cfg(not(loom))]
9pub use core::{
10    sync::atomic,
11    sync::atomic::{AtomicBool, AtomicI8, AtomicI32, AtomicU32, AtomicUsize, Ordering},
12};
13#[cfg(loom)]
14use std::collections::HashMap;
15
16#[cfg(not(loom))]
17pub use core::cell::Cell;
18#[cfg(loom)]
19pub use loom::cell::Cell;
20
21// Map a *virtual* address to a list of Condvars. This doesn't support mapping into multiple
22// processes, or into different virtual addresses in the same process, etc.
23#[cfg(loom)]
24use loom::sync::{Condvar, Mutex};
25#[cfg(loom)]
26pub use loom::{
27    sync::Arc,
28    sync::atomic,
29    sync::atomic::{AtomicBool, AtomicI8, AtomicI32, AtomicU32, AtomicUsize, Ordering},
30};
31#[cfg(not(loom))]
32use vasi::VirtualAddressSpaceIndependent;
33#[cfg(loom)]
34loom::lazy_static! {
35    pub static ref FUTEXES: Mutex<HashMap<usize, Arc<Condvar>>> = Mutex::new(HashMap::new());
36}
37
38#[cfg(not(loom))]
39pub fn sched_yield() {
40    rustix::thread::sched_yield();
41}
42#[cfg(loom)]
43pub fn sched_yield() {
44    loom::thread::yield_now();
45}
46
47// Rustix doesn't define its `FutexOperation` type under miri, so we can't use it in
48// our interfaces. Use our own type and translate in our futex "backends".
49enum FutexOperation {
50    Wait,
51    Wake,
52}
53
54#[cfg(not(loom))]
55unsafe fn futex(
56    futex_word: &AtomicU32,
57    futex_operation: FutexOperation,
58    val: u32,
59) -> rustix::io::Result<usize> {
60    #[cfg(not(miri))]
61    {
62        let flags = rustix::thread::futex::Flags::empty();
63        match futex_operation {
64            FutexOperation::Wait => {
65                rustix::thread::futex::wait(futex_word, flags, val, None).map(|_| 0)
66            }
67            FutexOperation::Wake => rustix::thread::futex::wake(futex_word, flags, val),
68        }
69    }
70    // Rustix doesn't include `futex` at all under miri. miri understands
71    // futex syscalls made through libc.
72    #[cfg(miri)]
73    {
74        let futex_operation = match futex_operation {
75            FutexOperation::Wait => libc::FUTEX_WAIT,
76            FutexOperation::Wake => libc::FUTEX_WAKE,
77        };
78        let rv = unsafe {
79            libc::syscall(
80                libc::SYS_futex,
81                futex_word.as_ptr(),
82                futex_operation,
83                val,
84                core::ptr::null() as *const libc::timespec,
85                core::ptr::null_mut() as *mut u32,
86                0u32,
87            )
88        };
89        if rv >= 0 {
90            Ok(rv.try_into().unwrap())
91        } else {
92            Err(rustix::io::Errno::from_raw_os_error(unsafe {
93                *libc::__errno_location()
94            }))
95        }
96    }
97}
98
99#[inline]
100pub fn futex_wait(futex_word: &AtomicU32, val: u32) -> rustix::io::Result<usize> {
101    // In "production" we use linux_syscall to avoid going through libc, and to
102    // avoid touching libc's `errno` in particular.
103    #[cfg(not(loom))]
104    {
105        unsafe { futex(futex_word, FutexOperation::Wait, val) }
106    }
107    #[cfg(loom)]
108    {
109        // From futex(2):
110        //   This load, the comparison with the expected value, and starting to
111        //   sleep are performed atomically and totally ordered with
112        //   respect to other futex operations on the same futex word.
113        //
114        // We hold a lock on our FUTEXES to represent this.
115        // TODO: If we want to run loom tests with multiple interacting locks,
116        // we should have per-futex mutexes here, and not hold a lock over the
117        // whole list the whole time.
118        let mut hashmap = FUTEXES.lock().unwrap();
119        let futex_word_val = futex_word.load(Ordering::Relaxed);
120        if futex_word_val != val {
121            return Err(rustix::io::Errno::AGAIN);
122        }
123        let condvar = hashmap
124            .entry(std::ptr::from_ref(futex_word) as usize)
125            .or_insert(Arc::new(Condvar::new()))
126            .clone();
127        // We could get a spurious wakeup here, but that's ok.
128        // Futexes are subject to spurious wakeups too.
129        condvar.wait(hashmap).unwrap();
130        Ok(0)
131    }
132}
133
134#[inline]
135pub fn futex_wake_one(futex_word: &AtomicU32) -> rustix::io::Result<()> {
136    #[cfg(not(loom))]
137    {
138        unsafe { futex(futex_word, FutexOperation::Wake, 1) }.map(|_| ())
139    }
140    // loom doesn't understand syscalls; emulate via loom primitives.
141    #[cfg(loom)]
142    {
143        let hashmap = FUTEXES.lock().unwrap();
144        let Some(condvar) = hashmap.get(&(std::ptr::from_ref(futex_word) as usize)) else {
145            return Ok(());
146        };
147        condvar.notify_one();
148        Ok(())
149    }
150}
151
152#[inline]
153pub fn futex_wake_all(futex_word: &AtomicU32) -> rustix::io::Result<()> {
154    #[cfg(not(loom))]
155    {
156        // u32::MAX seems like it'd make sense here, but the man page says to use INT_MAX.
157        // Better to go with that than risk some unexpected behavior...
158        unsafe {
159            futex(
160                futex_word,
161                FutexOperation::Wake,
162                u32::try_from(i32::MAX).unwrap(),
163            )
164        }
165        .map(|_| ())
166    }
167    // loom doesn't understand syscalls; emulate via loom primitives.
168    #[cfg(loom)]
169    {
170        let hashmap = FUTEXES.lock().unwrap();
171        let Some(condvar) = hashmap.get(&(std::ptr::from_ref(futex_word) as usize)) else {
172            return Ok(());
173        };
174        condvar.notify_all();
175        Ok(())
176    }
177}
178
179#[cfg(not(loom))]
180pub struct MutPtr<T: ?Sized>(*mut T);
181#[cfg(not(loom))]
182impl<T: ?Sized> MutPtr<T> {
183    /// # Safety
184    ///
185    /// See `loom::cell::MutPtr::deref`.
186    #[inline]
187    #[allow(clippy::mut_from_ref)]
188    pub unsafe fn deref(&self) -> &mut T {
189        unsafe { &mut *self.0 }
190    }
191
192    #[inline]
193    pub fn with<F, R>(&self, f: F) -> R
194    where
195        F: FnOnce(*mut T) -> R,
196    {
197        f(self.0)
198    }
199}
200// We have to wrap loom's MutPtr as well, since it's otherwise !Send.
201// https://github.com/tokio-rs/loom/issues/294
202#[cfg(loom)]
203pub struct MutPtr<T: ?Sized>(loom::cell::MutPtr<T>);
204#[cfg(loom)]
205impl<T: ?Sized> MutPtr<T> {
206    #[inline]
207    #[allow(clippy::mut_from_ref)]
208    pub unsafe fn deref(&self) -> &mut T {
209        unsafe { self.0.deref() }
210    }
211
212    #[inline]
213    pub fn with<F, R>(&self, f: F) -> R
214    where
215        F: FnOnce(*mut T) -> R,
216    {
217        self.0.with(f)
218    }
219}
220
221unsafe impl<T: ?Sized> Send for MutPtr<T> where T: Send {}
222
223#[cfg(not(loom))]
224pub struct ConstPtr<T: ?Sized>(*const T);
225#[cfg(not(loom))]
226impl<T: ?Sized> ConstPtr<T> {
227    /// # Safety
228    ///
229    /// See `loom::cell::ConstPtr::deref`.
230    pub unsafe fn deref(&self) -> &T {
231        unsafe { &*self.0 }
232    }
233
234    pub fn with<F, R>(&self, f: F) -> R
235    where
236        F: FnOnce(*const T) -> R,
237    {
238        f(self.0)
239    }
240}
241
242#[cfg(loom)]
243pub use loom::cell::ConstPtr;
244
245/// From <https://docs.rs/loom/latest/loom/#handling-loom-api-differences>
246#[cfg(not(loom))]
247#[derive(Debug, VirtualAddressSpaceIndependent)]
248#[repr(transparent)]
249pub struct UnsafeCell<T>(core::cell::UnsafeCell<T>);
250#[cfg(not(loom))]
251impl<T> UnsafeCell<T> {
252    #[inline]
253    pub const fn new(data: T) -> UnsafeCell<T> {
254        UnsafeCell(core::cell::UnsafeCell::new(data))
255    }
256
257    /// Note that this has a different signature from the method
258    /// of the same name in `core::cell::UnsafeCell`.
259    /// See <https://docs.rs/loom/latest/loom/#handling-loom-api-differences>
260    #[inline]
261    pub fn get_mut(&self) -> MutPtr<T> {
262        MutPtr(self.0.get())
263    }
264
265    #[inline]
266    pub fn get(&self) -> ConstPtr<T> {
267        ConstPtr(self.0.get())
268    }
269
270    /// This is analogous to `core::UnsafeCell::get` in that it returns
271    /// a raw pointer instead of an object.
272    ///
273    /// We can't provide this method under loom without giving up some of loom's
274    /// analysis.
275    pub fn untracked_get(&self) -> *mut T {
276        self.0.get()
277    }
278}
279#[cfg(loom)]
280#[derive(Debug)]
281pub struct UnsafeCell<T>(loom::cell::UnsafeCell<T>);
282#[cfg(loom)]
283impl<T> UnsafeCell<T> {
284    // TODO: make this `const` if and when loom's UnsafeCell supports a const new.
285    pub fn new(data: T) -> UnsafeCell<T> {
286        UnsafeCell(loom::cell::UnsafeCell::new(data))
287    }
288
289    pub fn get_mut(&self) -> MutPtr<T> {
290        MutPtr(self.0.get_mut())
291    }
292
293    pub fn get(&self) -> ConstPtr<T> {
294        self.0.get()
295    }
296}
297
298/// Lets us clear global state in between loom iterations, in loom tests.
299#[cfg(loom)]
300pub fn loom_reset() {
301    FUTEXES.lock().unwrap().clear();
302}
vasi_sync/sync.rs

vasi_sync/
sync.rs
