diff --git a/library/std/src/sys/pal/unix/mod.rs b/library/std/src/sys/pal/unix/mod.rs index 4fe18daa2040f..12785481e6958 100644 --- a/library/std/src/sys/pal/unix/mod.rs +++ b/library/std/src/sys/pal/unix/mod.rs @@ -23,6 +23,7 @@ pub mod net; #[cfg(target_os = "l4re")] pub use self::l4re::net; pub mod os; +pub mod pi_futex; pub mod pipe; pub mod process; pub mod stack_overflow; diff --git a/library/std/src/sys/pal/unix/pi_futex.rs b/library/std/src/sys/pal/unix/pi_futex.rs new file mode 100644 index 0000000000000..0ebc056ad3931 --- /dev/null +++ b/library/std/src/sys/pal/unix/pi_futex.rs @@ -0,0 +1,195 @@ +#[cfg(any(target_os = "linux", target_os = "android"))] +mod linux { + use crate::cell::Cell; + use crate::ops::Deref; + use crate::sync::atomic::AtomicU32; + use crate::sys::cvt; + use crate::{io, ptr, thread_local}; + + thread_local! { + static TID: Cell = Cell::new(0); + } + + pub type State = u32; + + pub struct Futex(AtomicU32); + + impl Futex { + pub const fn new() -> Futex { + Futex(AtomicU32::new(0)) + } + } + + impl Deref for Futex { + type Target = AtomicU32; + fn deref(&self) -> &AtomicU32 { + &self.0 + } + } + + pub const fn unlocked() -> State { + 0 + } + + pub fn locked() -> State { + let tid = TID.get(); + if tid == 0 { + let tid = (unsafe { libc::gettid() }) as u32; + TID.set(tid); + tid + } else { + tid + } + } + + pub fn is_contended(futex_val: State) -> bool { + (futex_val & libc::FUTEX_WAITERS) != 0 + } + + pub fn is_owned_died(futex_val: State) -> bool { + (futex_val & libc::FUTEX_OWNER_DIED) != 0 + } + + pub fn futex_lock(futex: &Futex) -> io::Result<()> { + loop { + match cvt(unsafe { + libc::syscall( + libc::SYS_futex, + ptr::from_ref(futex.deref()), + libc::FUTEX_LOCK_PI | libc::FUTEX_PRIVATE_FLAG, + 0, + ptr::null::(), + // remaining args are unused + ) + }) { + Ok(_) => return Ok(()), + Err(e) if e.raw_os_error() == Some(libc::EINTR) => continue, + Err(e) => return Err(e), + } + } + } + + pub fn futex_unlock(futex: &Futex) -> io::Result<()> { + cvt(unsafe { + libc::syscall( + libc::SYS_futex, + ptr::from_ref(futex.deref()), + libc::FUTEX_UNLOCK_PI | libc::FUTEX_PRIVATE_FLAG, + // remaining args are unused + ) + }) + .map(|_| ()) + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +pub use linux::*; + +#[cfg(target_os = "freebsd")] +mod freebsd { + use crate::cell::Cell; + use crate::mem::transmute; + use crate::ops::Deref; + use crate::sync::atomic::AtomicU32; + use crate::sys::cvt; + use crate::{io, ptr, thread_local}; + + thread_local! { + static TID: Cell = Cell::new(0); + } + + pub type State = u32; + + #[repr(C)] + pub struct umutex { + m_owner: libc::lwpid_t, + m_flags: u32, + m_ceilings: [u32; 2], + m_rb_link: libc::uintptr_t, + #[cfg(target_pointer_width = "32")] + m_pad: u32, + m_spare: [u32; 2], + } + + pub struct Futex(umutex); + + impl Futex { + pub const fn new() -> Futex { + Futex(umutex { + m_owner: 0, + m_flags: UMUTEX_PRIO_INHERIT, + m_ceilings: [0, 0], + m_rb_link: 0, + #[cfg(target_pointer_width = "32")] + m_pad: 0, + m_spare: [0, 0], + }) + } + } + + impl Deref for Futex { + type Target = AtomicU32; + fn deref(&self) -> &AtomicU32 { + unsafe { transmute(&self.0.m_owner) } + } + } + + const UMUTEX_PRIO_INHERIT: u32 = 0x0004; + const UMUTEX_CONTESTED: u32 = 0x80000000; + + pub const fn unlocked() -> State { + 0 + } + + pub fn locked() -> State { + let tid = TID.get(); + if tid == 0 { + let mut tid: libc::c_long = 0; + let _ = unsafe { libc::thr_self(ptr::from_mut(&mut tid)) }; + let tid = tid as u32; + TID.set(tid); + tid + } else { + tid + } + } + + pub fn is_contended(futex_val: State) -> bool { + (futex_val & UMUTEX_CONTESTED) != 0 + } + + pub fn is_owned_died(futex_val: State) -> bool { + // never happens for non-robust mutex + let _ = futex_val; + false + } + + pub fn futex_lock(futex: &Futex) -> io::Result<()> { + cvt(unsafe { + libc::_umtx_op( + ptr::from_ref(futex.deref()) as _, + libc::UMTX_OP_MUTEX_LOCK, + 0, + ptr::null_mut::(), + ptr::null_mut::(), + ) + }) + .map(|_| ()) + } + + pub fn futex_unlock(futex: &Futex) -> io::Result<()> { + cvt(unsafe { + libc::_umtx_op( + ptr::from_ref(futex.deref()) as _, + libc::UMTX_OP_MUTEX_UNLOCK, + 0, + ptr::null_mut::(), + ptr::null_mut::(), + ) + }) + .map(|_| ()) + } +} + +#[cfg(target_os = "freebsd")] +pub use freebsd::*; diff --git a/library/std/src/sys/sync/mutex/mod.rs b/library/std/src/sys/sync/mutex/mod.rs index 360df3fc4b55d..937cc88aa04b6 100644 --- a/library/std/src/sys/sync/mutex/mod.rs +++ b/library/std/src/sys/sync/mutex/mod.rs @@ -1,9 +1,13 @@ cfg_if::cfg_if! { if #[cfg(any( - all(target_os = "windows", not(target_vendor = "win7")), target_os = "linux", target_os = "android", target_os = "freebsd", + ))] { + mod pi_futex; + pub use pi_futex::Mutex; + } else if #[cfg(any( + all(target_os = "windows", not(target_vendor = "win7")), target_os = "openbsd", target_os = "dragonfly", all(target_family = "wasm", target_feature = "atomics"), diff --git a/library/std/src/sys/sync/mutex/pi_futex.rs b/library/std/src/sys/sync/mutex/pi_futex.rs new file mode 100644 index 0000000000000..d7d1e61c81464 --- /dev/null +++ b/library/std/src/sys/sync/mutex/pi_futex.rs @@ -0,0 +1,82 @@ +use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release}; +use crate::sys::pi_futex as pi; + +pub struct Mutex { + futex: pi::Futex, +} + +impl Mutex { + #[inline] + pub const fn new() -> Self { + Self { futex: pi::Futex::new() } + } + + #[inline] + pub fn try_lock(&self) -> bool { + self.futex.compare_exchange(pi::unlocked(), pi::locked(), Acquire, Relaxed).is_ok() + } + + #[inline] + pub fn lock(&self) { + if self.futex.compare_exchange(pi::unlocked(), pi::locked(), Acquire, Relaxed).is_err() { + self.lock_contended(); + } + } + + #[cold] + fn lock_contended(&self) { + // Spin first to speed things up if the lock is released quickly. + let state = self.spin(); + + // If it's unlocked now, attempt to take the lock. + if state == pi::unlocked() { + if self.try_lock() { + return; + } + }; + + pi::futex_lock(&self.futex).expect("failed to lock mutex"); + + let state = self.futex.load(Relaxed); + if pi::is_owned_died(state) { + panic!( + "failed to lock mutex because the thread owning it finished without unlocking it" + ); + } + } + + fn spin(&self) -> pi::State { + let mut spin = 100; + loop { + // We only use `load` (and not `swap` or `compare_exchange`) + // while spinning, to be easier on the caches. + let state = self.futex.load(Relaxed); + + // We stop spinning when the mutex is unlocked, + // but also when it's contended. + if state == pi::unlocked() || pi::is_contended(state) || spin == 0 { + return state; + } + + crate::hint::spin_loop(); + spin -= 1; + } + } + + #[inline] + pub unsafe fn unlock(&self) { + if self.futex.compare_exchange(pi::locked(), pi::unlocked(), Release, Relaxed).is_err() { + // We only wake up one thread. When that thread locks the mutex, + // the kernel will mark the mutex as contended automatically + // (futex != pi::locked() in this case), + // which makes sure that any other waiting threads will also be + // woken up eventually. + self.wake(); + } + } + + #[cold] + fn wake(&self) { + pi::futex_unlock(&self.futex).unwrap(); + } +}