Use AtomicPtr for head_available

This enables Miri to track pointer provenance even for our raw pointers,
which allows us to check with `-Zmiri-tag-raw-pointers`. Yay!

See rust-lang/miri#1993 for details.

Author: jonhoo

.github/workflows/miri.yml

@@ -22,12 +22,4 @@ jobs:
           command: miri
           args: test
         env:
-          # I would _love_ to enable -Zmiri-track-raw-pointers, but it doesn't
-          # work if the program casts pointers to integers and back, which we
-          # don't do _explicitly_, but we do do _implicitly_ through
-          # `AtomicPtr::compare_exchange`. `load` and `store` work correctly
-          # (https://github.com/rust-lang/rust/pull/77611), as does `swap`
-          # (https://github.com/rust-lang/rust/pull/80236), but we also need
-          # atomic exchange internally. See also
-          # https://github.com/rust-lang/miri/issues/1574.
-          MIRIFLAGS: ""
+          MIRIFLAGS: "-Zmiri-tag-raw-pointers"

src/domain.rs

@@ -186,7 +186,7 @@ macro_rules! new {
             Self {
                 hazptrs: HazPtrRecords {
                     head: AtomicPtr::new(core::ptr::null_mut()),
-                    head_available: AtomicUsize::new(0),
+                    head_available: AtomicPtr::new(core::ptr::null_mut()),
                     count: AtomicIsize::new(0),
                 },
                 untagged,
@@ -257,24 +257,28 @@ impl<F> Domain<F> {
 
         loop {
             let avail = self.hazptrs.head_available.load(Ordering::Acquire);
-            if avail == core::ptr::null::<HazPtrRecord>() as usize {
-                return (core::ptr::null_mut(), 0);
+            if avail.is_null() {
+                return (avail, 0);
             }
-            debug_assert_ne!(avail, core::ptr::null::<HazPtrRecord>() as usize | LOCK_BIT);
+            debug_assert_ne!(avail, LOCK_BIT as *mut _);
             if (avail as usize & LOCK_BIT) == 0 {
-                // Definitely a valid pointer now.
-                let avail: *const HazPtrRecord = avail as _;
-
                 // The available list is not currently locked.
                 //
                 // XXX: This could be a fetch_or and allow progress even if there's a new (but
-                // unlocked) head.
+                // unlocked) head. However, `AtomicPtr` doesn't support fetch_or at the moment, so
+                // we'd have to convert it to an `AtomicUsize`. This will in turn make Miri fail
+                // (with -Zmiri-tag-raw-pointers, which we want enabled) to track the provenance of
+                // the pointer in question through the int-to-ptr conversion. The workaround is
+                // probably to mock a type that is `AtomicUsize` with `fetch_or` with
+                // `#[cfg(not(miri))]`, but is `AtomicPtr` with `compare_exchange` with
+                // `#[cfg(miri)]`. It ain't pretty, but should do the job. The issue is tracked in
+                // https://github.com/rust-lang/miri/issues/1993.
                 if self
                     .hazptrs
                     .head_available
                     .compare_exchange_weak(
-                        avail as usize,
-                        avail as usize | LOCK_BIT,
+                        avail,
+                        with_lock_bit(avail),
                         Ordering::AcqRel,
                         Ordering::Relaxed,
                     )
@@ -317,9 +321,7 @@ impl<F> Domain<F> {
         }
 
         // NOTE: This releases the lock
-        self.hazptrs
-            .head_available
-            .store(next as usize, Ordering::Release);
+        self.hazptrs.head_available.store(next, Ordering::Release);
         unsafe { &*tail }
             .available_next
             .store(core::ptr::null_mut(), Ordering::Relaxed);
@@ -335,15 +337,15 @@ impl<F> Domain<F> {
         debug_assert_eq!(head as *const _ as usize & LOCK_BIT, 0);
         loop {
             let avail = self.hazptrs.head_available.load(Ordering::Acquire);
-            if (avail & LOCK_BIT) == 0 {
+            if (avail as usize & LOCK_BIT) == 0 {
                 tail.available_next
                     .store(avail as *mut _, Ordering::Relaxed);
                 if self
                     .hazptrs
                     .head_available
                     .compare_exchange_weak(
                         avail,
-                        head as *const _ as usize,
+                        head as *const _ as *mut _,
                         Ordering::AcqRel,
                         Ordering::Relaxed,
                     )
@@ -724,7 +726,7 @@ impl<F> Drop for Domain<F> {
 
 struct HazPtrRecords {
     head: AtomicPtr<HazPtrRecord>,
-    head_available: AtomicUsize, // really *mut HazPtrRecord
+    head_available: AtomicPtr<HazPtrRecord>,
     count: AtomicIsize,
 }
 
@@ -815,6 +817,19 @@ impl RetiredList {
     }
 }
 
+// Helpers to set and unset the lock bit on a `*mut HazPtrRecord` without losing pointer
+// provenance. See https://github.com/rust-lang/miri/issues/1993 for details.
+fn with_lock_bit(ptr: *mut HazPtrRecord) -> *mut HazPtrRecord {
+    int_to_ptr_with_provenance(ptr as usize | LOCK_BIT, ptr)
+}
+fn without_lock_bit(ptr: *mut HazPtrRecord) -> *mut HazPtrRecord {
+    int_to_ptr_with_provenance(ptr as usize & !LOCK_BIT, ptr)
+}
+fn int_to_ptr_with_provenance<T>(addr: usize, prov: *mut T) -> *mut T {
+    let ptr = prov.cast::<u8>();
+    ptr.wrapping_add(addr.wrapping_sub(ptr as usize)).cast()
+}
+
 /*
 fn foo() {
     let domain = Domain::new();