Skip to content

Commit d3929b2

Browse files
borsbors
committed
Auto merge of #30969 - Amanieu:extended_atomic_cmpxchg, r=alexcrichton
This is an implementation of rust-lang/rfcs#1443.
2 parents d1f422e + 4fdbc2f commit d3929b2

File tree

10 files changed

+577
-49
lines changed

10 files changed

+577
-49
lines changed

‎src/libcore/intrinsics.rs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,33 @@ extern "rust-intrinsic" {
5858
pub fn atomic_cxchg_rel<T>(dst: *mut T, old: T, src: T) -> T;
5959
pub fn atomic_cxchg_acqrel<T>(dst: *mut T, old: T, src: T) -> T;
6060
pub fn atomic_cxchg_relaxed<T>(dst: *mut T, old: T, src: T) -> T;
61+
#[cfg(not(stage0))]
62+
pub fn atomic_cxchg_failrelaxed<T>(dst: *mut T, old: T, src: T) -> T;
63+
#[cfg(not(stage0))]
64+
pub fn atomic_cxchg_failacq<T>(dst: *mut T, old: T, src: T) -> T;
65+
#[cfg(not(stage0))]
66+
pub fn atomic_cxchg_acq_failrelaxed<T>(dst: *mut T, old: T, src: T) -> T;
67+
#[cfg(not(stage0))]
68+
pub fn atomic_cxchg_acqrel_failrelaxed<T>(dst: *mut T, old: T, src: T) -> T;
69+
70+
#[cfg(not(stage0))]
71+
pub fn atomic_cxchgweak<T>(dst: *mut T, old: T, src: T) -> (T, bool);
72+
#[cfg(not(stage0))]
73+
pub fn atomic_cxchgweak_acq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
74+
#[cfg(not(stage0))]
75+
pub fn atomic_cxchgweak_rel<T>(dst: *mut T, old: T, src: T) -> (T, bool);
76+
#[cfg(not(stage0))]
77+
pub fn atomic_cxchgweak_acqrel<T>(dst: *mut T, old: T, src: T) -> (T, bool);
78+
#[cfg(not(stage0))]
79+
pub fn atomic_cxchgweak_relaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
80+
#[cfg(not(stage0))]
81+
pub fn atomic_cxchgweak_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
82+
#[cfg(not(stage0))]
83+
pub fn atomic_cxchgweak_failacq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
84+
#[cfg(not(stage0))]
85+
pub fn atomic_cxchgweak_acq_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
86+
#[cfg(not(stage0))]
87+
pub fn atomic_cxchgweak_acqrel_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
6188

6289
pub fn atomic_load<T>(src: *const T) -> T;
6390
pub fn atomic_load_acq<T>(src: *const T) -> T;

‎src/libcore/sync/atomic.rs‎

Lines changed: 426 additions & 9 deletions
Large diffs are not rendered by default.

‎src/librustc_llvm/lib.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1584,7 +1584,8 @@ extern {
15841584
CMP: ValueRef,
15851585
RHS: ValueRef,
15861586
Order: AtomicOrdering,
1587-
FailureOrder: AtomicOrdering)
1587+
FailureOrder: AtomicOrdering,
1588+
Weak: Bool)
15881589
-> ValueRef;
15891590
pub fn LLVMBuildAtomicRMW(B: BuilderRef,
15901591
Op: AtomicBinOp,

‎src/librustc_trans/trans/build.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1067,8 +1067,9 @@ pub fn Resume(cx: Block, exn: ValueRef) -> ValueRef {
10671067
pub fn AtomicCmpXchg(cx: Block, dst: ValueRef,
10681068
cmp: ValueRef, src: ValueRef,
10691069
order: AtomicOrdering,
1070-
failure_order: AtomicOrdering) -> ValueRef {
1071-
B(cx).atomic_cmpxchg(dst, cmp, src, order, failure_order)
1070+
failure_order: AtomicOrdering,
1071+
weak: llvm::Bool) -> ValueRef {
1072+
B(cx).atomic_cmpxchg(dst, cmp, src, order, failure_order, weak)
10721073
}
10731074
pub fn AtomicRMW(cx: Block, op: AtomicBinOp,
10741075
dst: ValueRef, src: ValueRef,

‎src/librustc_trans/trans/builder.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1077,10 +1077,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
10771077
pub fn atomic_cmpxchg(&self, dst: ValueRef,
10781078
cmp: ValueRef, src: ValueRef,
10791079
order: AtomicOrdering,
1080-
failure_order: AtomicOrdering) -> ValueRef {
1080+
failure_order: AtomicOrdering,
1081+
weak: llvm::Bool) -> ValueRef {
10811082
unsafe {
10821083
llvm::LLVMBuildAtomicCmpXchg(self.llbuilder, dst, cmp, src,
1083-
order, failure_order)
1084+
order, failure_order, weak)
10841085
}
10851086
}
10861087
pub fn atomic_rmw(&self, op: AtomicBinOp,

‎src/librustc_trans/trans/intrinsic.rs‎

Lines changed: 35 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -678,49 +678,54 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>,
678678
// "atomic_<operation>[_<ordering>]", and no ordering means SeqCst
679679
(_, name) if name.starts_with("atomic_") => {
680680
let split: Vec<&str> = name.split('_').collect();
681-
assert!(split.len() >= 2, "Atomic intrinsic not correct format");
682681

683-
let order = if split.len() == 2 {
684-
llvm::SequentiallyConsistent
685-
} else {
686-
match split[2] {
687-
"unordered" => llvm::Unordered,
688-
"relaxed" => llvm::Monotonic,
689-
"acq" => llvm::Acquire,
690-
"rel" => llvm::Release,
691-
"acqrel" => llvm::AcquireRelease,
682+
let (order, failorder) = match split.len() {
683+
2 => (llvm::SequentiallyConsistent, llvm::SequentiallyConsistent),
684+
3 => match split[2] {
685+
"unordered" => (llvm::Unordered, llvm::Unordered),
686+
"relaxed" => (llvm::Monotonic, llvm::Monotonic),
687+
"acq" => (llvm::Acquire, llvm::Acquire),
688+
"rel" => (llvm::Release, llvm::Monotonic),
689+
"acqrel" => (llvm::AcquireRelease, llvm::Acquire),
690+
"failrelaxed" if split[1] == "cxchg" || split[1] == "cxchgweak" =>
691+
(llvm::SequentiallyConsistent, llvm::Monotonic),
692+
"failacq" if split[1] == "cxchg" || split[1] == "cxchgweak" =>
693+
(llvm::SequentiallyConsistent, llvm::Acquire),
692694
_ => ccx.sess().fatal("unknown ordering in atomic intrinsic")
693-
}
695+
},
696+
4 => match (split[2], split[3]) {
697+
("acq", "failrelaxed") if split[1] == "cxchg" || split[1] == "cxchgweak" =>
698+
(llvm::Acquire, llvm::Monotonic),
699+
("acqrel", "failrelaxed") if split[1] == "cxchg" || split[1] == "cxchgweak" =>
700+
(llvm::AcquireRelease, llvm::Monotonic),
701+
_ => ccx.sess().fatal("unknown ordering in atomic intrinsic")
702+
},
703+
_ => ccx.sess().fatal("Atomic intrinsic not in correct format"),
694704
};
695705

696706
match split[1] {
697707
"cxchg" => {
698-
// See include/llvm/IR/Instructions.h for their implementation
699-
// of this, I assume that it's good enough for us to use for
700-
// now.
701-
let strongest_failure_ordering = match order {
702-
llvm::NotAtomic | llvm::Unordered =>
703-
ccx.sess().fatal("cmpxchg must be atomic"),
704-
705-
llvm::Monotonic | llvm::Release =>
706-
llvm::Monotonic,
707-
708-
llvm::Acquire | llvm::AcquireRelease =>
709-
llvm::Acquire,
710-
711-
llvm::SequentiallyConsistent =>
712-
llvm::SequentiallyConsistent
713-
};
714-
715708
let tp_ty = *substs.types.get(FnSpace, 0);
716709
let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
717710
let cmp = from_arg_ty(bcx, llargs[1], tp_ty);
718711
let src = from_arg_ty(bcx, llargs[2], tp_ty);
719-
let res = AtomicCmpXchg(bcx, ptr, cmp, src, order,
720-
strongest_failure_ordering);
712+
let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, failorder, llvm::False);
721713
ExtractValue(bcx, res, 0)
722714
}
723715

716+
"cxchgweak" => {
717+
let tp_ty = *substs.types.get(FnSpace, 0);
718+
let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
719+
let cmp = from_arg_ty(bcx, llargs[1], tp_ty);
720+
let src = from_arg_ty(bcx, llargs[2], tp_ty);
721+
let val = AtomicCmpXchg(bcx, ptr, cmp, src, order, failorder, llvm::True);
722+
let result = ExtractValue(bcx, val, 0);
723+
let success = ZExt(bcx, ExtractValue(bcx, val, 1), Type::bool(bcx.ccx()));
724+
Store(bcx, result, StructGEP(bcx, llresult, 0));
725+
Store(bcx, success, StructGEP(bcx, llresult, 1));
726+
C_nil(ccx)
727+
}
728+
724729
"load" => {
725730
let tp_ty = *substs.types.get(FnSpace, 0);
726731
let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);

‎src/librustc_typeck/check/intrinsic.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ pub fn check_intrinsic_type(ccx: &CrateCtxt, it: &hir::ForeignItem) {
8383
param(ccx, 0),
8484
param(ccx, 0)),
8585
param(ccx, 0)),
86+
"cxchgweak" => (1, vec!(tcx.mk_mut_ptr(param(ccx, 0)),
87+
param(ccx, 0),
88+
param(ccx, 0)),
89+
tcx.mk_tup(vec!(param(ccx, 0), tcx.types.bool))),
8690
"load" => (1, vec!(tcx.mk_imm_ptr(param(ccx, 0))),
8791
param(ccx, 0)),
8892
"store" => (1, vec!(tcx.mk_mut_ptr(param(ccx, 0)), param(ccx, 0)),

‎src/rustllvm/RustWrapper.cpp‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -191,11 +191,15 @@ extern "C" LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B,
191191
LLVMValueRef old,
192192
LLVMValueRef source,
193193
AtomicOrdering order,
194-
AtomicOrdering failure_order) {
195-
return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(target), unwrap(old),
196-
unwrap(source), order,
197-
failure_order
198-
));
194+
AtomicOrdering failure_order,
195+
LLVMBool weak) {
196+
AtomicCmpXchgInst* acxi = unwrap(B)->CreateAtomicCmpXchg(unwrap(target),
197+
unwrap(old),
198+
unwrap(source),
199+
order,
200+
failure_order);
201+
acxi->setWeak(weak);
202+
return wrap(acxi);
199203
}
200204
extern "C" LLVMValueRef LLVMBuildAtomicFence(LLVMBuilderRef B,
201205
AtomicOrdering order,

‎src/test/run-pass/atomic-compare_exchange.rs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
2+
// file at the top-level directory of this distribution and at
3+
// http://rust-lang.org/COPYRIGHT.
4+
//
5+
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6+
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8+
// option. This file may not be copied, modified, or distributed
9+
// except according to those terms.
10+
11+
#![feature(extended_compare_and_swap)]
12+
use std::sync::atomic::{AtomicIsize, ATOMIC_ISIZE_INIT};
13+
use std::sync::atomic::Ordering::*;
14+
15+
static ATOMIC: AtomicIsize = ATOMIC_ISIZE_INIT;
16+
17+
fn main() {
18+
// Make sure trans can emit all the intrinsics correctly
19+
ATOMIC.compare_exchange(0, 1, Relaxed, Relaxed);
20+
ATOMIC.compare_exchange(0, 1, Acquire, Relaxed);
21+
ATOMIC.compare_exchange(0, 1, Release, Relaxed);
22+
ATOMIC.compare_exchange(0, 1, AcqRel, Relaxed);
23+
ATOMIC.compare_exchange(0, 1, SeqCst, Relaxed);
24+
ATOMIC.compare_exchange(0, 1, Acquire, Acquire);
25+
ATOMIC.compare_exchange(0, 1, AcqRel, Acquire);
26+
ATOMIC.compare_exchange(0, 1, SeqCst, Acquire);
27+
ATOMIC.compare_exchange(0, 1, SeqCst, SeqCst);
28+
ATOMIC.compare_exchange_weak(0, 1, Relaxed, Relaxed);
29+
ATOMIC.compare_exchange_weak(0, 1, Acquire, Relaxed);
30+
ATOMIC.compare_exchange_weak(0, 1, Release, Relaxed);
31+
ATOMIC.compare_exchange_weak(0, 1, AcqRel, Relaxed);
32+
ATOMIC.compare_exchange_weak(0, 1, SeqCst, Relaxed);
33+
ATOMIC.compare_exchange_weak(0, 1, Acquire, Acquire);
34+
ATOMIC.compare_exchange_weak(0, 1, AcqRel, Acquire);
35+
ATOMIC.compare_exchange_weak(0, 1, SeqCst, Acquire);
36+
ATOMIC.compare_exchange_weak(0, 1, SeqCst, SeqCst);
37+
}

‎src/test/run-pass/intrinsic-atomics.rs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ mod rusti {
1919
pub fn atomic_cxchg_acq<T>(dst: *mut T, old: T, src: T) -> T;
2020
pub fn atomic_cxchg_rel<T>(dst: *mut T, old: T, src: T) -> T;
2121

22+
pub fn atomic_cxchgweak<T>(dst: *mut T, old: T, src: T) -> (T, bool);
23+
pub fn atomic_cxchgweak_acq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
24+
pub fn atomic_cxchgweak_rel<T>(dst: *mut T, old: T, src: T) -> (T, bool);
25+
2226
pub fn atomic_load<T>(src: *const T) -> T;
2327
pub fn atomic_load_acq<T>(src: *const T) -> T;
2428

@@ -79,5 +83,32 @@ pub fn main() {
7983
assert_eq!(rusti::atomic_xsub_acq(&mut *x, 1), 2);
8084
assert_eq!(rusti::atomic_xsub_rel(&mut *x, 1), 1);
8185
assert_eq!(*x, 0);
86+
87+
loop {
88+
let res = rusti::atomic_cxchgweak(&mut *x, 0, 1);
89+
assert_eq!(res.0, 0);
90+
if res.1 {
91+
break;
92+
}
93+
}
94+
assert_eq!(*x, 1);
95+
96+
loop {
97+
let res = rusti::atomic_cxchgweak_acq(&mut *x, 1, 2);
98+
assert_eq!(res.0, 1);
99+
if res.1 {
100+
break;
101+
}
102+
}
103+
assert_eq!(*x, 2);
104+
105+
loop {
106+
let res = rusti::atomic_cxchgweak_rel(&mut *x, 2, 3);
107+
assert_eq!(res.0, 2);
108+
if res.1 {
109+
break;
110+
}
111+
}
112+
assert_eq!(*x, 3);
82113
}
83114
}

0 commit comments

Comments
 (0)