diff --git a/src/librustc_codegen_llvm/abi.rs b/src/librustc_codegen_llvm/abi.rs index e50534a4e1dc9..bd7a1c2c3293e 100644 --- a/src/librustc_codegen_llvm/abi.rs +++ b/src/librustc_codegen_llvm/abi.rs @@ -225,9 +225,10 @@ impl ArgTypeExt<'ll, 'tcx> for ArgType<'tcx, Ty<'tcx>> { // ...and then memcpy it to the intended destination. base::call_memcpy(bx, bx.pointercast(dst.llval, Type::i8p(cx)), + self.layout.align, bx.pointercast(llscratch, Type::i8p(cx)), + scratch_align, C_usize(cx, self.layout.size.bytes()), - self.layout.align.min(scratch_align), MemFlags::empty()); bx.lifetime_end(llscratch, scratch_size); diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index 9736994fc0755..dd1348cdf2b6a 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -53,7 +53,7 @@ use mir::place::PlaceRef; use attributes; use builder::{Builder, MemFlags}; use callee; -use common::{C_bool, C_bytes_in_context, C_i32, C_usize}; +use common::{C_bool, C_bytes_in_context, C_usize}; use rustc_mir::monomorphize::item::DefPathBasedNames; use common::{C_struct_in_context, C_array, val_ty}; use consts; @@ -77,7 +77,6 @@ use rustc_data_structures::sync::Lrc; use std::any::Any; use std::cmp; use std::ffi::CString; -use std::i32; use std::ops::{Deref, DerefMut}; use std::sync::mpsc; use std::time::{Instant, Duration}; @@ -319,8 +318,8 @@ pub fn coerce_unsized_into( } if src_f.layout.ty == dst_f.layout.ty { - memcpy_ty(bx, dst_f.llval, src_f.llval, src_f.layout, - src_f.align.min(dst_f.align), MemFlags::empty()); + memcpy_ty(bx, dst_f.llval, dst_f.align, src_f.llval, src_f.align, + src_f.layout, MemFlags::empty()); } else { coerce_unsized_into(bx, src_f, dst_f); } @@ -420,36 +419,34 @@ pub fn to_immediate_scalar( pub fn call_memcpy( bx: &Builder<'_, 'll, '_>, dst: &'ll Value, + dst_align: Align, src: &'ll Value, + src_align: Align, n_bytes: &'ll Value, - align: Align, flags: MemFlags, ) { if flags.contains(MemFlags::NONTEMPORAL) { // HACK(nox): This is inefficient but there is no nontemporal memcpy. - let val = bx.load(src, align); + let val = bx.load(src, src_align); let ptr = bx.pointercast(dst, val_ty(val).ptr_to()); - bx.store_with_flags(val, ptr, align, flags); + bx.store_with_flags(val, ptr, dst_align, flags); return; } let cx = bx.cx; - let ptr_width = &cx.sess().target.target.target_pointer_width; - let key = format!("llvm.memcpy.p0i8.p0i8.i{}", ptr_width); - let memcpy = cx.get_intrinsic(&key); let src_ptr = bx.pointercast(src, Type::i8p(cx)); let dst_ptr = bx.pointercast(dst, Type::i8p(cx)); let size = bx.intcast(n_bytes, cx.isize_ty, false); - let align = C_i32(cx, align.abi() as i32); - let volatile = C_bool(cx, flags.contains(MemFlags::VOLATILE)); - bx.call(memcpy, &[dst_ptr, src_ptr, size, align, volatile], None); + let volatile = flags.contains(MemFlags::VOLATILE); + bx.memcpy(dst_ptr, dst_align.abi(), src_ptr, src_align.abi(), size, volatile); } pub fn memcpy_ty( bx: &Builder<'_, 'll, 'tcx>, dst: &'ll Value, + dst_align: Align, src: &'ll Value, + src_align: Align, layout: TyLayout<'tcx>, - align: Align, flags: MemFlags, ) { let size = layout.size.bytes(); @@ -457,7 +454,7 @@ pub fn memcpy_ty( return; } - call_memcpy(bx, dst, src, C_usize(bx.cx, size), align, flags); + call_memcpy(bx, dst, dst_align, src, src_align, C_usize(bx.cx, size), flags); } pub fn call_memset( diff --git a/src/librustc_codegen_llvm/builder.rs b/src/librustc_codegen_llvm/builder.rs index 2fe6a0377f81b..eaaa450fbf419 100644 --- a/src/librustc_codegen_llvm/builder.rs +++ b/src/librustc_codegen_llvm/builder.rs @@ -784,6 +784,24 @@ impl Builder<'a, 'll, 'tcx> { } } + pub fn memcpy(&self, dst: &'ll Value, dst_align: u64, + src: &'ll Value, src_align: u64, + size: &'ll Value, is_volatile: bool) -> &'ll Value { + unsafe { + llvm::LLVMRustBuildMemCpy(self.llbuilder, dst, dst_align as c_uint, + src, src_align as c_uint, size, is_volatile) + } + } + + pub fn memmove(&self, dst: &'ll Value, dst_align: u64, + src: &'ll Value, src_align: u64, + size: &'ll Value, is_volatile: bool) -> &'ll Value { + unsafe { + llvm::LLVMRustBuildMemMove(self.llbuilder, dst, dst_align as c_uint, + src, src_align as c_uint, size, is_volatile) + } + } + pub fn minnum(&self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { self.count_insn("minnum"); unsafe { diff --git a/src/librustc_codegen_llvm/context.rs b/src/librustc_codegen_llvm/context.rs index 241f7989e1681..59dc0f61ecccd 100644 --- a/src/librustc_codegen_llvm/context.rs +++ b/src/librustc_codegen_llvm/context.rs @@ -530,12 +530,6 @@ fn declare_intrinsic(cx: &CodegenCx<'ll, '_>, key: &str) -> Option<&'ll Value> { let t_v4f64 = Type::vector(t_f64, 4); let t_v8f64 = Type::vector(t_f64, 8); - ifn!("llvm.memcpy.p0i8.p0i8.i16", fn(i8p, i8p, t_i16, t_i32, i1) -> void); - ifn!("llvm.memcpy.p0i8.p0i8.i32", fn(i8p, i8p, t_i32, t_i32, i1) -> void); - ifn!("llvm.memcpy.p0i8.p0i8.i64", fn(i8p, i8p, t_i64, t_i32, i1) -> void); - ifn!("llvm.memmove.p0i8.p0i8.i16", fn(i8p, i8p, t_i16, t_i32, i1) -> void); - ifn!("llvm.memmove.p0i8.p0i8.i32", fn(i8p, i8p, t_i32, t_i32, i1) -> void); - ifn!("llvm.memmove.p0i8.p0i8.i64", fn(i8p, i8p, t_i64, t_i32, i1) -> void); ifn!("llvm.memset.p0i8.i16", fn(i8p, t_i8, t_i16, t_i32, i1) -> void); ifn!("llvm.memset.p0i8.i32", fn(i8p, t_i8, t_i32, t_i32, i1) -> void); ifn!("llvm.memset.p0i8.i64", fn(i8p, t_i8, t_i64, t_i32, i1) -> void); diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs index 03244c18ac3e4..f7cef6be5b529 100644 --- a/src/librustc_codegen_llvm/intrinsic.rs +++ b/src/librustc_codegen_llvm/intrinsic.rs @@ -23,7 +23,7 @@ use glue; use type_::Type; use type_of::LayoutLlvmExt; use rustc::ty::{self, Ty}; -use rustc::ty::layout::{HasDataLayout, LayoutOf}; +use rustc::ty::layout::LayoutOf; use rustc::hir; use syntax::ast; use syntax::symbol::Symbol; @@ -690,28 +690,14 @@ fn copy_intrinsic( let cx = bx.cx; let (size, align) = cx.size_and_align_of(ty); let size = C_usize(cx, size.bytes()); - let align = C_i32(cx, align.abi() as i32); - - let operation = if allow_overlap { - "memmove" - } else { - "memcpy" - }; - - let name = format!("llvm.{}.p0i8.p0i8.i{}", operation, - cx.data_layout().pointer_size.bits()); - + let align = align.abi(); let dst_ptr = bx.pointercast(dst, Type::i8p(cx)); let src_ptr = bx.pointercast(src, Type::i8p(cx)); - let llfn = cx.get_intrinsic(&name); - - bx.call(llfn, - &[dst_ptr, - src_ptr, - bx.mul(size, count), - align, - C_bool(cx, volatile)], - None) + if allow_overlap { + bx.memmove(dst_ptr, align, src_ptr, align, bx.mul(size, count), volatile) + } else { + bx.memcpy(dst_ptr, align, src_ptr, align, bx.mul(size, count), volatile) + } } fn memset_intrinsic( diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs index f046ea030272a..64066d4d786a5 100644 --- a/src/librustc_codegen_llvm/llvm/ffi.rs +++ b/src/librustc_codegen_llvm/llvm/ffi.rs @@ -998,6 +998,22 @@ extern "C" { Bundle: Option<&OperandBundleDef<'a>>, Name: *const c_char) -> &'a Value; + pub fn LLVMRustBuildMemCpy(B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Src: &'a Value, + SrcAlign: c_uint, + Size: &'a Value, + IsVolatile: bool) + -> &'a Value; + pub fn LLVMRustBuildMemMove(B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Src: &'a Value, + SrcAlign: c_uint, + Size: &'a Value, + IsVolatile: bool) + -> &'a Value; pub fn LLVMBuildSelect(B: &Builder<'a>, If: &'a Value, Then: &'a Value, diff --git a/src/librustc_codegen_llvm/mir/block.rs b/src/librustc_codegen_llvm/mir/block.rs index a7f4c48c89bd6..3f9921a5cf930 100644 --- a/src/librustc_codegen_llvm/mir/block.rs +++ b/src/librustc_codegen_llvm/mir/block.rs @@ -784,7 +784,8 @@ impl FunctionCx<'a, 'll, 'tcx> { // have scary latent bugs around. let scratch = PlaceRef::alloca(bx, arg.layout, "arg"); - base::memcpy_ty(bx, scratch.llval, llval, op.layout, align, MemFlags::empty()); + base::memcpy_ty(bx, scratch.llval, scratch.align, llval, align, + op.layout, MemFlags::empty()); (scratch.llval, scratch.align, true) } else { (llval, align, true) diff --git a/src/librustc_codegen_llvm/mir/operand.rs b/src/librustc_codegen_llvm/mir/operand.rs index d1b6aa7fc4280..c76cbfcd97177 100644 --- a/src/librustc_codegen_llvm/mir/operand.rs +++ b/src/librustc_codegen_llvm/mir/operand.rs @@ -282,8 +282,8 @@ impl OperandValue<'ll> { } match self { OperandValue::Ref(r, None, source_align) => { - base::memcpy_ty(bx, dest.llval, r, dest.layout, - source_align.min(dest.align), flags) + base::memcpy_ty(bx, dest.llval, dest.align, r, source_align, + dest.layout, flags) } OperandValue::Ref(_, Some(_), _) => { bug!("cannot directly store unsized values"); @@ -324,7 +324,7 @@ impl OperandValue<'ll> { // Allocate an appropriate region on the stack, and copy the value into it let (llsize, _) = glue::size_and_align_of_dst(&bx, unsized_ty, Some(llextra)); let lldst = bx.array_alloca(Type::i8(bx.cx), llsize, "unsized_tmp", max_align); - base::call_memcpy(&bx, lldst, llptr, llsize, min_align, flags); + base::call_memcpy(&bx, lldst, max_align, llptr, min_align, llsize, flags); // Store the allocated region and the extra to the indirect place. let indirect_operand = OperandValue::Pair(lldst, llextra); diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp index affec73e3ac62..3a1e082b7550d 100644 --- a/src/rustllvm/RustWrapper.cpp +++ b/src/rustllvm/RustWrapper.cpp @@ -1239,6 +1239,40 @@ extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMValueRef Fn, unwrap(Fn), makeArrayRef(unwrap(Args), NumArgs), Bundles, Name)); } +extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, + LLVMValueRef Size, bool IsVolatile) { +#if LLVM_VERSION_GE(7, 0) + return wrap(unwrap(B)->CreateMemCpy( + unwrap(Dst), DstAlign, + unwrap(Src), SrcAlign, + unwrap(Size), IsVolatile)); +#else + unsigned Align = std::min(DstAlign, SrcAlign); + return wrap(unwrap(B)->CreateMemCpy( + unwrap(Dst), unwrap(Src), + unwrap(Size), Align, IsVolatile)); +#endif +} + +extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, + LLVMValueRef Size, bool IsVolatile) { +#if LLVM_VERSION_GE(7, 0) + return wrap(unwrap(B)->CreateMemMove( + unwrap(Dst), DstAlign, + unwrap(Src), SrcAlign, + unwrap(Size), IsVolatile)); +#else + unsigned Align = std::min(DstAlign, SrcAlign); + return wrap(unwrap(B)->CreateMemMove( + unwrap(Dst), unwrap(Src), + unwrap(Size), Align, IsVolatile)); +#endif +} + extern "C" LLVMValueRef LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, LLVMBasicBlockRef Then, diff --git a/src/test/codegen/packed.rs b/src/test/codegen/packed.rs index 10dd12909b644..b50f5b6f16fed 100644 --- a/src/test/codegen/packed.rs +++ b/src/test/codegen/packed.rs @@ -65,7 +65,7 @@ pub struct BigPacked2 { pub fn call_pkd1(f: fn() -> Array) -> BigPacked1 { // CHECK: [[ALLOCA:%[_a-z0-9]+]] = alloca %Array // CHECK: call void %{{.*}}(%Array* noalias nocapture sret dereferenceable(32) [[ALLOCA]]) -// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 %{{.*}}, i8* align 1 %{{.*}}, i{{[0-9]+}} 32, i1 false) +// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 %{{.*}}, i8* align 4 %{{.*}}, i{{[0-9]+}} 32, i1 false) // check that calls whose destination is a field of a packed struct // go through an alloca rather than calling the function with an // unaligned destination. @@ -77,7 +77,7 @@ pub fn call_pkd1(f: fn() -> Array) -> BigPacked1 { pub fn call_pkd2(f: fn() -> Array) -> BigPacked2 { // CHECK: [[ALLOCA:%[_a-z0-9]+]] = alloca %Array // CHECK: call void %{{.*}}(%Array* noalias nocapture sret dereferenceable(32) [[ALLOCA]]) -// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 2 %{{.*}}, i8* align 2 %{{.*}}, i{{[0-9]+}} 32, i1 false) +// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 2 %{{.*}}, i8* align 4 %{{.*}}, i{{[0-9]+}} 32, i1 false) // check that calls whose destination is a field of a packed struct // go through an alloca rather than calling the function with an // unaligned destination. diff --git a/src/test/codegen/stores.rs b/src/test/codegen/stores.rs index 0aaf00bfdbe8e..871bee13b1931 100644 --- a/src/test/codegen/stores.rs +++ b/src/test/codegen/stores.rs @@ -31,7 +31,7 @@ pub fn small_array_alignment(x: &mut [i8; 4], y: [i8; 4]) { // CHECK: store i32 %0, i32* [[TMP]] // CHECK: [[Y8:%[0-9]+]] = bitcast [4 x i8]* %y to i8* // CHECK: [[TMP8:%[0-9]+]] = bitcast i32* [[TMP]] to i8* -// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 1 [[TMP8]], i{{[0-9]+}} 4, i1 false) +// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 4 [[TMP8]], i{{[0-9]+}} 4, i1 false) *x = y; } @@ -45,6 +45,6 @@ pub fn small_struct_alignment(x: &mut Bytes, y: Bytes) { // CHECK: store i32 %0, i32* [[TMP]] // CHECK: [[Y8:%[0-9]+]] = bitcast %Bytes* %y to i8* // CHECK: [[TMP8:%[0-9]+]] = bitcast i32* [[TMP]] to i8* -// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 1 [[TMP8]], i{{[0-9]+}} 4, i1 false) +// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 4 [[TMP8]], i{{[0-9]+}} 4, i1 false) *x = y; }