From 6187e11158f4ed2e4e2d56f33e051995b0f08e2c Mon Sep 17 00:00:00 2001 From: James Miller Date: Sat, 8 Aug 2015 22:28:18 +1200 Subject: [PATCH 1/2] Improve handling of immediate values The primary improvement here is to handle function calls that return imemdiate values such that it doesn't force the use of a stack slot. It also avoids loads from slots that have an store to it in the same block. This especially improves the codegen for intrinsics, as most of them produce immediate values. --- src/librustc_llvm/lib.rs | 1 + src/librustc_trans/trans/base.rs | 123 ++- src/librustc_trans/trans/basic_block.rs | 2 +- src/librustc_trans/trans/builder.rs | 24 +- src/librustc_trans/trans/callee.rs | 60 +- src/librustc_trans/trans/expr.rs | 132 ++- src/librustc_trans/trans/foreign.rs | 15 +- src/librustc_trans/trans/intrinsic.rs | 1101 ++++++++++++----------- src/librustc_trans/trans/type_.rs | 19 +- src/librustc_trans/trans/value.rs | 13 +- 10 files changed, 900 insertions(+), 590 deletions(-) diff --git a/src/librustc_llvm/lib.rs b/src/librustc_llvm/lib.rs index 9ee046915daca..4562281653388 100644 --- a/src/librustc_llvm/lib.rs +++ b/src/librustc_llvm/lib.rs @@ -1559,6 +1559,7 @@ extern { /* Selected entries from the downcasts. */ pub fn LLVMIsATerminatorInst(Inst: ValueRef) -> ValueRef; pub fn LLVMIsAStoreInst(Inst: ValueRef) -> ValueRef; + pub fn LLVMIsAZExtInst(Inst: ValueRef) -> ValueRef; /// Writes a module to the specified path. Returns 0 on success. pub fn LLVMWriteBitcodeToFile(M: ModuleRef, Path: *const c_char) -> c_int; diff --git a/src/librustc_trans/trans/base.rs b/src/librustc_trans/trans/base.rs index d59cc4f4298e9..7a0228f3966b1 100644 --- a/src/librustc_trans/trans/base.rs +++ b/src/librustc_trans/trans/base.rs @@ -54,7 +54,7 @@ use trans::callee; use trans::cleanup::{self, CleanupMethods, DropHint}; use trans::closure; use trans::common::{Block, C_bool, C_bytes_in_context, C_i32, C_int, C_integral}; -use trans::common::{C_null, C_struct_in_context, C_u64, C_u8, C_undef}; +use trans::common::{C_null, C_struct_in_context, C_u64, C_u8, C_undef, C_nil}; use trans::common::{CrateContext, DropFlagHintsMap, Field, FunctionContext}; use trans::common::{Result, NodeIdAndSpan, VariantInfo}; use trans::common::{node_id_type, return_type_is_void}; @@ -699,8 +699,22 @@ pub fn invoke<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, debug_loc: DebugLoc) -> (ValueRef, Block<'blk, 'tcx>) { let _icx = push_ctxt("invoke_"); + + let ret_ty = match fn_ty.sty { + ty::TyBareFn(_, ref f) => { + let output = bcx.tcx().erase_late_bound_regions(&f.sig.output()); + output + } + _ => panic!("expected bare rust fn or closure in trans_call_inner") + }; + if bcx.unreachable.get() { - return (C_null(Type::i8(bcx.ccx())), bcx); + if let ty::FnConverging(ty) = ret_ty { + let llty = type_of::arg_type_of(bcx.ccx(), ty); + return (C_undef(llty), bcx); + } else { + return (C_nil(bcx.ccx()), bcx); + } } let attributes = attributes::from_fn_type(bcx.ccx(), fn_ty); @@ -722,13 +736,20 @@ pub fn invoke<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, let normal_bcx = bcx.fcx.new_temp_block("normal-return"); let landing_pad = bcx.fcx.get_landing_pad(); - let llresult = Invoke(bcx, + let mut llresult = Invoke(bcx, llfn, &llargs[..], normal_bcx.llbb, landing_pad, Some(attributes), debug_loc); + if let ty::FnConverging(ty) = ret_ty { + if return_type_is_void(bcx.ccx(), ty) { + llresult = C_nil(bcx.ccx()); + } + } else { + llresult = C_nil(bcx.ccx()); + } return (llresult, normal_bcx); } else { debug!("calling {} at {:?}", bcx.val_to_string(llfn), bcx.llbb); @@ -736,11 +757,18 @@ pub fn invoke<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, debug!("arg: {}", bcx.val_to_string(llarg)); } - let llresult = Call(bcx, - llfn, - &llargs[..], - Some(attributes), - debug_loc); + let mut llresult = Call(bcx, + llfn, + &llargs[..], + Some(attributes), + debug_loc); + if let ty::FnConverging(ty) = ret_ty { + if return_type_is_void(bcx.ccx(), ty) { + llresult = C_nil(bcx.ccx()); + } + } else { + llresult = C_nil(bcx.ccx()); + } return (llresult, bcx); } } @@ -775,15 +803,33 @@ pub fn load_if_immediate<'blk, 'tcx>(cx: Block<'blk, 'tcx>, /// gives us better information about what we are loading. pub fn load_ty<'blk, 'tcx>(cx: Block<'blk, 'tcx>, ptr: ValueRef, t: Ty<'tcx>) -> ValueRef { + use trans::basic_block::BasicBlock; + if cx.unreachable.get() || type_is_zero_size(cx.ccx(), t) { return C_undef(type_of::type_of(cx.ccx(), t)); } let ptr = to_arg_ty_ptr(cx, ptr, t); + let ptr = Value(ptr); let align = type_of::align_of(cx.ccx(), t); + let bb = BasicBlock(cx.llbb); + if type_is_immediate(cx.ccx(), t) && type_of::type_of(cx.ccx(), t).is_aggregate() { - let load = Load(cx, ptr); + if let Some(val) = ptr.get_dominating_store(cx) { + let valbb = val.get_parent(); + + if Some(bb) == valbb { + if let Some(val) = val.get_operand(0) { + debug!("Eliding load from {}", cx.ccx().tn().val_to_string(ptr.get())); + debug!(" Using previous stored value: {}", + cx.ccx().tn().val_to_string(val.get())); + return val.get(); + } + } + } + + let load = Load(cx, ptr.get()); unsafe { llvm::LLVMSetAlignment(load, align); } @@ -791,7 +837,7 @@ pub fn load_ty<'blk, 'tcx>(cx: Block<'blk, 'tcx>, } unsafe { - let global = llvm::LLVMIsAGlobalVariable(ptr); + let global = llvm::LLVMIsAGlobalVariable(ptr.get()); if !global.is_null() && llvm::LLVMIsGlobalConstant(global) == llvm::True { let val = llvm::LLVMGetInitializer(global); if !val.is_null() { @@ -800,17 +846,30 @@ pub fn load_ty<'blk, 'tcx>(cx: Block<'blk, 'tcx>, } } + if let Some(val) = ptr.get_dominating_store(cx) { + let valbb = val.get_parent(); + + if Some(bb) == valbb { + if let Some(val) = val.get_operand(0) { + debug!("Eliding load from {}", cx.ccx().tn().val_to_string(ptr.get())); + debug!(" Using previous stored value: {}", + cx.ccx().tn().val_to_string(val.get())); + return to_arg_ty(cx, val.get(), t); + } + } + } + let val = if t.is_bool() { - LoadRangeAssert(cx, ptr, 0, 2, llvm::False) + LoadRangeAssert(cx, ptr.get(), 0, 2, llvm::False) } else if t.is_char() { // a char is a Unicode codepoint, and so takes values from 0 // to 0x10FFFF inclusive only. - LoadRangeAssert(cx, ptr, 0, 0x10FFFF + 1, llvm::False) + LoadRangeAssert(cx, ptr.get(), 0, 0x10FFFF + 1, llvm::False) } else if (t.is_region_ptr() || t.is_unique()) && !common::type_is_fat_ptr(cx.tcx(), t) { - LoadNonNull(cx, ptr) + LoadNonNull(cx, ptr.get()) } else { - Load(cx, ptr) + Load(cx, ptr.get()) }; unsafe { @@ -831,7 +890,18 @@ pub fn store_ty<'blk, 'tcx>(cx: Block<'blk, 'tcx>, v: ValueRef, dst: ValueRef, t Store(cx, ExtractValue(cx, v, abi::FAT_PTR_ADDR), expr::get_dataptr(cx, dst)); Store(cx, ExtractValue(cx, v, abi::FAT_PTR_EXTRA), expr::get_len(cx, dst)); } else { - let store = Store(cx, from_arg_ty(cx, v, t), to_arg_ty_ptr(cx, dst, t)); + let vty = val_ty(v); + let dstty = val_ty(dst).element_type(); + + // If the source and destination have the same type, then don't try any conversion, + // this can happen with the return values of functions, as they don't touch an alloca + let (v, dst) = if vty == dstty { + (v, dst) + } else { + (from_arg_ty(cx, v, t), to_arg_ty_ptr(cx, dst, t)) + }; + + let store = Store(cx, v, dst); unsafe { llvm::LLVMSetAlignment(store, type_of::align_of(cx.ccx(), t)); } @@ -848,7 +918,16 @@ pub fn from_arg_ty(bcx: Block, val: ValueRef, ty: Ty) -> ValueRef { pub fn to_arg_ty(bcx: Block, val: ValueRef, ty: Ty) -> ValueRef { if ty.is_bool() { - Trunc(bcx, val, Type::i1(bcx.ccx())) + let val = Value(val); + if let Some(zext) = val.as_zext_inst() { + if let Some(val) = zext.get_operand(0) { + let valty = val_ty(val.get()); + if valty == Type::i1(bcx.ccx()) { + return val.get(); + } + } + } + Trunc(bcx, val.get(), Type::i1(bcx.ccx())) } else { val } @@ -1709,8 +1788,7 @@ pub fn trans_named_tuple_constructor<'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, disr: ty::Disr, args: callee::CallArgs, dest: expr::Dest, - debug_loc: DebugLoc) - -> Result<'blk, 'tcx> { + debug_loc: DebugLoc) -> Result<'blk, 'tcx> { let ccx = bcx.fcx.ccx; @@ -1753,6 +1831,12 @@ pub fn trans_named_tuple_constructor<'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, } } + let llretval = if type_is_immediate(bcx.ccx(), result_ty) { + load_ty(bcx, llresult, result_ty) + } else { + C_undef(type_of::type_of(bcx.ccx(), result_ty)) + }; + // If the caller doesn't care about the result // drop the temporary we made let bcx = match dest { @@ -1766,7 +1850,8 @@ pub fn trans_named_tuple_constructor<'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, } }; - Result::new(bcx, llresult) + + Result::new(bcx, llretval) } pub fn trans_tuple_struct<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, diff --git a/src/librustc_trans/trans/basic_block.rs b/src/librustc_trans/trans/basic_block.rs index d3d055cda1202..290f6d20a9153 100644 --- a/src/librustc_trans/trans/basic_block.rs +++ b/src/librustc_trans/trans/basic_block.rs @@ -13,7 +13,7 @@ use llvm::BasicBlockRef; use trans::value::{Users, Value}; use std::iter::{Filter, Map}; -#[derive(Copy, Clone)] +#[derive(Copy, Clone, PartialEq)] pub struct BasicBlock(pub BasicBlockRef); pub type Preds = Map bool>, fn(Value) -> BasicBlock>; diff --git a/src/librustc_trans/trans/builder.rs b/src/librustc_trans/trans/builder.rs index 107ae378ac446..28c188604c627 100644 --- a/src/librustc_trans/trans/builder.rs +++ b/src/librustc_trans/trans/builder.rs @@ -16,6 +16,7 @@ use llvm::{Opcode, IntPredicate, RealPredicate, False}; use llvm::{ValueRef, BasicBlockRef, BuilderRef, ModuleRef}; use trans::base; use trans::common::*; +use trans::llrepr::LlvmRepr; use trans::machine::llalign_of_pref; use trans::type_::Type; use util::nodemap::FnvHashMap; @@ -158,8 +159,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { args: &[ValueRef], then: BasicBlockRef, catch: BasicBlockRef, - attributes: Option) - -> ValueRef { + attributes: Option) -> ValueRef { self.count_insn("invoke"); debug!("Invoke {} with args ({})", @@ -169,6 +169,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { .collect::>() .join(", ")); + let llfnty = val_ty(llfn); + + for (i, (pty, &a)) in llfnty.func_params().into_iter().zip(args.iter()).enumerate() { + let aty = val_ty(a); + assert!(pty == aty, "Type mismatch for arg {}. {} is not of type {}", + i, self.ccx.tn().val_to_string(a), self.ccx.tn().type_to_string(pty)); + } + unsafe { let v = llvm::LLVMBuildInvoke(self.llbuilder, llfn, @@ -499,9 +507,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } pub fn volatile_store(&self, val: ValueRef, ptr: ValueRef) -> ValueRef { - debug!("Store {} -> {}", - self.ccx.tn().val_to_string(val), - self.ccx.tn().val_to_string(ptr)); assert!(!self.llbuilder.is_null()); self.count_insn("store.volatile"); unsafe { @@ -524,6 +529,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } pub fn gep(&self, ptr: ValueRef, indices: &[ValueRef]) -> ValueRef { + debug!("GEP from {}, indices: {}", + self.ccx.tn().val_to_string(ptr), + indices.llrepr(self.ccx)); self.count_insn("gep"); unsafe { llvm::LLVMBuildGEP(self.llbuilder, ptr, indices.as_ptr(), @@ -535,6 +543,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // in C_i32() #[inline] pub fn gepi(&self, base: ValueRef, ixs: &[usize]) -> ValueRef { + debug!("GEPi from {}, indices: {:?}", + self.ccx.tn().val_to_string(base), + ixs); // Small vector optimization. This should catch 100% of the cases that // we care about. if ixs.len() < 16 { @@ -559,6 +570,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } pub fn struct_gep(&self, ptr: ValueRef, idx: usize) -> ValueRef { + debug!("Struct GEP from {}, index: {}", + self.ccx.tn().val_to_string(ptr), + idx); self.count_insn("structgep"); unsafe { llvm::LLVMBuildStructGEP(self.llbuilder, ptr, idx as c_uint, noname()) diff --git a/src/librustc_trans/trans/callee.rs b/src/librustc_trans/trans/callee.rs index d0d5b46ab2839..e10188bf19038 100644 --- a/src/librustc_trans/trans/callee.rs +++ b/src/librustc_trans/trans/callee.rs @@ -21,7 +21,7 @@ pub use self::CallArgs::*; use arena::TypedArena; use back::link; use session; -use llvm::{self, ValueRef, get_params}; +use llvm::{ValueRef, get_params}; use middle::def; use middle::subst; use middle::subst::{Subst, Substs}; @@ -45,7 +45,6 @@ use trans::foreign; use trans::intrinsic; use trans::meth; use trans::monomorphize; -use trans::type_::Type; use trans::type_of; use middle::ty::{self, Ty, HasTypeFlags, RegionEscape}; use middle::ty::MethodCall; @@ -583,6 +582,23 @@ pub fn trans_call<'a, 'blk, 'tcx>(bcx: Block<'blk, 'tcx>, Some(dest)).bcx } +pub fn trans_datum_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, + call_expr: &ast::Expr, + f: &ast::Expr, + args: CallArgs<'a, 'tcx>) -> DatumBlock<'blk, 'tcx, Expr> { + let ty = common::expr_ty(bcx, call_expr); + + let _icx = push_ctxt("trans_call"); + let val = unpack_result!( + bcx, trans_call_inner(bcx, + call_expr.debug_loc(), + |bcx, _| trans(bcx, f), + args, + Some(expr::Ignore))); + + immediate_rvalue_bcx(bcx, val, ty).to_expr_datumblock() +} + pub fn trans_method_call<'a, 'blk, 'tcx>(bcx: Block<'blk, 'tcx>, call_expr: &ast::Expr, rcvr: &ast::Expr, @@ -705,6 +721,14 @@ pub fn trans_call_inner<'a, 'blk, 'tcx, F>(bcx: Block<'blk, 'tcx>, let is_rust_fn = abi == synabi::Rust || abi == synabi::RustCall; + let llretty = { + let ret_ty = match ret_ty { + ty::FnConverging(ret_ty) => ret_ty, + ty::FnDiverging => ccx.tcx().mk_nil() + }; + type_of::type_of(ccx, ret_ty) + }; + // Generate a location to store the result. If the user does // not care about the result, just make a stack slot. let opt_llretslot = dest.and_then(|dest| match dest { @@ -715,13 +739,12 @@ pub fn trans_call_inner<'a, 'blk, 'tcx, F>(bcx: Block<'blk, 'tcx>, ty::FnDiverging => ccx.tcx().mk_nil() }; if !is_rust_fn || - type_of::return_uses_outptr(ccx, ret_ty) || - bcx.fcx.type_needs_drop(ret_ty) { + type_of::return_uses_outptr(ccx, ret_ty) || + bcx.fcx.type_needs_drop(ret_ty) { // Push the out-pointer if we use an out-pointer for this // return type, otherwise push "undef". if common::type_is_zero_size(ccx, ret_ty) { - let llty = type_of::type_of(ccx, ret_ty); - Some(common::C_undef(llty.ptr_to())) + Some(common::C_undef(llretty.ptr_to())) } else { Some(alloc_ty(bcx, ret_ty, "__llret")) } @@ -731,9 +754,7 @@ pub fn trans_call_inner<'a, 'blk, 'tcx, F>(bcx: Block<'blk, 'tcx>, } }); - let mut llresult = unsafe { - llvm::LLVMGetUndef(Type::nil(ccx).ptr_to().to_ref()) - }; + let llresult; // The code below invokes the function, using either the Rust // conventions (if it is a rust fn) or the native conventions @@ -814,13 +835,16 @@ pub fn trans_call_inner<'a, 'blk, 'tcx, F>(bcx: Block<'blk, 'tcx>, abi); fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean(); - bcx = foreign::trans_native_call(bcx, - callee.ty, - llfn, - opt_llretslot.unwrap(), - &llargs[..], - arg_tys, - debug_loc); + let (llret, b) = foreign::trans_native_call(bcx, + callee.ty, + llfn, + opt_llretslot.unwrap(), + &llargs[..], + arg_tys, + debug_loc); + + bcx = b; + llresult = llret; } fcx.pop_and_trans_custom_cleanup_scope(bcx, arg_cleanup_scope); @@ -1126,8 +1150,8 @@ pub fn trans_arg_datum<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, debug!("--- trans_arg_datum passing {}", bcx.val_to_string(val)); if common::type_is_fat_ptr(bcx.tcx(), formal_arg_ty) { - llargs.push(Load(bcx, expr::get_dataptr(bcx, val))); - llargs.push(Load(bcx, expr::get_len(bcx, val))); + llargs.push(expr::extract_dataptr(bcx, val)); + llargs.push(expr::extract_len(bcx, val)); } else { llargs.push(val); } diff --git a/src/librustc_trans/trans/expr.rs b/src/librustc_trans/trans/expr.rs index c5043f867ded0..80aa065e3a359 100644 --- a/src/librustc_trans/trans/expr.rs +++ b/src/librustc_trans/trans/expr.rs @@ -184,7 +184,7 @@ pub fn trans_into<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, false); bcx.fcx.push_ast_cleanup_scope(cleanup_debug_loc); - let kind = expr_kind(bcx.tcx(), expr); + let kind = expr_kind(bcx, expr); bcx = match kind { ExprKind::Lvalue | ExprKind::RvalueDatum => { trans_unadjusted(bcx, expr).store_to_dest(dest, expr.id) @@ -289,9 +289,27 @@ pub fn get_dataptr(bcx: Block, fat_ptr: ValueRef) -> ValueRef { GEPi(bcx, fat_ptr, &[0, abi::FAT_PTR_ADDR]) } -pub fn copy_fat_ptr(bcx: Block, src_ptr: ValueRef, dst_ptr: ValueRef) { - Store(bcx, Load(bcx, get_dataptr(bcx, src_ptr)), get_dataptr(bcx, dst_ptr)); - Store(bcx, Load(bcx, get_len(bcx, src_ptr)), get_len(bcx, dst_ptr)); +pub fn extract_len(bcx: Block, fat_ptr: ValueRef) -> ValueRef { + let llty = val_ty(fat_ptr); + if llty.is_pointer() { + Load(bcx, get_len(bcx, fat_ptr)) + } else { + ExtractValue(bcx, fat_ptr, abi::FAT_PTR_EXTRA) + } +} + +pub fn extract_dataptr(bcx: Block, fat_ptr: ValueRef) -> ValueRef { + let llty = val_ty(fat_ptr); + if llty.is_pointer() { + Load(bcx, get_dataptr(bcx, fat_ptr)) + } else { + ExtractValue(bcx, fat_ptr, abi::FAT_PTR_ADDR) + } +} + +pub fn copy_fat_ptr(bcx: Block, src: ValueRef, dst_ptr: ValueRef) { + Store(bcx, extract_dataptr(bcx, src), get_dataptr(bcx, dst_ptr)); + Store(bcx, extract_len(bcx, src), get_len(bcx, dst_ptr)); } /// Retrieve the information we are losing (making dynamic) in an unsizing @@ -452,8 +470,8 @@ fn coerce_unsized<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, // to use a different vtable. In that case, we want to // load out the original data pointer so we can repackage // it. - (Load(bcx, get_dataptr(bcx, source.val)), - Some(Load(bcx, get_len(bcx, source.val)))) + (extract_dataptr(bcx, source.val), + Some(extract_len(bcx, source.val))) } else { let val = if source.kind.is_by_ref() { load_ty(bcx, source.val, source.ty) @@ -577,7 +595,7 @@ fn trans_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, debuginfo::set_source_location(bcx.fcx, expr.id, expr.span); - return match expr_kind(bcx.tcx(), expr) { + return match expr_kind(bcx, expr) { ExprKind::Lvalue | ExprKind::RvalueDatum => { let datum = unpack_datum!(bcx, { trans_datum_unadjusted(bcx, expr) @@ -693,6 +711,19 @@ fn trans_datum_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, // Datum output mode means this is a scalar cast: trans_imm_cast(bcx, &**val, expr.id) } + ast::ExprCall(ref f, ref args) => { + if bcx.tcx().is_method_call(expr.id) { + trans_overloaded_datum_call(bcx, + expr, + &**f, + &args[..]) + } else { + callee::trans_datum_call(bcx, + expr, + &**f, + callee::ArgExprs(&args[..])) + } + } _ => { bcx.tcx().sess.span_bug( expr.span, @@ -1494,7 +1525,7 @@ pub fn trans_adt<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, // Second, trans the base to the dest. assert_eq!(discr, 0); - match expr_kind(bcx.tcx(), &*base.expr) { + match expr_kind(bcx, &*base.expr) { ExprKind::RvalueDps | ExprKind::RvalueDatum if !bcx.fcx.type_needs_drop(ty) => { bcx = trans_into(bcx, &*base.expr, SaveIn(addr)); }, @@ -1942,6 +1973,36 @@ fn trans_overloaded_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, bcx } +fn trans_overloaded_datum_call<'a, 'blk, 'tcx>( + mut bcx: Block<'blk, 'tcx>, + expr: &ast::Expr, + callee: &'a ast::Expr, + args: &'a [P]) -> DatumBlock<'blk, 'tcx, Expr> { + + let ty = expr_ty(bcx, expr); + + debug!("trans_overloaded_call_datum {}", expr.id); + let method_call = MethodCall::expr(expr.id); + let mut all_args = vec!(callee); + all_args.extend(args.iter().map(|e| &**e)); + + let val = unpack_result!( + bcx, + callee::trans_call_inner(bcx, + expr.debug_loc(), + |bcx, arg_cleanup_scope| { + meth::trans_method_callee( + bcx, + method_call, + None, + arg_cleanup_scope) + }, + callee::ArgOverloadedCall(all_args), + Some(Ignore))); + + immediate_rvalue_bcx(bcx, val, ty).to_expr_datumblock() +} + pub fn cast_is_noop<'tcx>(tcx: &ty::ctxt<'tcx>, expr: &ast::Expr, t_in: Ty<'tcx>, @@ -2031,18 +2092,36 @@ fn trans_imm_cast<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, } if type_is_fat_ptr(bcx.tcx(), t_in) { - assert!(datum.kind.is_by_ref()); if type_is_fat_ptr(bcx.tcx(), t_out) { - return DatumBlock::new(bcx, Datum::new( - PointerCast(bcx, datum.val, ll_t_out.ptr_to()), - t_out, - Rvalue::new(ByRef) - )).to_expr_datumblock(); + // If the datum is by-ref, just do a pointer cast, otherwise + // construct the casted fat pointer manually + if datum.kind.is_by_ref() { + return DatumBlock::new(bcx, Datum::new( + PointerCast(bcx, datum.val, ll_t_out.ptr_to()), + t_out, + Rvalue::new(ByRef))).to_expr_datumblock(); + } else { + let ftys = ll_t_out.field_types(); + + let val = C_undef(ll_t_out); + let val = InsertValue(bcx, val, + BitCast(bcx, + extract_dataptr(bcx, datum.val), + ftys[abi::FAT_PTR_ADDR]), + abi::FAT_PTR_ADDR); + let val = InsertValue(bcx, val, + BitCast(bcx, + extract_len(bcx, datum.val), + ftys[abi::FAT_PTR_EXTRA]), + abi::FAT_PTR_EXTRA); + + return immediate_rvalue_bcx(bcx, val, t_out).to_expr_datumblock(); + } } else { // Return the address return immediate_rvalue_bcx(bcx, PointerCast(bcx, - Load(bcx, get_dataptr(bcx, datum.val)), + extract_dataptr(bcx, datum.val), ll_t_out), t_out).to_expr_datumblock(); } @@ -2379,13 +2458,10 @@ impl OverflowOpViaIntrinsic { let val = Call(bcx, llfn, &[lhs, rhs], None, binop_debug_loc); let result = ExtractValue(bcx, val, 0); // iN operation result - let overflow = ExtractValue(bcx, val, 1); // i1 "did it overflow?" - - let cond = ICmp(bcx, llvm::IntEQ, overflow, C_integral(Type::i1(bcx.ccx()), 1, false), - binop_debug_loc); + let cond = ExtractValue(bcx, val, 1); // i1 "did it overflow?" let expect = bcx.ccx().get_intrinsic(&"llvm.expect.i1"); - Call(bcx, expect, &[cond, C_integral(Type::i1(bcx.ccx()), 0, false)], + let cond = Call(bcx, expect, &[cond, C_integral(Type::i1(bcx.ccx()), 0, false)], None, binop_debug_loc); let bcx = @@ -2565,7 +2641,9 @@ enum ExprKind { RvalueStmt } -fn expr_kind(tcx: &ty::ctxt, expr: &ast::Expr) -> ExprKind { +fn expr_kind(bcx: Block, expr: &ast::Expr) -> ExprKind { + let tcx = bcx.tcx(); + if tcx.is_method_call(expr.id) { // Overloaded operations are generally calls, and hence they are // generated via DPS, but there are a few exceptions: @@ -2632,7 +2710,15 @@ fn expr_kind(tcx: &ty::ctxt, expr: &ast::Expr) -> ExprKind { ExprKind::Lvalue } - ast::ExprCall(..) | + ast::ExprCall(..) => { + let ty = expr_ty(bcx, expr); + + if type_of::return_uses_outptr(bcx.ccx(), ty) || bcx.fcx.type_needs_drop(ty) { + ExprKind::RvalueDps + } else { + ExprKind::RvalueDatum + } + } ast::ExprMethodCall(..) | ast::ExprStruct(..) | ast::ExprRange(..) | @@ -2694,7 +2780,7 @@ fn expr_kind(tcx: &ty::ctxt, expr: &ast::Expr) -> ExprKind { } } - ast::ExprParen(ref e) => expr_kind(tcx, &**e), + ast::ExprParen(ref e) => expr_kind(bcx, &**e), ast::ExprMac(..) => { tcx.sess.span_bug( diff --git a/src/librustc_trans/trans/foreign.rs b/src/librustc_trans/trans/foreign.rs index 225ff52a63c59..8fef50a2c595a 100644 --- a/src/librustc_trans/trans/foreign.rs +++ b/src/librustc_trans/trans/foreign.rs @@ -229,7 +229,7 @@ pub fn trans_native_call<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, llargs_rust: &[ValueRef], passed_arg_tys: Vec>, call_debug_loc: DebugLoc) - -> Block<'blk, 'tcx> + -> (ValueRef, Block<'blk, 'tcx>) { let ccx = bcx.ccx(); @@ -441,7 +441,18 @@ pub fn trans_native_call<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, } } - return bcx; + let mut llretval = C_undef(llsig.llret_ty); + match fn_sig.output { + ty::FnConverging(result_ty) => { + if !type_of::return_uses_outptr(ccx, result_ty) { + llretval = base::load_ty(bcx, llretptr, result_ty); + } + } + ty::FnDiverging => {} + } + + + return (llretval, bcx); } // feature gate SIMD types in FFI, since I (huonw) am not sure the diff --git a/src/librustc_trans/trans/intrinsic.rs b/src/librustc_trans/trans/intrinsic.rs index 0400771dff15c..b0b5cd2ee7fbc 100644 --- a/src/librustc_trans/trans/intrinsic.rs +++ b/src/librustc_trans/trans/intrinsic.rs @@ -157,8 +157,7 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, args: callee::CallArgs<'a, 'tcx>, dest: expr::Dest, substs: subst::Substs<'tcx>, - call_info: NodeIdAndSpan) - -> Result<'blk, 'tcx> { + call_info: NodeIdAndSpan) -> Result<'blk, 'tcx> { let fcx = bcx.fcx; let ccx = fcx.ccx; let tcx = bcx.tcx(); @@ -183,6 +182,7 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, let (in_type, out_type) = (*substs.types.get(FnSpace, 0), *substs.types.get(FnSpace, 1)); + let llintype = type_of::type_of(ccx, in_type); let llouttype = type_of::type_of(ccx, out_type); @@ -206,12 +206,16 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, // but does, importantly, cover SIMD types. let in_kind = llintype.kind(); let ret_kind = llret_ty.kind(); - let bitcast_compatible = - (nonpointer_nonaggregate(in_kind) && nonpointer_nonaggregate(ret_kind)) || { - in_kind == TypeKind::Pointer && ret_kind == TypeKind::Pointer - }; + let bitcast_compatible = match (in_kind, ret_kind) { + (TypeKind::Pointer, TypeKind::Pointer) | + (TypeKind::Pointer, TypeKind::Integer) | + (TypeKind::Integer, TypeKind::Pointer) => true, + _ => { + nonpointer_nonaggregate(in_kind) && nonpointer_nonaggregate(ret_kind) + } + }; - let dest = if bitcast_compatible { + let val = if bitcast_compatible { // if we're here, the type is scalar-like (a primitive, a // SIMD type or a pointer), and so can be handled as a // by-value ValueRef and can also be directly bitcast to the @@ -229,7 +233,12 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, from_arg_ty(bcx, datum.val, datum.ty) }; - let cast_val = BitCast(bcx, val, llret_ty); + let cast_val = match (in_kind, ret_kind) { + (TypeKind::Pointer, TypeKind::Integer) => PtrToInt(bcx, val, llret_ty), + (TypeKind::Integer, TypeKind::Pointer) => IntToPtr(bcx, val, llret_ty), + (TypeKind::Pointer, TypeKind::Pointer) => PointerCast(bcx, val, llret_ty), + _ => BitCast(bcx, val, llret_ty) + }; match dest { expr::SaveIn(d) => { @@ -239,27 +248,46 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, } expr::Ignore => {} } - dest + cast_val } else { // The types are too complicated to do with a by-value // bitcast, so pointer cast instead. We need to cast the // dest so the types work out. - let dest = match dest { + let cast_dest = match dest { expr::SaveIn(d) => expr::SaveIn(PointerCast(bcx, d, llintype.ptr_to())), - expr::Ignore => expr::Ignore + expr::Ignore => if type_of::return_uses_outptr(bcx.ccx(), out_type) { + expr::Ignore + } else { + expr::SaveIn(alloca(bcx, llintype, "transmute_temp")) + } }; - bcx = expr::trans_into(bcx, &*arg_exprs[0], dest); - dest + + bcx = expr::trans_into(bcx, &*arg_exprs[0], cast_dest); + + let llretty = type_of::arg_type_of(ccx, out_type); + + if type_of::return_uses_outptr(bcx.ccx(), out_type) { + C_nil(ccx) + } else { + match cast_dest { + expr::SaveIn(d) => { + let val = load_ty(bcx, + PointerCast(bcx, d, llouttype.ptr_to()), + out_type); + if let expr::Ignore = dest { + call_lifetime_end(bcx, d) + } + val + } + expr::Ignore => C_undef(llretty) + } + } }; fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean(); fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); - return match dest { - expr::SaveIn(d) => Result::new(bcx, d), - expr::Ignore => Result::new(bcx, C_undef(llret_ty.ptr_to())) - }; - + return Result::new(bcx, val); } _ => { @@ -324,9 +352,11 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, let data = unpack_datum!(bcx, expr::trans(bcx, data)); let data = unpack_datum!(bcx, data.to_rvalue_datum(bcx, "data")); + let ret_ty = tcx.mk_mut_ptr(tcx.types.i8); + let dest = match dest { expr::SaveIn(d) => d, - expr::Ignore => alloc_ty(bcx, tcx.mk_mut_ptr(tcx.types.i8), + expr::Ignore => alloc_ty(bcx, ret_ty, "try_result"), }; @@ -335,7 +365,10 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, call_debug_location); fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); - return Result::new(bcx, dest); + + let retval = load_ty(bcx, dest, ret_ty); + + return Result::new(bcx, retval); } else { ccx.sess().bug("expected two exprs as arguments for \ `try` intrinsic"); @@ -374,548 +407,578 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, let llret_ty = type_of::type_of(ccx, ret_ty); - // Get location to store the result. If the user does - // not care about the result, just make a stack slot - let llresult = match dest { - expr::SaveIn(d) => d, - expr::Ignore => { - if !type_is_zero_size(ccx, ret_ty) { - alloc_ty(bcx, ret_ty, "intrinsic_result") + let mut llslot = None; + let llval = { + let mut get_slot = || { + if let Some(slot) = llslot { + slot } else { - C_undef(llret_ty.ptr_to()) + let slot = match dest { + expr::SaveIn(d) => d, + expr::Ignore => { + if !type_is_zero_size(ccx, ret_ty) { + alloc_ty(bcx, ret_ty, "intrinsic_result") + } else { + C_undef(llret_ty.ptr_to()) + } + } + }; + llslot = Some(slot); + slot } - } - }; + }; - let simple = get_simple_intrinsic(ccx, &*foreign_item); - let llval = match (simple, &*name) { - (Some(llfn), _) => { - Call(bcx, llfn, &llargs, None, call_debug_location) - } - (_, "breakpoint") => { - let llfn = ccx.get_intrinsic(&("llvm.debugtrap")); - Call(bcx, llfn, &[], None, call_debug_location) - } - (_, "size_of") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let lltp_ty = type_of::type_of(ccx, tp_ty); - C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty)) - } - (_, "size_of_val") => { - let tp_ty = *substs.types.get(FnSpace, 0); - if !type_is_sized(tcx, tp_ty) { - let (llsize, _) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]); - llsize - } else { + let simple = get_simple_intrinsic(ccx, &*foreign_item); + match (simple, &*name) { + (Some(llfn), _) => { + Call(bcx, llfn, &llargs, None, call_debug_location) + } + (_, "breakpoint") => { + let llfn = ccx.get_intrinsic(&("llvm.debugtrap")); + Call(bcx, llfn, &[], None, call_debug_location) + } + (_, "size_of") => { + let tp_ty = *substs.types.get(FnSpace, 0); let lltp_ty = type_of::type_of(ccx, tp_ty); C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty)) } - } - (_, "min_align_of") => { - let tp_ty = *substs.types.get(FnSpace, 0); - C_uint(ccx, type_of::align_of(ccx, tp_ty)) - } - (_, "min_align_of_val") => { - let tp_ty = *substs.types.get(FnSpace, 0); - if !type_is_sized(tcx, tp_ty) { - let (_, llalign) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]); - llalign - } else { + (_, "size_of_val") => { + let tp_ty = *substs.types.get(FnSpace, 0); + if !type_is_sized(tcx, tp_ty) { + let (llsize, _) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]); + llsize + } else { + let lltp_ty = type_of::type_of(ccx, tp_ty); + C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty)) + } + } + (_, "min_align_of") => { + let tp_ty = *substs.types.get(FnSpace, 0); C_uint(ccx, type_of::align_of(ccx, tp_ty)) } - } - (_, "pref_align_of") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let lltp_ty = type_of::type_of(ccx, tp_ty); - C_uint(ccx, machine::llalign_of_pref(ccx, lltp_ty)) - } - (_, "drop_in_place") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = if type_is_sized(tcx, tp_ty) { - llargs[0] - } else { - let scratch = rvalue_scratch_datum(bcx, tp_ty, "tmp"); - Store(bcx, llargs[0], expr::get_dataptr(bcx, scratch.val)); - Store(bcx, llargs[1], expr::get_len(bcx, scratch.val)); - fcx.schedule_lifetime_end(cleanup::CustomScope(cleanup_scope), scratch.val); - scratch.val - }; - glue::drop_ty(bcx, ptr, tp_ty, call_debug_location); - C_nil(ccx) - } - (_, "type_name") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ty_name = token::intern_and_get_ident(&tp_ty.to_string()); - C_str_slice(ccx, ty_name) - } - (_, "type_id") => { - let hash = ccx.tcx().hash_crate_independent(*substs.types.get(FnSpace, 0), - &ccx.link_meta().crate_hash); - C_u64(ccx, hash) - } - (_, "init_dropped") => { - let tp_ty = *substs.types.get(FnSpace, 0); - if !return_type_is_void(ccx, tp_ty) { - drop_done_fill_mem(bcx, llresult, tp_ty); + (_, "min_align_of_val") => { + let tp_ty = *substs.types.get(FnSpace, 0); + if !type_is_sized(tcx, tp_ty) { + let (_, llalign) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]); + llalign + } else { + C_uint(ccx, type_of::align_of(ccx, tp_ty)) + } } - C_nil(ccx) - } - (_, "init") => { - let tp_ty = *substs.types.get(FnSpace, 0); - if !return_type_is_void(ccx, tp_ty) { - // Just zero out the stack slot. (See comment on base::memzero for explanation) - init_zero_mem(bcx, llresult, tp_ty); + (_, "pref_align_of") => { + let tp_ty = *substs.types.get(FnSpace, 0); + let lltp_ty = type_of::type_of(ccx, tp_ty); + C_uint(ccx, machine::llalign_of_pref(ccx, lltp_ty)) } - C_nil(ccx) - } - // Effectively no-ops - (_, "uninit") | (_, "forget") => { - C_nil(ccx) - } - (_, "needs_drop") => { - let tp_ty = *substs.types.get(FnSpace, 0); - - C_bool(ccx, bcx.fcx.type_needs_drop(tp_ty)) - } - (_, "offset") => { - let ptr = llargs[0]; - let offset = llargs[1]; - InBoundsGEP(bcx, ptr, &[offset]) - } - (_, "arith_offset") => { - let ptr = llargs[0]; - let offset = llargs[1]; - GEP(bcx, ptr, &[offset]) - } + (_, "drop_in_place") => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = if type_is_sized(tcx, tp_ty) { + llargs[0] + } else { + let scratch = rvalue_scratch_datum(bcx, tp_ty, "tmp"); + Store(bcx, llargs[0], expr::get_dataptr(bcx, scratch.val)); + Store(bcx, llargs[1], expr::get_len(bcx, scratch.val)); + fcx.schedule_lifetime_end(cleanup::CustomScope(cleanup_scope), scratch.val); + scratch.val + }; + glue::drop_ty(bcx, ptr, tp_ty, call_debug_location); + C_nil(ccx) + } + (_, "type_name") => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ty_name = token::intern_and_get_ident(&tp_ty.to_string()); + C_str_slice(ccx, ty_name) + } + (_, "type_id") => { + let hash = ccx.tcx().hash_crate_independent(*substs.types.get(FnSpace, 0), + &ccx.link_meta().crate_hash); + C_u64(ccx, hash) + } + (_, "init_dropped") => { + let tp_ty = *substs.types.get(FnSpace, 0); + if !return_type_is_void(ccx, tp_ty) { + drop_done_fill_mem(bcx, get_slot(), tp_ty); + } + C_nil(ccx) + } + (_, "init") => { + let tp_ty = *substs.types.get(FnSpace, 0); + if type_of::return_uses_outptr(ccx, tp_ty) { + // Just zero out the stack slot. (See comment on base::memzero for explanation) + init_zero_mem(bcx, get_slot(), tp_ty); + C_nil(ccx) + } else { + let lltp_ty = type_of::arg_type_of(ccx, tp_ty); + C_null(lltp_ty) + } + } + // Effectively no-ops + (_, "uninit") | (_, "forget") => { + let tp_ty = *substs.types.get(FnSpace, 0); + if type_of::return_uses_outptr(ccx, tp_ty) { + C_nil(ccx) + } else { + let lltp_ty = type_of::arg_type_of(ccx, tp_ty); + C_undef(lltp_ty) + } + } + (_, "needs_drop") => { + let tp_ty = *substs.types.get(FnSpace, 0); - (_, "copy_nonoverlapping") => { - copy_intrinsic(bcx, - false, - false, - *substs.types.get(FnSpace, 0), - llargs[1], - llargs[0], - llargs[2], - call_debug_location) - } - (_, "copy") => { - copy_intrinsic(bcx, - true, - false, - *substs.types.get(FnSpace, 0), - llargs[1], - llargs[0], - llargs[2], - call_debug_location) - } - (_, "write_bytes") => { - memset_intrinsic(bcx, - false, - *substs.types.get(FnSpace, 0), - llargs[0], - llargs[1], - llargs[2], - call_debug_location) - } + C_bool(ccx, bcx.fcx.type_needs_drop(tp_ty)) + } + (_, "offset") => { + let ptr = llargs[0]; + let offset = llargs[1]; + InBoundsGEP(bcx, ptr, &[offset]) + } + (_, "arith_offset") => { + let ptr = llargs[0]; + let offset = llargs[1]; + GEP(bcx, ptr, &[offset]) + } - (_, "volatile_copy_nonoverlapping_memory") => { - copy_intrinsic(bcx, - false, - true, - *substs.types.get(FnSpace, 0), - llargs[0], - llargs[1], - llargs[2], - call_debug_location) - } - (_, "volatile_copy_memory") => { - copy_intrinsic(bcx, - true, - true, - *substs.types.get(FnSpace, 0), - llargs[0], - llargs[1], - llargs[2], - call_debug_location) - } - (_, "volatile_set_memory") => { - memset_intrinsic(bcx, - true, - *substs.types.get(FnSpace, 0), - llargs[0], - llargs[1], - llargs[2], - call_debug_location) - } - (_, "volatile_load") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - let load = VolatileLoad(bcx, ptr); - unsafe { - llvm::LLVMSetAlignment(load, type_of::align_of(ccx, tp_ty)); + (_, "copy_nonoverlapping") => { + copy_intrinsic(bcx, + false, + false, + *substs.types.get(FnSpace, 0), + llargs[1], + llargs[0], + llargs[2], + call_debug_location) } - to_arg_ty(bcx, load, tp_ty) - }, - (_, "volatile_store") => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - let val = from_arg_ty(bcx, llargs[1], tp_ty); - let store = VolatileStore(bcx, val, ptr); - unsafe { - llvm::LLVMSetAlignment(store, type_of::align_of(ccx, tp_ty)); + (_, "copy") => { + copy_intrinsic(bcx, + true, + false, + *substs.types.get(FnSpace, 0), + llargs[1], + llargs[0], + llargs[2], + call_debug_location) } - C_nil(ccx) - }, - - (_, "ctlz8") => count_zeros_intrinsic(bcx, - "llvm.ctlz.i8", - llargs[0], - call_debug_location), - (_, "ctlz16") => count_zeros_intrinsic(bcx, - "llvm.ctlz.i16", - llargs[0], - call_debug_location), - (_, "ctlz32") => count_zeros_intrinsic(bcx, - "llvm.ctlz.i32", - llargs[0], - call_debug_location), - (_, "ctlz64") => count_zeros_intrinsic(bcx, - "llvm.ctlz.i64", - llargs[0], - call_debug_location), - (_, "cttz8") => count_zeros_intrinsic(bcx, - "llvm.cttz.i8", - llargs[0], - call_debug_location), - (_, "cttz16") => count_zeros_intrinsic(bcx, - "llvm.cttz.i16", - llargs[0], - call_debug_location), - (_, "cttz32") => count_zeros_intrinsic(bcx, - "llvm.cttz.i32", - llargs[0], - call_debug_location), - (_, "cttz64") => count_zeros_intrinsic(bcx, - "llvm.cttz.i64", - llargs[0], - call_debug_location), - - (_, "i8_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.sadd.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i16_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.sadd.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i32_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.sadd.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i64_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.sadd.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - - (_, "u8_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.uadd.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u16_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.uadd.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u32_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.uadd.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u64_add_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.uadd.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i8_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.ssub.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i16_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.ssub.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i32_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.ssub.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i64_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.ssub.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u8_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.usub.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u16_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.usub.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u32_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.usub.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u64_sub_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.usub.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i8_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.smul.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i16_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.smul.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i32_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.smul.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "i64_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.smul.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u8_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.umul.with.overflow.i8", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u16_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.umul.with.overflow.i16", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u32_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.umul.with.overflow.i32", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - (_, "u64_mul_with_overflow") => - with_overflow_intrinsic(bcx, - "llvm.umul.with.overflow.i64", - ret_ty, - llargs[0], - llargs[1], - call_debug_location), - - (_, "unchecked_udiv") => UDiv(bcx, llargs[0], llargs[1], call_debug_location), - (_, "unchecked_sdiv") => SDiv(bcx, llargs[0], llargs[1], call_debug_location), - (_, "unchecked_urem") => URem(bcx, llargs[0], llargs[1], call_debug_location), - (_, "unchecked_srem") => SRem(bcx, llargs[0], llargs[1], call_debug_location), - - (_, "overflowing_add") => Add(bcx, llargs[0], llargs[1], call_debug_location), - (_, "overflowing_sub") => Sub(bcx, llargs[0], llargs[1], call_debug_location), - (_, "overflowing_mul") => Mul(bcx, llargs[0], llargs[1], call_debug_location), - - (_, "return_address") => { - if !fcx.caller_expects_out_pointer { - tcx.sess.span_err(call_info.span, - "invalid use of `return_address` intrinsic: function \ - does not use out pointer"); - C_null(Type::i8p(ccx)) - } else { - PointerCast(bcx, llvm::get_param(fcx.llfn, 0), Type::i8p(ccx)) + (_, "write_bytes") => { + memset_intrinsic(bcx, + false, + *substs.types.get(FnSpace, 0), + llargs[0], + llargs[1], + llargs[2], + call_debug_location) } - } - (_, "discriminant_value") => { - let val_ty = substs.types.get(FnSpace, 0); - match val_ty.sty { - ty::TyEnum(..) => { - let repr = adt::represent_type(ccx, *val_ty); - adt::trans_get_discr(bcx, &*repr, llargs[0], Some(llret_ty)) + (_, "volatile_copy_nonoverlapping_memory") => { + copy_intrinsic(bcx, + false, + true, + *substs.types.get(FnSpace, 0), + llargs[0], + llargs[1], + llargs[2], + call_debug_location) + } + (_, "volatile_copy_memory") => { + copy_intrinsic(bcx, + true, + true, + *substs.types.get(FnSpace, 0), + llargs[0], + llargs[1], + llargs[2], + call_debug_location) + } + (_, "volatile_set_memory") => { + memset_intrinsic(bcx, + true, + *substs.types.get(FnSpace, 0), + llargs[0], + llargs[1], + llargs[2], + call_debug_location) + } + (_, "volatile_load") => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let load = VolatileLoad(bcx, ptr); + unsafe { + llvm::LLVMSetAlignment(load, type_of::align_of(ccx, tp_ty)); + } + to_arg_ty(bcx, load, tp_ty) + }, + (_, "volatile_store") => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let val = from_arg_ty(bcx, llargs[1], tp_ty); + let store = VolatileStore(bcx, val, ptr); + unsafe { + llvm::LLVMSetAlignment(store, type_of::align_of(ccx, tp_ty)); + } + C_nil(ccx) + }, + + (_, "ctlz8") => count_zeros_intrinsic(bcx, + "llvm.ctlz.i8", + llargs[0], + call_debug_location), + (_, "ctlz16") => count_zeros_intrinsic(bcx, + "llvm.ctlz.i16", + llargs[0], + call_debug_location), + (_, "ctlz32") => count_zeros_intrinsic(bcx, + "llvm.ctlz.i32", + llargs[0], + call_debug_location), + (_, "ctlz64") => count_zeros_intrinsic(bcx, + "llvm.ctlz.i64", + llargs[0], + call_debug_location), + (_, "cttz8") => count_zeros_intrinsic(bcx, + "llvm.cttz.i8", + llargs[0], + call_debug_location), + (_, "cttz16") => count_zeros_intrinsic(bcx, + "llvm.cttz.i16", + llargs[0], + call_debug_location), + (_, "cttz32") => count_zeros_intrinsic(bcx, + "llvm.cttz.i32", + llargs[0], + call_debug_location), + (_, "cttz64") => count_zeros_intrinsic(bcx, + "llvm.cttz.i64", + llargs[0], + call_debug_location), + + (_, "i8_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.sadd.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i16_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.sadd.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i32_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.sadd.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i64_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.sadd.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + + (_, "u8_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.uadd.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u16_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.uadd.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u32_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.uadd.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u64_add_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.uadd.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i8_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.ssub.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i16_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.ssub.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i32_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.ssub.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i64_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.ssub.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u8_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.usub.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u16_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.usub.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u32_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.usub.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u64_sub_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.usub.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i8_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.smul.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i16_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.smul.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i32_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.smul.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "i64_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.smul.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u8_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.umul.with.overflow.i8", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u16_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.umul.with.overflow.i16", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u32_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.umul.with.overflow.i32", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + (_, "u64_mul_with_overflow") => + with_overflow_intrinsic(bcx, + "llvm.umul.with.overflow.i64", + ret_ty, + llargs[0], + llargs[1], + call_debug_location), + + (_, "unchecked_udiv") => UDiv(bcx, llargs[0], llargs[1], call_debug_location), + (_, "unchecked_sdiv") => SDiv(bcx, llargs[0], llargs[1], call_debug_location), + (_, "unchecked_urem") => URem(bcx, llargs[0], llargs[1], call_debug_location), + (_, "unchecked_srem") => SRem(bcx, llargs[0], llargs[1], call_debug_location), + + (_, "overflowing_add") => Add(bcx, llargs[0], llargs[1], call_debug_location), + (_, "overflowing_sub") => Sub(bcx, llargs[0], llargs[1], call_debug_location), + (_, "overflowing_mul") => Mul(bcx, llargs[0], llargs[1], call_debug_location), + + (_, "return_address") => { + if !fcx.caller_expects_out_pointer { + tcx.sess.span_err(call_info.span, + "invalid use of `return_address` intrinsic: function \ + does not use out pointer"); + C_null(Type::i8p(ccx)) + } else { + PointerCast(bcx, llvm::get_param(fcx.llfn, 0), Type::i8p(ccx)) } - _ => C_null(llret_ty) } - } - // This requires that atomic intrinsics follow a specific naming pattern: - // "atomic_[_]", and no ordering means SeqCst - (_, name) if name.starts_with("atomic_") => { - let split: Vec<&str> = name.split('_').collect(); - assert!(split.len() >= 2, "Atomic intrinsic not correct format"); - - let order = if split.len() == 2 { - llvm::SequentiallyConsistent - } else { - match split[2] { - "unordered" => llvm::Unordered, - "relaxed" => llvm::Monotonic, - "acq" => llvm::Acquire, - "rel" => llvm::Release, - "acqrel" => llvm::AcquireRelease, - _ => ccx.sess().fatal("unknown ordering in atomic intrinsic") + (_, "discriminant_value") => { + let val_ty = substs.types.get(FnSpace, 0); + match val_ty.sty { + ty::TyEnum(..) => { + let repr = adt::represent_type(ccx, *val_ty); + adt::trans_get_discr(bcx, &*repr, llargs[0], Some(llret_ty)) + } + _ => C_null(llret_ty) } - }; - - match split[1] { - "cxchg" => { - // See include/llvm/IR/Instructions.h for their implementation - // of this, I assume that it's good enough for us to use for - // now. - let strongest_failure_ordering = match order { - llvm::NotAtomic | llvm::Unordered => - ccx.sess().fatal("cmpxchg must be atomic"), + } - llvm::Monotonic | llvm::Release => - llvm::Monotonic, + // This requires that atomic intrinsics follow a specific naming pattern: + // "atomic_[_]", and no ordering means SeqCst + (_, name) if name.starts_with("atomic_") => { + let split: Vec<&str> = name.split('_').collect(); + assert!(split.len() >= 2, "Atomic intrinsic not correct format"); - llvm::Acquire | llvm::AcquireRelease => - llvm::Acquire, + let order = if split.len() == 2 { + llvm::SequentiallyConsistent + } else { + match split[2] { + "unordered" => llvm::Unordered, + "relaxed" => llvm::Monotonic, + "acq" => llvm::Acquire, + "rel" => llvm::Release, + "acqrel" => llvm::AcquireRelease, + _ => ccx.sess().fatal("unknown ordering in atomic intrinsic") + } + }; - llvm::SequentiallyConsistent => - llvm::SequentiallyConsistent - }; + match split[1] { + "cxchg" => { + // See include/llvm/IR/Instructions.h for their implementation + // of this, I assume that it's good enough for us to use for + // now. + let strongest_failure_ordering = match order { + llvm::NotAtomic | llvm::Unordered => + ccx.sess().fatal("cmpxchg must be atomic"), + + llvm::Monotonic | llvm::Release => + llvm::Monotonic, + + llvm::Acquire | llvm::AcquireRelease => + llvm::Acquire, + + llvm::SequentiallyConsistent => + llvm::SequentiallyConsistent + }; + + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let cmp = from_arg_ty(bcx, llargs[1], tp_ty); + let src = from_arg_ty(bcx, llargs[2], tp_ty); + let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, + strongest_failure_ordering); + ExtractValue(bcx, res, 0) + } - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - let cmp = from_arg_ty(bcx, llargs[1], tp_ty); - let src = from_arg_ty(bcx, llargs[2], tp_ty); - let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, - strongest_failure_ordering); - ExtractValue(bcx, res, 0) - } + "load" => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + to_arg_ty(bcx, AtomicLoad(bcx, ptr, order), tp_ty) + } + "store" => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let val = from_arg_ty(bcx, llargs[1], tp_ty); + AtomicStore(bcx, val, ptr, order); + C_nil(ccx) + } - "load" => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - to_arg_ty(bcx, AtomicLoad(bcx, ptr, order), tp_ty) - } - "store" => { - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - let val = from_arg_ty(bcx, llargs[1], tp_ty); - AtomicStore(bcx, val, ptr, order); - C_nil(ccx) - } + "fence" => { + AtomicFence(bcx, order, llvm::CrossThread); + C_nil(ccx) + } - "fence" => { - AtomicFence(bcx, order, llvm::CrossThread); - C_nil(ccx) - } + "singlethreadfence" => { + AtomicFence(bcx, order, llvm::SingleThread); + C_nil(ccx) + } - "singlethreadfence" => { - AtomicFence(bcx, order, llvm::SingleThread); - C_nil(ccx) + // These are all AtomicRMW ops + op => { + let atom_op = match op { + "xchg" => llvm::AtomicXchg, + "xadd" => llvm::AtomicAdd, + "xsub" => llvm::AtomicSub, + "and" => llvm::AtomicAnd, + "nand" => llvm::AtomicNand, + "or" => llvm::AtomicOr, + "xor" => llvm::AtomicXor, + "max" => llvm::AtomicMax, + "min" => llvm::AtomicMin, + "umax" => llvm::AtomicUMax, + "umin" => llvm::AtomicUMin, + _ => ccx.sess().fatal("unknown atomic operation") + }; + + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let val = from_arg_ty(bcx, llargs[1], tp_ty); + AtomicRMW(bcx, atom_op, ptr, val, order) + } } - // These are all AtomicRMW ops - op => { - let atom_op = match op { - "xchg" => llvm::AtomicXchg, - "xadd" => llvm::AtomicAdd, - "xsub" => llvm::AtomicSub, - "and" => llvm::AtomicAnd, - "nand" => llvm::AtomicNand, - "or" => llvm::AtomicOr, - "xor" => llvm::AtomicXor, - "max" => llvm::AtomicMax, - "min" => llvm::AtomicMin, - "umax" => llvm::AtomicUMax, - "umin" => llvm::AtomicUMin, - _ => ccx.sess().fatal("unknown atomic operation") - }; - - let tp_ty = *substs.types.get(FnSpace, 0); - let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); - let val = from_arg_ty(bcx, llargs[1], tp_ty); - AtomicRMW(bcx, atom_op, ptr, val, order) - } } + (_, _) => ccx.sess().span_bug(foreign_item.span, "unknown intrinsic") } - - (_, _) => ccx.sess().span_bug(foreign_item.span, "unknown intrinsic") }; - if val_ty(llval) != Type::void(ccx) && - machine::llsize_of_alloc(ccx, val_ty(llval)) != 0 { - store_ty(bcx, llval, llresult, ret_ty); - } + let llretval = if let Some(slot) = llslot { + if !type_of::return_uses_outptr(ccx, ret_ty) { + load_ty(bcx, slot, ret_ty) + } else { + C_nil(ccx) + } + } else { + llval + }; // If we made a temporary stack slot, let's clean it up match dest { expr::Ignore => { - bcx = glue::drop_ty(bcx, llresult, ret_ty, call_debug_location); + if let Some(slot) = llslot { + bcx = glue::drop_ty(bcx, slot, ret_ty, call_debug_location); + } + } + expr::SaveIn(slot) => { + if val_ty(llval) != Type::void(ccx) + && machine::llsize_of_alloc(ccx, val_ty(llval)) != 0 { + store_ty(bcx, llval, slot, ret_ty); + } } - expr::SaveIn(_) => {} } fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); - Result::new(bcx, llresult) + Result::new(bcx, llretval) } fn copy_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, diff --git a/src/librustc_trans/trans/type_.rs b/src/librustc_trans/trans/type_.rs index 699115a070986..d2ba23caaad4f 100644 --- a/src/librustc_trans/trans/type_.rs +++ b/src/librustc_trans/trans/type_.rs @@ -227,6 +227,10 @@ impl Type { } } + pub fn is_pointer(&self) -> bool { + self.kind() == TypeKind::Pointer + } + pub fn is_packed(&self) -> bool { unsafe { llvm::LLVMIsPackedStruct(self.to_ref()) == True @@ -270,10 +274,21 @@ impl Type { } pub fn func_params(&self) -> Vec { + let mut this = *self; + loop { + let kind = this.kind(); + if kind == TypeKind::Function { + break; + } else if kind == TypeKind::Pointer { + this = this.element_type(); + } else { + panic!("Type is not a function or function pointer"); + } + } unsafe { - let n_args = llvm::LLVMCountParamTypes(self.to_ref()) as usize; + let n_args = llvm::LLVMCountParamTypes(this.to_ref()) as usize; let mut args = vec![Type { rf: ptr::null_mut() }; n_args]; - llvm::LLVMGetParamTypes(self.to_ref(), + llvm::LLVMGetParamTypes(this.to_ref(), args.as_mut_ptr() as *mut TypeRef); args } diff --git a/src/librustc_trans/trans/value.rs b/src/librustc_trans/trans/value.rs index bc71278c15743..0c78417109e95 100644 --- a/src/librustc_trans/trans/value.rs +++ b/src/librustc_trans/trans/value.rs @@ -14,7 +14,7 @@ use trans::basic_block::BasicBlock; use trans::common::Block; use libc::c_uint; -#[derive(Copy, Clone)] +#[derive(Copy, Clone, PartialEq)] pub struct Value(pub ValueRef); macro_rules! opt_val { ($e:expr) => ( @@ -57,6 +57,13 @@ impl Value { pub fn get_dominating_store(self, bcx: Block) -> Option { match self.get_single_user().and_then(|user| user.as_store_inst()) { Some(store) => { + // Make sure that the store instruction is /to/ this value, + // not *of* this value + if let Some(loc) = store.get_operand(1) { + if loc != self { + return None; + } + } store.get_parent().and_then(|store_bb| { let mut bb = BasicBlock(bcx.llbb); let mut ret = Some(store); @@ -116,6 +123,10 @@ impl Value { opt_val!(llvm::LLVMIsAStoreInst(self.get())) } + pub fn as_zext_inst(self) -> Option { + opt_val!(llvm::LLVMIsAZExtInst(self.get())) + } + /// Tests if this value is a terminator instruction pub fn is_a_terminator_inst(self) -> bool { unsafe { From 890534323ceb10d9aca615af321c7a7a1e587b90 Mon Sep 17 00:00:00 2001 From: James Miller Date: Mon, 10 Aug 2015 17:32:35 +1200 Subject: [PATCH 2/2] Clean up and refactor some code * Move the "get the value that will definitely be loaded" logic to it's own method to reduce the duplication. * Refactor base::invoke to avoid duplicating the diverging/zero-sized return logic. * Re-add debugging output I accidentally removed * Add some more comments explaining stuff --- src/librustc_trans/trans/base.rs | 84 ++++++++++++----------------- src/librustc_trans/trans/builder.rs | 3 ++ src/librustc_trans/trans/expr.rs | 2 +- src/librustc_trans/trans/value.rs | 13 +++++ 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/librustc_trans/trans/base.rs b/src/librustc_trans/trans/base.rs index 7a0228f3966b1..1efc0c22838f2 100644 --- a/src/librustc_trans/trans/base.rs +++ b/src/librustc_trans/trans/base.rs @@ -728,7 +728,7 @@ pub fn invoke<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, } } - if need_invoke(bcx) { + let (mut llresult, bcx) = if need_invoke(bcx) { debug!("invoking {} at {:?}", bcx.val_to_string(llfn), bcx.llbb); for &llarg in llargs { debug!("arg: {}", bcx.val_to_string(llarg)); @@ -736,41 +736,37 @@ pub fn invoke<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, let normal_bcx = bcx.fcx.new_temp_block("normal-return"); let landing_pad = bcx.fcx.get_landing_pad(); - let mut llresult = Invoke(bcx, + let llresult = Invoke(bcx, llfn, &llargs[..], normal_bcx.llbb, landing_pad, Some(attributes), debug_loc); - if let ty::FnConverging(ty) = ret_ty { - if return_type_is_void(bcx.ccx(), ty) { - llresult = C_nil(bcx.ccx()); - } - } else { - llresult = C_nil(bcx.ccx()); - } - return (llresult, normal_bcx); + (llresult, normal_bcx) } else { debug!("calling {} at {:?}", bcx.val_to_string(llfn), bcx.llbb); for &llarg in llargs { debug!("arg: {}", bcx.val_to_string(llarg)); } - let mut llresult = Call(bcx, + let llresult = Call(bcx, llfn, &llargs[..], Some(attributes), debug_loc); - if let ty::FnConverging(ty) = ret_ty { - if return_type_is_void(bcx.ccx(), ty) { - llresult = C_nil(bcx.ccx()); - } - } else { + (llresult, bcx) + }; + + if let ty::FnConverging(ty) = ret_ty { + if return_type_is_void(bcx.ccx(), ty) { llresult = C_nil(bcx.ccx()); } - return (llresult, bcx); + } else { + llresult = C_nil(bcx.ccx()); } + + (llresult, bcx) } pub fn need_invoke(bcx: Block) -> bool { @@ -803,33 +799,22 @@ pub fn load_if_immediate<'blk, 'tcx>(cx: Block<'blk, 'tcx>, /// gives us better information about what we are loading. pub fn load_ty<'blk, 'tcx>(cx: Block<'blk, 'tcx>, ptr: ValueRef, t: Ty<'tcx>) -> ValueRef { - use trans::basic_block::BasicBlock; - if cx.unreachable.get() || type_is_zero_size(cx.ccx(), t) { return C_undef(type_of::type_of(cx.ccx(), t)); } let ptr = to_arg_ty_ptr(cx, ptr, t); - let ptr = Value(ptr); let align = type_of::align_of(cx.ccx(), t); - let bb = BasicBlock(cx.llbb); - if type_is_immediate(cx.ccx(), t) && type_of::type_of(cx.ccx(), t).is_aggregate() { - if let Some(val) = ptr.get_dominating_store(cx) { - let valbb = val.get_parent(); - - if Some(bb) == valbb { - if let Some(val) = val.get_operand(0) { - debug!("Eliding load from {}", cx.ccx().tn().val_to_string(ptr.get())); - debug!(" Using previous stored value: {}", - cx.ccx().tn().val_to_string(val.get())); - return val.get(); - } - } + if let Some(val) = Value(ptr).get_stored_value_opt(cx) { + debug!("Eliding load from {}", cx.ccx().tn().val_to_string(ptr)); + debug!(" Using previous stored value: {}", + cx.ccx().tn().val_to_string(val.get())); + return val.get(); } - let load = Load(cx, ptr.get()); + let load = Load(cx, ptr); unsafe { llvm::LLVMSetAlignment(load, align); } @@ -837,7 +822,7 @@ pub fn load_ty<'blk, 'tcx>(cx: Block<'blk, 'tcx>, } unsafe { - let global = llvm::LLVMIsAGlobalVariable(ptr.get()); + let global = llvm::LLVMIsAGlobalVariable(ptr); if !global.is_null() && llvm::LLVMIsGlobalConstant(global) == llvm::True { let val = llvm::LLVMGetInitializer(global); if !val.is_null() { @@ -846,30 +831,24 @@ pub fn load_ty<'blk, 'tcx>(cx: Block<'blk, 'tcx>, } } - if let Some(val) = ptr.get_dominating_store(cx) { - let valbb = val.get_parent(); - - if Some(bb) == valbb { - if let Some(val) = val.get_operand(0) { - debug!("Eliding load from {}", cx.ccx().tn().val_to_string(ptr.get())); - debug!(" Using previous stored value: {}", - cx.ccx().tn().val_to_string(val.get())); - return to_arg_ty(cx, val.get(), t); - } - } + if let Some(val) = Value(ptr).get_stored_value_opt(cx) { + debug!("Eliding load from {}", cx.ccx().tn().val_to_string(ptr)); + debug!(" Using previous stored value: {}", + cx.ccx().tn().val_to_string(val.get())); + return to_arg_ty(cx, val.get(), t); } let val = if t.is_bool() { - LoadRangeAssert(cx, ptr.get(), 0, 2, llvm::False) + LoadRangeAssert(cx, ptr, 0, 2, llvm::False) } else if t.is_char() { // a char is a Unicode codepoint, and so takes values from 0 // to 0x10FFFF inclusive only. - LoadRangeAssert(cx, ptr.get(), 0, 0x10FFFF + 1, llvm::False) + LoadRangeAssert(cx, ptr, 0, 0x10FFFF + 1, llvm::False) } else if (t.is_region_ptr() || t.is_unique()) && !common::type_is_fat_ptr(cx.tcx(), t) { - LoadNonNull(cx, ptr.get()) + LoadNonNull(cx, ptr) } else { - Load(cx, ptr.get()) + Load(cx, ptr) }; unsafe { @@ -918,11 +897,16 @@ pub fn from_arg_ty(bcx: Block, val: ValueRef, ty: Ty) -> ValueRef { pub fn to_arg_ty(bcx: Block, val: ValueRef, ty: Ty) -> ValueRef { if ty.is_bool() { + // Look for cases where we're truncating a zext from a bool already and grab the original + // value. This can happen with an elided load from a boolean location. While this can be + // easily optimized out, the indirection can interfere with some intrinsics. let val = Value(val); if let Some(zext) = val.as_zext_inst() { + // `val` == zext %foo if let Some(val) = zext.get_operand(0) { let valty = val_ty(val.get()); if valty == Type::i1(bcx.ccx()) { + // %foo : i1 return val.get(); } } diff --git a/src/librustc_trans/trans/builder.rs b/src/librustc_trans/trans/builder.rs index 28c188604c627..a47ee5a0a7c07 100644 --- a/src/librustc_trans/trans/builder.rs +++ b/src/librustc_trans/trans/builder.rs @@ -507,6 +507,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } pub fn volatile_store(&self, val: ValueRef, ptr: ValueRef) -> ValueRef { + debug!("Volatile Store {} -> {}", + self.ccx.tn().val_to_string(val), + self.ccx.tn().val_to_string(ptr)); assert!(!self.llbuilder.is_null()); self.count_insn("store.volatile"); unsafe { diff --git a/src/librustc_trans/trans/expr.rs b/src/librustc_trans/trans/expr.rs index 80aa065e3a359..d129027d5e5f8 100644 --- a/src/librustc_trans/trans/expr.rs +++ b/src/librustc_trans/trans/expr.rs @@ -2094,7 +2094,7 @@ fn trans_imm_cast<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, if type_is_fat_ptr(bcx.tcx(), t_in) { if type_is_fat_ptr(bcx.tcx(), t_out) { // If the datum is by-ref, just do a pointer cast, otherwise - // construct the casted fat pointer manually + // construct the casted fat pointer manually. if datum.kind.is_by_ref() { return DatumBlock::new(bcx, Datum::new( PointerCast(bcx, datum.val, ll_t_out.ptr_to()), diff --git a/src/librustc_trans/trans/value.rs b/src/librustc_trans/trans/value.rs index 0c78417109e95..23acb57ad821f 100644 --- a/src/librustc_trans/trans/value.rs +++ b/src/librustc_trans/trans/value.rs @@ -80,6 +80,19 @@ impl Value { } } + pub fn get_stored_value_opt(self, bcx: Block) -> Option { + let bb = Some(BasicBlock(bcx.llbb)); + if let Some(val) = self.get_dominating_store(bcx) { + let valbb = val.get_parent(); + + if bb == valbb { + return val.get_operand(0); + } + } + + None + } + /// Returns the first use of this value, if any pub fn get_first_use(self) -> Option { unsafe {