Skip to content

Commit ab17641

Browse files
Mark-SimulacrumMark-Simulacrum
committed
Omit non-needs_drop drop_in_place in vtables
This replaces the drop_in_place reference with null in vtables. On librustc_driver.so, this drops about ~17k dynamic relocations from the output, since many vtables can now be placed in read-only memory, rather than having a relocated pointer included. This makes a tradeoff by adding a null check at vtable call sites. That's hard to avoid without changing the vtable format (e.g., to use a pc-relative relocation instead of an absolute address, and avoid the dynamic relocation that way). But it seems likely that the check is cheap at runtime.
1 parent e82c861 commit ab17641

File tree

7 files changed

+206
-116
lines changed

7 files changed

+206
-116
lines changed

‎compiler/rustc_codegen_cranelift/src/abi/mod.rs

+16
Original file line numberDiff line numberDiff line change
@@ -593,6 +593,7 @@ pub(crate) fn codegen_drop<'tcx>(
593593
fx: &mut FunctionCx<'_, '_, 'tcx>,
594594
source_info: mir::SourceInfo,
595595
drop_place: CPlace<'tcx>,
596+
target: BasicBlock,
596597
) {
597598
let ty = drop_place.layout().ty;
598599
let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx);
@@ -618,6 +619,12 @@ pub(crate) fn codegen_drop<'tcx>(
618619
let ptr = ptr.get_addr(fx);
619620
let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable);
620621

622+
let is_null = fx.bcx.ins().icmp_imm(IntCC::Equal, drop_fn, 0);
623+
let target_block = fx.get_block(target);
624+
let continued = fx.bcx.create_block();
625+
fx.bcx.ins().brif(is_null, target_block, &[], continued, &[]);
626+
fx.bcx.switch_to_block(continued);
627+
621628
// FIXME(eddyb) perhaps move some of this logic into
622629
// `Instance::resolve_drop_in_place`?
623630
let virtual_drop = Instance {
@@ -657,6 +664,12 @@ pub(crate) fn codegen_drop<'tcx>(
657664
let (data, vtable) = drop_place.to_cvalue(fx).dyn_star_force_data_on_stack(fx);
658665
let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable);
659666

667+
let is_null = fx.bcx.ins().icmp_imm(IntCC::Equal, drop_fn, 0);
668+
let target_block = fx.get_block(target);
669+
let continued = fx.bcx.create_block();
670+
fx.bcx.ins().brif(is_null, target_block, &[], continued, &[]);
671+
fx.bcx.switch_to_block(continued);
672+
660673
let virtual_drop = Instance {
661674
def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0),
662675
args: drop_instance.args,
@@ -695,4 +708,7 @@ pub(crate) fn codegen_drop<'tcx>(
695708
}
696709
}
697710
}
711+
712+
let target_block = fx.get_block(target);
713+
fx.bcx.ins().jump(target_block, &[]);
698714
}

‎compiler/rustc_codegen_cranelift/src/base.rs

+1-4
Original file line numberDiff line numberDiff line change
@@ -515,10 +515,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
515515
}
516516
TerminatorKind::Drop { place, target, unwind: _, replace: _ } => {
517517
let drop_place = codegen_place(fx, *place);
518-
crate::abi::codegen_drop(fx, source_info, drop_place);
519-
520-
let target_block = fx.get_block(*target);
521-
fx.bcx.ins().jump(target_block, &[]);
518+
crate::abi::codegen_drop(fx, source_info, drop_place, *target);
522519
}
523520
};
524521
}

‎compiler/rustc_codegen_ssa/src/meth.rs

+25-2
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,13 @@ impl<'a, 'tcx> VirtualIndex {
1414
VirtualIndex(index as u64)
1515
}
1616

17-
pub fn get_fn<Bx: BuilderMethods<'a, 'tcx>>(
17+
fn get_fn_inner<Bx: BuilderMethods<'a, 'tcx>>(
1818
self,
1919
bx: &mut Bx,
2020
llvtable: Bx::Value,
2121
ty: Ty<'tcx>,
2222
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
23+
nonnull: bool,
2324
) -> Bx::Value {
2425
// Load the function pointer from the object.
2526
debug!("get_fn({llvtable:?}, {ty:?}, {self:?})");
@@ -40,13 +41,35 @@ impl<'a, 'tcx> VirtualIndex {
4041
} else {
4142
let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset));
4243
let ptr = bx.load(llty, gep, ptr_align);
43-
bx.nonnull_metadata(ptr);
4444
// VTable loads are invariant.
4545
bx.set_invariant_load(ptr);
46+
if nonnull {
47+
bx.nonnull_metadata(ptr);
48+
}
4649
ptr
4750
}
4851
}
4952

53+
pub fn get_optional_fn<Bx: BuilderMethods<'a, 'tcx>>(
54+
self,
55+
bx: &mut Bx,
56+
llvtable: Bx::Value,
57+
ty: Ty<'tcx>,
58+
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
59+
) -> Bx::Value {
60+
self.get_fn_inner(bx, llvtable, ty, fn_abi, false)
61+
}
62+
63+
pub fn get_fn<Bx: BuilderMethods<'a, 'tcx>>(
64+
self,
65+
bx: &mut Bx,
66+
llvtable: Bx::Value,
67+
ty: Ty<'tcx>,
68+
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
69+
) -> Bx::Value {
70+
self.get_fn_inner(bx, llvtable, ty, fn_abi, true)
71+
}
72+
5073
pub fn get_usize<Bx: BuilderMethods<'a, 'tcx>>(
5174
self,
5275
bx: &mut Bx,

‎compiler/rustc_codegen_ssa/src/mir/block.rs

+115-88
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
499499
&mut self,
500500
helper: TerminatorCodegenHelper<'tcx>,
501501
bx: &mut Bx,
502+
source_info: &mir::SourceInfo,
502503
location: mir::Place<'tcx>,
503504
target: mir::BasicBlock,
504505
unwind: mir::UnwindAction,
@@ -522,90 +523,109 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
522523
args1 = [place.val.llval];
523524
&args1[..]
524525
};
525-
let (drop_fn, fn_abi, drop_instance) =
526-
match ty.kind() {
527-
// FIXME(eddyb) perhaps move some of this logic into
528-
// `Instance::resolve_drop_in_place`?
529-
ty::Dynamic(_, _, ty::Dyn) => {
530-
// IN THIS ARM, WE HAVE:
531-
// ty = *mut (dyn Trait)
532-
// which is: exists<T> ( *mut T, Vtable<T: Trait> )
533-
// args[0] args[1]
534-
//
535-
// args = ( Data, Vtable )
536-
// |
537-
// v
538-
// /-------\
539-
// | ... |
540-
// \-------/
541-
//
542-
let virtual_drop = Instance {
543-
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function
544-
args: drop_fn.args,
545-
};
546-
debug!("ty = {:?}", ty);
547-
debug!("drop_fn = {:?}", drop_fn);
548-
debug!("args = {:?}", args);
549-
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
550-
let vtable = args[1];
551-
// Truncate vtable off of args list
552-
args = &args[..1];
553-
(
554-
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
555-
.get_fn(bx, vtable, ty, fn_abi),
556-
fn_abi,
557-
virtual_drop,
558-
)
559-
}
560-
ty::Dynamic(_, _, ty::DynStar) => {
561-
// IN THIS ARM, WE HAVE:
562-
// ty = *mut (dyn* Trait)
563-
// which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
564-
//
565-
// args = [ * ]
566-
// |
567-
// v
568-
// ( Data, Vtable )
569-
// |
570-
// v
571-
// /-------\
572-
// | ... |
573-
// \-------/
574-
//
575-
//
576-
// WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
577-
//
578-
// data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer)
579-
// vtable = (*args[0]).1 // loads the vtable out
580-
// (data, vtable) // an equivalent Rust `*mut dyn Trait`
581-
//
582-
// SO THEN WE CAN USE THE ABOVE CODE.
583-
let virtual_drop = Instance {
584-
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function
585-
args: drop_fn.args,
586-
};
587-
debug!("ty = {:?}", ty);
588-
debug!("drop_fn = {:?}", drop_fn);
589-
debug!("args = {:?}", args);
590-
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
591-
let meta_ptr = place.project_field(bx, 1);
592-
let meta = bx.load_operand(meta_ptr);
593-
// Truncate vtable off of args list
594-
args = &args[..1];
595-
debug!("args' = {:?}", args);
596-
(
597-
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
598-
.get_fn(bx, meta.immediate(), ty, fn_abi),
599-
fn_abi,
600-
virtual_drop,
601-
)
602-
}
603-
_ => (
604-
bx.get_fn_addr(drop_fn),
605-
bx.fn_abi_of_instance(drop_fn, ty::List::empty()),
606-
drop_fn,
607-
),
608-
};
526+
let (maybe_null, drop_fn, fn_abi, drop_instance) = match ty.kind() {
527+
// FIXME(eddyb) perhaps move some of this logic into
528+
// `Instance::resolve_drop_in_place`?
529+
ty::Dynamic(_, _, ty::Dyn) => {
530+
// IN THIS ARM, WE HAVE:
531+
// ty = *mut (dyn Trait)
532+
// which is: exists<T> ( *mut T, Vtable<T: Trait> )
533+
// args[0] args[1]
534+
//
535+
// args = ( Data, Vtable )
536+
// |
537+
// v
538+
// /-------\
539+
// | ... |
540+
// \-------/
541+
//
542+
let virtual_drop = Instance {
543+
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function
544+
args: drop_fn.args,
545+
};
546+
debug!("ty = {:?}", ty);
547+
debug!("drop_fn = {:?}", drop_fn);
548+
debug!("args = {:?}", args);
549+
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
550+
let vtable = args[1];
551+
// Truncate vtable off of args list
552+
args = &args[..1];
553+
(
554+
true,
555+
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
556+
.get_optional_fn(bx, vtable, ty, fn_abi),
557+
fn_abi,
558+
virtual_drop,
559+
)
560+
}
561+
ty::Dynamic(_, _, ty::DynStar) => {
562+
// IN THIS ARM, WE HAVE:
563+
// ty = *mut (dyn* Trait)
564+
// which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
565+
//
566+
// args = [ * ]
567+
// |
568+
// v
569+
// ( Data, Vtable )
570+
// |
571+
// v
572+
// /-------\
573+
// | ... |
574+
// \-------/
575+
//
576+
//
577+
// WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
578+
//
579+
// data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer)
580+
// vtable = (*args[0]).1 // loads the vtable out
581+
// (data, vtable) // an equivalent Rust `*mut dyn Trait`
582+
//
583+
// SO THEN WE CAN USE THE ABOVE CODE.
584+
let virtual_drop = Instance {
585+
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function
586+
args: drop_fn.args,
587+
};
588+
debug!("ty = {:?}", ty);
589+
debug!("drop_fn = {:?}", drop_fn);
590+
debug!("args = {:?}", args);
591+
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
592+
let meta_ptr = place.project_field(bx, 1);
593+
let meta = bx.load_operand(meta_ptr);
594+
// Truncate vtable off of args list
595+
args = &args[..1];
596+
debug!("args' = {:?}", args);
597+
(
598+
true,
599+
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
600+
.get_optional_fn(bx, meta.immediate(), ty, fn_abi),
601+
fn_abi,
602+
virtual_drop,
603+
)
604+
}
605+
_ => (
606+
false,
607+
bx.get_fn_addr(drop_fn),
608+
bx.fn_abi_of_instance(drop_fn, ty::List::empty()),
609+
drop_fn,
610+
),
611+
};
612+
613+
// We generate a null check for the drop_fn. This saves a bunch of relocations being
614+
// generated for no-op drops.
615+
if maybe_null {
616+
let is_not_null = bx.append_sibling_block("is_not_null");
617+
let llty = bx.fn_ptr_backend_type(fn_abi);
618+
let null = bx.const_null(llty);
619+
let non_null = bx.icmp(
620+
base::bin_op_to_icmp_predicate(mir::BinOp::Ne.to_hir_binop(), false),
621+
drop_fn,
622+
null,
623+
);
624+
bx.cond_br(non_null, is_not_null, self.llbb(target));
625+
bx.switch_to_block(is_not_null);
626+
self.set_debug_loc(bx, *source_info);
627+
}
628+
609629
helper.do_call(
610630
self,
611631
bx,
@@ -616,7 +636,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
616636
unwind,
617637
&[],
618638
Some(drop_instance),
619-
mergeable_succ,
639+
!maybe_null && mergeable_succ,
620640
)
621641
}
622642

@@ -1345,9 +1365,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
13451365
MergingSucc::False
13461366
}
13471367

1348-
mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => {
1349-
self.codegen_drop_terminator(helper, bx, place, target, unwind, mergeable_succ())
1350-
}
1368+
mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => self
1369+
.codegen_drop_terminator(
1370+
helper,
1371+
bx,
1372+
&terminator.source_info,
1373+
place,
1374+
target,
1375+
unwind,
1376+
mergeable_succ(),
1377+
),
13511378

13521379
mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, unwind } => self
13531380
.codegen_assert_terminator(

‎compiler/rustc_middle/src/ty/vtable.rs

+8-4
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,14 @@ pub(super) fn vtable_allocation_provider<'tcx>(
8484
let idx: u64 = u64::try_from(idx).unwrap();
8585
let scalar = match entry {
8686
VtblEntry::MetadataDropInPlace => {
87-
let instance = ty::Instance::resolve_drop_in_place(tcx, ty);
88-
let fn_alloc_id = tcx.reserve_and_set_fn_alloc(instance);
89-
let fn_ptr = Pointer::from(fn_alloc_id);
90-
Scalar::from_pointer(fn_ptr, &tcx)
87+
if ty.needs_drop(tcx, ty::ParamEnv::reveal_all()) {
88+
let instance = ty::Instance::resolve_drop_in_place(tcx, ty);
89+
let fn_alloc_id = tcx.reserve_and_set_fn_alloc(instance);
90+
let fn_ptr = Pointer::from(fn_alloc_id);
91+
Scalar::from_pointer(fn_ptr, &tcx)
92+
} else {
93+
Scalar::from_maybe_pointer(Pointer::null(), &tcx)
94+
}
9195
}
9296
VtblEntry::MetadataSize => Scalar::from_uint(size, ptr_size),
9397
VtblEntry::MetadataAlign => Scalar::from_uint(align, ptr_size),

0 commit comments

Comments
 (0)