diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 261d9dd448d47..8cb0ff24e09db 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -65,10 +65,18 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { let mut checker = CostChecker { tcx, callee_body: mir, calls: 0, statements: 0, landing_pads: 0, resumes: 0 }; checker.visit_body(mir); - checker.calls == 0 + let is_leaf = checker.calls == 0 && checker.resumes == 0 && checker.landing_pads == 0 - && checker.statements <= threshold + && checker.statements <= threshold; + + let is_wrapper = checker.calls == 1 + && checker.resumes == 0 + && checker.landing_pads == 0 + && checker.statements == 0 + && mir.basic_blocks.len() == 2; + + is_leaf || is_wrapper } struct CostChecker<'b, 'tcx> { diff --git a/tests/codegen-units/item-collection/implicit-panic-call.rs b/tests/codegen-units/item-collection/implicit-panic-call.rs index abec7ad50aed6..a361c67f04a91 100644 --- a/tests/codegen-units/item-collection/implicit-panic-call.rs +++ b/tests/codegen-units/item-collection/implicit-panic-call.rs @@ -48,6 +48,7 @@ impl Div for i32 { } #[allow(unconditional_panic)] +#[inline(never)] pub fn foo() { // This implicitly generates a panic call. let _ = 1 / 0; diff --git a/tests/codegen/call-metadata.rs b/tests/codegen/call-metadata.rs index 07cc0c963717e..4f8ce41946f5f 100644 --- a/tests/codegen/call-metadata.rs +++ b/tests/codegen/call-metadata.rs @@ -5,6 +5,7 @@ #![crate_type = "lib"] +#[inline(never)] pub fn test() { // CHECK: call noundef i8 @some_true(){{( #[0-9]+)?}}, !range [[R0:![0-9]+]] // CHECK: [[R0]] = !{i8 0, i8 3} diff --git a/tests/codegen/cffi/ffi-const.rs b/tests/codegen/cffi/ffi-const.rs index 9372050348088..a0f8b78d1f3e6 100644 --- a/tests/codegen/cffi/ffi-const.rs +++ b/tests/codegen/cffi/ffi-const.rs @@ -2,6 +2,7 @@ #![crate_type = "lib"] #![feature(ffi_const)] +#[inline(never)] pub fn bar() { unsafe { foo() } } extern "C" { diff --git a/tests/codegen/cffi/ffi-pure.rs b/tests/codegen/cffi/ffi-pure.rs index 2ed735813582f..e8d42ca5cb25b 100644 --- a/tests/codegen/cffi/ffi-pure.rs +++ b/tests/codegen/cffi/ffi-pure.rs @@ -2,6 +2,7 @@ #![crate_type = "lib"] #![feature(ffi_pure)] +#[inline(never)] pub fn bar() { unsafe { foo() } } extern "C" { diff --git a/tests/codegen/cffi/ffi-returns-twice.rs b/tests/codegen/cffi/ffi-returns-twice.rs index 0fbe03f0bb6f1..a0f4e39d600e5 100644 --- a/tests/codegen/cffi/ffi-returns-twice.rs +++ b/tests/codegen/cffi/ffi-returns-twice.rs @@ -2,6 +2,7 @@ #![crate_type = "lib"] #![feature(ffi_returns_twice)] +#[inline(never)] pub fn bar() { unsafe { foo() } } extern "C" { diff --git a/tests/codegen/cross-crate-inlining/auxiliary/leaf.rs b/tests/codegen/cross-crate-inlining/auxiliary/leaf.rs index 963f087f22d4d..5cb10b3b34258 100644 --- a/tests/codegen/cross-crate-inlining/auxiliary/leaf.rs +++ b/tests/codegen/cross-crate-inlining/auxiliary/leaf.rs @@ -14,6 +14,11 @@ pub fn stem_fn() -> String { inner() } +// This function contains multiple calls. +pub fn multi_stem_fn() -> String { + inner() + &inner() +} + #[inline(never)] fn inner() -> String { String::from("test") diff --git a/tests/codegen/cross-crate-inlining/leaf-inlining.rs b/tests/codegen/cross-crate-inlining/leaf-inlining.rs index 73b1a520b06cf..d95af9a715623 100644 --- a/tests/codegen/cross-crate-inlining/leaf-inlining.rs +++ b/tests/codegen/cross-crate-inlining/leaf-inlining.rs @@ -12,9 +12,16 @@ pub fn leaf_outer() -> String { leaf::leaf_fn() } -// Check that we do not inline a non-leaf cross-crate call +// Check that we inline a cross-crate call where the callee contains a single call #[no_mangle] pub fn stem_outer() -> String { - // CHECK: call {{.*}}stem_fn + // CHECK-NOT: call {{.*}}stem_fn leaf::stem_fn() } + +// Check that we do not inline a cross-crate call where the callee contains multiple calls +#[no_mangle] +pub fn multi_stem_outer() -> String { + // CHECK: call {{.*}}multi_stem_fn + leaf::multi_stem_fn() +} diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-generalized.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-generalized.rs index d200ed9798a62..ce0183ddd2f43 100644 --- a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-generalized.rs +++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-generalized.rs @@ -5,6 +5,7 @@ #![crate_type="lib"] +#[inline(never)] pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { // CHECK-LABEL: define{{.*}}foo // CHECK-SAME: {{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} @@ -12,6 +13,7 @@ pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { f(arg) } +#[inline(never)] pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { // CHECK-LABEL: define{{.*}}bar // CHECK-SAME: {{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} @@ -19,6 +21,7 @@ pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { f(arg1, arg2) } +#[inline(never)] pub fn baz(f: fn(i32, i32, i32) -> i32, arg1: i32, arg2: i32, arg3: i32) -> i32 { // CHECK-LABEL: define{{.*}}baz // CHECK-SAME: {{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs index cdefec17a1c78..95f8187740229 100644 --- a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs +++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs @@ -5,6 +5,7 @@ #![crate_type="lib"] +#[inline(never)] pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { // CHECK-LABEL: define{{.*}}foo // CHECK-SAME: {{.*}}![[TYPE1:[0-9]+]] @@ -12,6 +13,7 @@ pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { f(arg) } +#[inline(never)] pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { // CHECK-LABEL: define{{.*}}bar // CHECK-SAME: {{.*}}![[TYPE2:[0-9]+]] @@ -19,6 +21,7 @@ pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { f(arg1, arg2) } +#[inline(never)] pub fn baz(f: fn(i32, i32, i32) -> i32, arg1: i32, arg2: i32, arg3: i32) -> i32 { // CHECK-LABEL: define{{.*}}baz // CHECK-SAME: {{.*}}![[TYPE3:[0-9]+]] diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized.rs index f360b33ddcf30..d56de7a9f546c 100644 --- a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized.rs +++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized.rs @@ -5,6 +5,7 @@ #![crate_type="lib"] +#[inline(never)] pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { // CHECK-LABEL: define{{.*}}foo // CHECK-SAME: {{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} @@ -12,6 +13,7 @@ pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { f(arg) } +#[inline(never)] pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { // CHECK-LABEL: define{{.*}}bar // CHECK-SAME: {{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} @@ -19,6 +21,7 @@ pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { f(arg1, arg2) } +#[inline(never)] pub fn baz(f: fn(i32, i32, i32) -> i32, arg1: i32, arg2: i32, arg3: i32) -> i32 { // CHECK-LABEL: define{{.*}}baz // CHECK-SAME: {{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi.rs index 3cb817b212d18..97af795613849 100644 --- a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi.rs +++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi.rs @@ -5,6 +5,7 @@ #![crate_type="lib"] +#[inline(never)] pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { // CHECK-LABEL: define{{.*}}foo // CHECK-SAME: {{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}} @@ -12,6 +13,7 @@ pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { f(arg) } +#[inline(never)] pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { // CHECK-LABEL: define{{.*}}bar // CHECK-SAME: {{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}} @@ -19,6 +21,7 @@ pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { f(arg1, arg2) } +#[inline(never)] pub fn baz(f: fn(i32, i32, i32) -> i32, arg1: i32, arg2: i32, arg3: i32) -> i32 { // CHECK-LABEL: define{{.*}}baz // CHECK-SAME: {{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}} diff --git a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs b/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs index 29e4df3511fbb..23e4c26edcd00 100644 --- a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs +++ b/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs @@ -18,6 +18,7 @@ trait Copy { } impl Copy for i32 {} +#[inline(never)] pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { // CHECK-LABEL: define{{.*}}foo // CHECK-SAME: {{.*}}!{{|kcfi_type}} ![[TYPE1:[0-9]+]] @@ -25,6 +26,7 @@ pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { f(arg) } +#[inline(never)] pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { // CHECK-LABEL: define{{.*}}bar // CHECK-SAME: {{.*}}!{{|kcfi_type}} ![[TYPE2:[0-9]+]] @@ -32,6 +34,7 @@ pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { f(arg1, arg2) } +#[inline(never)] pub fn baz(f: fn(i32, i32, i32) -> i32, arg1: i32, arg2: i32, arg3: i32) -> i32 { // CHECK-LABEL: define{{.*}}baz // CHECK-SAME: {{.*}}!{{|kcfi_type}} ![[TYPE3:[0-9]+]] diff --git a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs b/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs index 84d678a33ba90..8c85198bb20d7 100644 --- a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs +++ b/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs @@ -18,6 +18,7 @@ trait Copy { } impl Copy for i32 {} +#[inline(never)] pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { // CHECK-LABEL: define{{.*}}foo // CHECK-SAME: {{.*}}!{{|kcfi_type}} ![[TYPE1:[0-9]+]] @@ -25,6 +26,7 @@ pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { f(arg) } +#[inline(never)] pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { // CHECK-LABEL: define{{.*}}bar // CHECK-SAME: {{.*}}!{{|kcfi_type}} ![[TYPE2:[0-9]+]] @@ -32,6 +34,7 @@ pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { f(arg1, arg2) } +#[inline(never)] pub fn baz(f: fn(i32, i32, i32) -> i32, arg1: i32, arg2: i32, arg3: i32) -> i32 { // CHECK-LABEL: define{{.*}}baz // CHECK-SAME: {{.*}}!{{|kcfi_type}} ![[TYPE3:[0-9]+]] diff --git a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs b/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs index 761c37a9e06a0..682147afa3dbf 100644 --- a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs +++ b/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs @@ -18,6 +18,7 @@ trait Copy { } impl Copy for i32 {} +#[inline(never)] pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { // CHECK-LABEL: define{{.*}}foo // CHECK-SAME: {{.*}}!{{|kcfi_type}} ![[TYPE1:[0-9]+]] @@ -25,6 +26,7 @@ pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 { f(arg) } +#[inline(never)] pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { // CHECK-LABEL: define{{.*}}bar // CHECK-SAME: {{.*}}!{{|kcfi_type}} ![[TYPE2:[0-9]+]] @@ -32,6 +34,7 @@ pub fn bar(f: fn(i32, i32) -> i32, arg1: i32, arg2: i32) -> i32 { f(arg1, arg2) } +#[inline(never)] pub fn baz(f: fn(i32, i32, i32) -> i32, arg1: i32, arg2: i32, arg3: i32) -> i32 { // CHECK-LABEL: define{{.*}}baz // CHECK-SAME: {{.*}}!{{|kcfi_type}} ![[TYPE3:[0-9]+]] diff --git a/tests/run-make/optimization-remarks-dir/foo.rs b/tests/run-make/optimization-remarks-dir/foo.rs index 6ac3af0dcad5e..0851b59b5a2e6 100644 --- a/tests/run-make/optimization-remarks-dir/foo.rs +++ b/tests/run-make/optimization-remarks-dir/foo.rs @@ -1,6 +1,7 @@ #[inline(never)] pub fn bar() {} +#[inline(never)] pub fn foo() { bar(); }