MultipleReturnTerminators + SingleReturnTerminator

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -361,7 +361,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         }
     }
 
-    fn codegen_return_terminator(&mut self, bx: &mut Bx) {
+    pub fn codegen_return_terminator(&mut self, bx: &mut Bx) {
         // Call `va_end` if this is the definition of a C-variadic function.
         if self.fn_abi.c_variadic {
             // The `VaList` "spoofed" argument is just after all the real arguments.
@@ -1296,7 +1296,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
 
             mir::TerminatorKind::Return => {
-                self.codegen_return_terminator(bx);
+                if let Some(return_block) = self.return_block {
+                    bx.br(return_block);
+                } else {
+                    self.codegen_return_terminator(bx);
+                }
                 MergingSucc::False
             }
 

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -72,6 +72,9 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
     /// Cached unreachable block
     unreachable_block: Option<Bx::BasicBlock>,
 
+    /// Cached return block
+    return_block: Option<Bx::BasicBlock>,
+
     /// Cached double unwind guarding block
     double_unwind_guard: Option<Bx::BasicBlock>,
 
@@ -184,6 +187,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         personality_slot: None,
         cached_llbbs,
         unreachable_block: None,
+        return_block: None,
         double_unwind_guard: None,
         cleanup_kinds,
         landing_pads: IndexVec::from_elem(None, &mir.basic_blocks),
@@ -256,10 +260,31 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     // Apply debuginfo to the newly allocated locals.
     fx.debug_introduce_locals(&mut start_bx);
 
+    // If we have multiple return terminators, create a special return block. If this block is
+    // present, return terminator codegen will be a jump to it instead of a return.
+    if mir
+        .basic_blocks
+        .iter()
+        .filter(|block| matches!(block.terminator().kind, mir::TerminatorKind::Return))
+        .count()
+        > 1
+    {
+        let llbb = Bx::append_block(fx.cx, fx.llfn, "return");
+        fx.return_block = Some(llbb);
+    }
+
     // Codegen the body of each block using reverse postorder
     for (bb, _) in traversal::reverse_postorder(&mir) {
         fx.codegen_block(bb);
     }
+
+    // Actually codegen the return block. We need to delay until here because our return place
+    // might not be generated yet, and will only be generated by handling codegen for the
+    // assignment to it.
+    if let Some(llbb) = fx.return_block {
+        let mut bx = Bx::build(fx.cx, llbb);
+        fx.codegen_return_terminator(&mut bx);
+    }
 }
 
 /// Produces, for each argument, a `Value` pointing at the

compiler/rustc_mir_transform/src/deduplicate_blocks.rs

@@ -16,12 +16,22 @@ pub struct DeduplicateBlocks;
 
 impl<'tcx> MirPass<'tcx> for DeduplicateBlocks {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
-        sess.mir_opt_level() >= 4
+        sess.mir_opt_level() >= 1
     }
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         debug!("Running DeduplicateBlocks on `{:?}`", body.source);
-        let duplicates = find_duplicates(body);
+
+        let limit = if tcx.sess.mir_opt_level() < 3 {
+            0
+        } else {
+            // Basic blocks can get really big, so to avoid checking for duplicates in basic blocks
+            // that are unlikely to have duplicates, we stop early. The early bail number has been
+            // found experimentally by eprintln while compiling the crates in the rustc-perf suite.
+            10
+        };
+
+        let duplicates = find_duplicates(body, limit);
         let has_opts_to_apply = !duplicates.is_empty();
 
         if has_opts_to_apply {
@@ -54,7 +64,7 @@ impl<'tcx> MutVisitor<'tcx> for OptApplier<'tcx> {
     }
 }
 
-fn find_duplicates(body: &Body<'_>) -> FxHashMap<BasicBlock, BasicBlock> {
+fn find_duplicates(body: &Body<'_>, limit: usize) -> FxHashMap<BasicBlock, BasicBlock> {
     let mut duplicates = FxHashMap::default();
 
     let bbs_to_go_through =
@@ -72,10 +82,11 @@ fn find_duplicates(body: &Body<'_>) -> FxHashMap<BasicBlock, BasicBlock> {
     // with replacement bb3.
     // When the duplicates are removed, we will end up with only bb3.
     for (bb, bbd) in body.basic_blocks.iter_enumerated().rev().filter(|(_, bbd)| !bbd.is_cleanup) {
-        // Basic blocks can get really big, so to avoid checking for duplicates in basic blocks
-        // that are unlikely to have duplicates, we stop early. The early bail number has been
-        // found experimentally by eprintln while compiling the crates in the rustc-perf suite.
-        if bbd.statements.len() > 10 {
+        if bbd.statements.len() > limit {
+            continue;
+        }
+
+        if bbd.terminator().kind != TerminatorKind::Return {
             continue;
         }
 

compiler/rustc_mir_transform/src/lib.rs

@@ -577,7 +577,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &o1(simplify::SimplifyCfg::new("final")),
             &nrvo::RenameReturnPlace,
             &simplify::SimplifyLocals::new("final"),
-            &multiple_return_terminators::MultipleReturnTerminators,
             &deduplicate_blocks::DeduplicateBlocks,
             // Some cleanup necessary at least for LLVM and potentially other codegen backends.
             &add_call_guards::CriticalCallEdges,

compiler/rustc_mir_transform/src/multiple_return_terminators.rs

@@ -10,7 +10,7 @@ pub struct MultipleReturnTerminators;
 
 impl<'tcx> MirPass<'tcx> for MultipleReturnTerminators {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
-        sess.mir_opt_level() >= 4
+        sess.mir_opt_level() >= 1
     }
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -26,14 +26,15 @@ impl<'tcx> MirPass<'tcx> for MultipleReturnTerminators {
             }
         }
 
-        for bb in bbs {
+        for bb in 0..bbs.len() {
+            let bb = bb.into();
             if !tcx.consider_optimizing(|| format!("MultipleReturnTerminators {:?} ", def_id)) {
                 break;
             }
 
-            if let TerminatorKind::Goto { target } = bb.terminator().kind {
+            if let TerminatorKind::Goto { target } = bbs[bb].terminator().kind {
                 if bbs_simple_returns.contains(target) {
-                    bb.terminator_mut().kind = TerminatorKind::Return;
+                    *bbs[bb].terminator_mut() = bbs[target].terminator().clone();
                 }
             }
         }

tests/mir-opt/76803_regression.encode.SimplifyBranchSame.diff

@@ -13,16 +13,12 @@
 
       bb1: {
           _0 = move _1;                    // scope 0 at $DIR/76803_regression.rs:+3:14: +3:15
-          goto -> bb3;                     // scope 0 at $DIR/76803_regression.rs:+3:14: +3:15
+          return;                          // scope 0 at $DIR/76803_regression.rs:+5:2: +5:2
       }
 
       bb2: {
           Deinit(_0);                      // scope 0 at $DIR/76803_regression.rs:+2:20: +2:27
           discriminant(_0) = 1;            // scope 0 at $DIR/76803_regression.rs:+2:20: +2:27
-          goto -> bb3;                     // scope 0 at $DIR/76803_regression.rs:+2:20: +2:27
-      }
-
-      bb3: {
           return;                          // scope 0 at $DIR/76803_regression.rs:+5:2: +5:2
       }
   }

tests/mir-opt/deduplicate_blocks.is_line_doc_comment_2.DeduplicateBlocks.diff

@@ -44,8 +44,7 @@
       }
 
       bb5: {
--         switchInt((*_2)[3 of 4]) -> [47: bb11, otherwise: bb6]; // scope 0 at $DIR/deduplicate_blocks.rs:+1:5: +1:23
-+         switchInt((*_2)[3 of 4]) -> [47: bb10, otherwise: bb6]; // scope 0 at $DIR/deduplicate_blocks.rs:+1:5: +1:23
+          switchInt((*_2)[3 of 4]) -> [47: bb11, otherwise: bb6]; // scope 0 at $DIR/deduplicate_blocks.rs:+1:5: +1:23
       }
 
       bb6: {
@@ -64,35 +63,30 @@
       }
 
       bb9: {
--         switchInt((*_2)[2 of 3]) -> [47: bb12, 33: bb13, otherwise: bb10]; // scope 0 at $DIR/deduplicate_blocks.rs:+1:5: +1:23
-+         switchInt((*_2)[2 of 3]) -> [47: bb11, 33: bb11, otherwise: bb10]; // scope 0 at $DIR/deduplicate_blocks.rs:+1:5: +1:23
+          switchInt((*_2)[2 of 3]) -> [47: bb12, 33: bb13, otherwise: bb10]; // scope 0 at $DIR/deduplicate_blocks.rs:+1:5: +1:23
       }
 
       bb10: {
--         _0 = const false;                // scope 0 at $DIR/deduplicate_blocks.rs:+5:14: +5:19
--         goto -> bb14;                    // scope 0 at $DIR/deduplicate_blocks.rs:+5:14: +5:19
--     }
-- 
--     bb11: {
+          _0 = const false;                // scope 0 at $DIR/deduplicate_blocks.rs:+5:14: +5:19
+          goto -> bb14;                    // scope 0 at $DIR/deduplicate_blocks.rs:+5:14: +5:19
+      }
+  
+      bb11: {
           _0 = const false;                // scope 0 at $DIR/deduplicate_blocks.rs:+2:41: +2:46
--         goto -> bb14;                    // scope 0 at $DIR/deduplicate_blocks.rs:+2:41: +2:46
-+         goto -> bb12;                    // scope 0 at $DIR/deduplicate_blocks.rs:+2:41: +2:46
+          goto -> bb14;                    // scope 0 at $DIR/deduplicate_blocks.rs:+2:41: +2:46
+      }
+
+      bb12: {
+          _0 = const true;                 // scope 0 at $DIR/deduplicate_blocks.rs:+3:35: +3:39
+          goto -> bb14;                    // scope 0 at $DIR/deduplicate_blocks.rs:+3:35: +3:39
       }
 
--     bb12: {
--         _0 = const true;                 // scope 0 at $DIR/deduplicate_blocks.rs:+3:35: +3:39
--         goto -> bb14;                    // scope 0 at $DIR/deduplicate_blocks.rs:+3:35: +3:39
--     }
-- 
--     bb13: {
-+     bb11: {
+      bb13: {
           _0 = const true;                 // scope 0 at $DIR/deduplicate_blocks.rs:+4:35: +4:39
--         goto -> bb14;                    // scope 0 at $DIR/deduplicate_blocks.rs:+4:35: +4:39
-+         goto -> bb12;                    // scope 0 at $DIR/deduplicate_blocks.rs:+4:35: +4:39
+          goto -> bb14;                    // scope 0 at $DIR/deduplicate_blocks.rs:+4:35: +4:39
       }
 
--     bb14: {
-+     bb12: {
+      bb14: {
           StorageDead(_2);                 // scope 0 at $DIR/deduplicate_blocks.rs:+7:1: +7:2
           return;                          // scope 0 at $DIR/deduplicate_blocks.rs:+7:2: +7:2
       }

tests/mir-opt/multiple_return_terminators.test.MultipleReturnTerminators.diff

@@ -4,8 +4,16 @@
   fn test(_1: bool) -> () {
       debug x => _1;                       // in scope 0 at $DIR/multiple_return_terminators.rs:+0:9: +0:10
       let mut _0: ();                      // return place in scope 0 at $DIR/multiple_return_terminators.rs:+0:18: +0:18
+      let mut _2: bool;                    // in scope 0 at $DIR/multiple_return_terminators.rs:+1:8: +1:9
+      let mut _3: bool;                    // in scope 0 at $DIR/multiple_return_terminators.rs:+1:8: +1:9
 
       bb0: {
+          StorageLive(_2);                 // scope 0 at $DIR/multiple_return_terminators.rs:+1:8: +1:9
+          _2 = _1;                         // scope 0 at $DIR/multiple_return_terminators.rs:+1:8: +1:9
+          StorageLive(_3);                 // scope 0 at $DIR/multiple_return_terminators.rs:+1:8: +1:9
+          _3 = move _2;                    // scope 0 at $DIR/multiple_return_terminators.rs:+1:8: +1:9
+          StorageDead(_3);                 // scope 0 at $DIR/multiple_return_terminators.rs:+1:8: +1:9
+          StorageDead(_2);                 // scope 0 at $DIR/multiple_return_terminators.rs:+5:5: +5:6
           return;                          // scope 0 at $DIR/multiple_return_terminators.rs:+6:2: +6:2
       }
   }

tests/mir-opt/separate_const_switch.identity.SeparateConstSwitch.diff

@@ -125,7 +125,7 @@
           discriminant(_3) = 1;            // scope 7 at $SRC_DIR/core/src/result.rs:LL:COL
           StorageDead(_14);                // scope 7 at $SRC_DIR/core/src/result.rs:LL:COL
           StorageDead(_13);                // scope 5 at $SRC_DIR/core/src/result.rs:LL:COL
--         goto -> bb1;                     // scope 5 at $SRC_DIR/core/src/result.rs:LL:COL
+-         goto -> bb1;                     // scope 0 at $SRC_DIR/core/src/result.rs:LL:COL
 +         StorageDead(_4);                 // scope 0 at $DIR/separate_const_switch.rs:+1:9: +1:10
 +         _5 = discriminant(_3);           // scope 0 at $DIR/separate_const_switch.rs:+1:8: +1:10
 +         switchInt(move _5) -> [0: bb1, 1: bb3, otherwise: bb2]; // scope 0 at $DIR/separate_const_switch.rs:+1:8: +1:10
@@ -147,7 +147,7 @@
           discriminant(_3) = 0;            // scope 6 at $SRC_DIR/core/src/result.rs:LL:COL
           StorageDead(_12);                // scope 6 at $SRC_DIR/core/src/result.rs:LL:COL
           StorageDead(_11);                // scope 5 at $SRC_DIR/core/src/result.rs:LL:COL
--         goto -> bb1;                     // scope 5 at $SRC_DIR/core/src/result.rs:LL:COL
+-         goto -> bb1;                     // scope 0 at $SRC_DIR/core/src/result.rs:LL:COL
 +         StorageDead(_4);                 // scope 0 at $DIR/separate_const_switch.rs:+1:9: +1:10
 +         _5 = discriminant(_3);           // scope 0 at $DIR/separate_const_switch.rs:+1:8: +1:10
 +         switchInt(move _5) -> [0: bb1, 1: bb3, otherwise: bb2]; // scope 0 at $DIR/separate_const_switch.rs:+1:8: +1:10

tests/mir-opt/try_identity_e2e.new.PreCodegen.after.mir

@@ -26,7 +26,7 @@ fn new(_1: Result<T, E>) -> Result<T, E> {
     bb0: {
         StorageLive(_2);                 // scope 0 at $DIR/try_identity_e2e.rs:+2:15: +7:10
         _3 = discriminant(_1);           // scope 0 at $DIR/try_identity_e2e.rs:+3:19: +3:20
-        switchInt(move _3) -> [0: bb2, 1: bb1, otherwise: bb4]; // scope 0 at $DIR/try_identity_e2e.rs:+3:13: +3:20
+        switchInt(move _3) -> [0: bb3, 1: bb1, otherwise: bb2]; // scope 0 at $DIR/try_identity_e2e.rs:+3:13: +3:20
     }
 
     bb1: {
@@ -35,37 +35,41 @@ fn new(_1: Result<T, E>) -> Result<T, E> {
         ((_2 as Break).0: E) = move _5;  // scope 2 at $DIR/try_identity_e2e.rs:+5:27: +5:48
         discriminant(_2) = 1;            // scope 2 at $DIR/try_identity_e2e.rs:+5:27: +5:48
         _6 = discriminant(_2);           // scope 0 at $DIR/try_identity_e2e.rs:+2:15: +7:10
-        switchInt(move _6) -> [0: bb5, 1: bb3, otherwise: bb4]; // scope 0 at $DIR/try_identity_e2e.rs:+2:9: +7:10
+        switchInt(move _6) -> [0: bb6, 1: bb4, otherwise: bb5]; // scope 0 at $DIR/try_identity_e2e.rs:+2:9: +7:10
     }
 
     bb2: {
+        unreachable;                     // scope 0 at $DIR/try_identity_e2e.rs:+3:19: +3:20
+    }
+
+    bb3: {
         _4 = move ((_1 as Ok).0: T);     // scope 0 at $DIR/try_identity_e2e.rs:+4:20: +4:21
         Deinit(_2);                      // scope 1 at $DIR/try_identity_e2e.rs:+4:26: +4:50
         ((_2 as Continue).0: T) = move _4; // scope 1 at $DIR/try_identity_e2e.rs:+4:26: +4:50
         discriminant(_2) = 0;            // scope 1 at $DIR/try_identity_e2e.rs:+4:26: +4:50
         _6 = discriminant(_2);           // scope 0 at $DIR/try_identity_e2e.rs:+2:15: +7:10
-        switchInt(move _6) -> [0: bb5, 1: bb3, otherwise: bb4]; // scope 0 at $DIR/try_identity_e2e.rs:+2:9: +7:10
+        switchInt(move _6) -> [0: bb6, 1: bb4, otherwise: bb5]; // scope 0 at $DIR/try_identity_e2e.rs:+2:9: +7:10
     }
 
-    bb3: {
+    bb4: {
         _8 = move ((_2 as Break).0: E);  // scope 0 at $DIR/try_identity_e2e.rs:+9:32: +9:33
         Deinit(_0);                      // scope 4 at $DIR/try_identity_e2e.rs:+9:45: +9:51
         ((_0 as Err).0: E) = move _8;    // scope 4 at $DIR/try_identity_e2e.rs:+9:45: +9:51
         discriminant(_0) = 1;            // scope 4 at $DIR/try_identity_e2e.rs:+9:45: +9:51
         StorageDead(_2);                 // scope 0 at $DIR/try_identity_e2e.rs:+12:1: +12:2
-        return;                          // scope 0 at $DIR/try_identity_e2e.rs:+12:1: +12:2
+        return;                          // scope 0 at $DIR/try_identity_e2e.rs:+12:2: +12:2
     }
 
-    bb4: {
+    bb5: {
         unreachable;                     // scope 0 at $DIR/try_identity_e2e.rs:+2:15: +7:10
     }
 
-    bb5: {
+    bb6: {
         _7 = move ((_2 as Continue).0: T); // scope 0 at $DIR/try_identity_e2e.rs:+8:35: +8:36
         Deinit(_0);                      // scope 0 at $DIR/try_identity_e2e.rs:+1:5: +11:6
         ((_0 as Ok).0: T) = move _7;     // scope 0 at $DIR/try_identity_e2e.rs:+1:5: +11:6
         discriminant(_0) = 0;            // scope 0 at $DIR/try_identity_e2e.rs:+1:5: +11:6
         StorageDead(_2);                 // scope 0 at $DIR/try_identity_e2e.rs:+12:1: +12:2
-        return;                          // scope 0 at $DIR/try_identity_e2e.rs:+12:1: +12:2
+        return;                          // scope 0 at $DIR/try_identity_e2e.rs:+12:2: +12:2
     }
 }

tests/mir-opt/try_identity_e2e.old.PreCodegen.after.mir

@@ -23,7 +23,7 @@ fn old(_1: Result<T, E>) -> Result<T, E> {
         Deinit(_0);                      // scope 2 at $DIR/try_identity_e2e.rs:+4:30: +4:36
         ((_0 as Err).0: E) = move _4;    // scope 2 at $DIR/try_identity_e2e.rs:+4:30: +4:36
         discriminant(_0) = 1;            // scope 2 at $DIR/try_identity_e2e.rs:+4:30: +4:36
-        return;                          // scope 0 at $DIR/try_identity_e2e.rs:+7:1: +7:2
+        return;                          // scope 0 at $DIR/try_identity_e2e.rs:+7:2: +7:2
     }
 
     bb2: {
@@ -35,6 +35,6 @@ fn old(_1: Result<T, E>) -> Result<T, E> {
         Deinit(_0);                      // scope 0 at $DIR/try_identity_e2e.rs:+1:5: +6:6
         ((_0 as Ok).0: T) = move _3;     // scope 0 at $DIR/try_identity_e2e.rs:+1:5: +6:6
         discriminant(_0) = 0;            // scope 0 at $DIR/try_identity_e2e.rs:+1:5: +6:6
-        return;                          // scope 0 at $DIR/try_identity_e2e.rs:+7:1: +7:2
+        return;                          // scope 0 at $DIR/try_identity_e2e.rs:+7:2: +7:2
     }
 }