@@ -448,13 +448,29 @@ VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
448448}
449449
450450void VPIRBasicBlock::execute (VPTransformState *State) {
451- assert (getHierarchicalPredecessors ().empty () &&
452- " VPIRBasicBlock cannot have predecessors at the moment"
453451 assert (getHierarchicalSuccessors ().empty () &&
454452 " VPIRBasicBlock cannot have successors at the moment"
455453
456454 State->Builder .SetInsertPoint (getIRBasicBlock ()->getTerminator ());
457455 executeRecipes (State, getIRBasicBlock ());
456+
457+ for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors ()) {
458+ VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock ();
459+ BasicBlock *PredBB = State->CFG .VPBB2IRBB [PredVPBB];
460+ assert (PredBB && " Predecessor basic-block not found building successor."
461+ LLVM_DEBUG (dbgs () << " LV: draw edge from" getName () << ' \n '
462+
463+ auto *PredBBTerminator = PredBB->getTerminator ();
464+ auto *TermBr = cast<BranchInst>(PredBBTerminator);
465+ // Set each forward successor here when it is created, excluding
466+ // backedges. A backward successor is set when the branch is created.
467+ const auto &PredVPSuccessors = PredVPBB->getHierarchicalSuccessors ();
468+ unsigned idx = PredVPSuccessors.front () == this ? 0 : 1 ;
469+ assert (!TermBr->getSuccessor (idx) &&
470+ " Trying to reset an existing successor block."
471+ TermBr->setSuccessor (idx, IRBB);
472+ State->CFG .DTU .applyUpdates ({{DominatorTree::Insert, PredBB, IRBB}});
473+ }
458474}
459475
460476void VPBasicBlock::execute (VPTransformState *State) {
@@ -468,30 +484,14 @@ void VPBasicBlock::execute(VPTransformState *State) {
468484 return R && !R->isReplicator ();
469485 };
470486
471- // 1. Create an IR basic block, or reuse the last one or ExitBB if possible.
472- if (getPlan ()->getVectorLoopRegion ()->getSingleSuccessor () == this ) {
473- // ExitBB can be re-used for the exit block of the Plan.
474- NewBB = State->CFG .ExitBB ;
475- State->CFG .PrevBB = NewBB;
476- State->Builder .SetInsertPoint (NewBB->getFirstNonPHI ());
477-
478- // Update the branch instruction in the predecessor to branch to ExitBB.
479- VPBlockBase *PredVPB = getSingleHierarchicalPredecessor ();
480- VPBasicBlock *ExitingVPBB = PredVPB->getExitingBasicBlock ();
481- assert (PredVPB->getSingleSuccessor () == this &&
482- " predecessor must have the current block as only successor"
483- BasicBlock *ExitingBB = State->CFG .VPBB2IRBB [ExitingVPBB];
484- // The Exit block of a loop is always set to be successor 0 of the Exiting
485- // block.
486- cast<BranchInst>(ExitingBB->getTerminator ())->setSuccessor (0 , NewBB);
487- State->CFG .DTU .applyUpdates ({{DominatorTree::Insert, ExitingBB, NewBB}});
488- } else if (PrevVPBB && /* A */
489- !((SingleHPred = getSingleHierarchicalPredecessor ()) &&
490- SingleHPred->getExitingBasicBlock () == PrevVPBB &&
491- PrevVPBB->getSingleHierarchicalSuccessor () &&
492- (SingleHPred->getParent () == getEnclosingLoopRegion () &&
493- !IsLoopRegion (SingleHPred))) && /* B */
494- !(Replica && getPredecessors ().empty ())) { /* C */
487+ // 1. Create an IR basic block.
488+ if (PrevVPBB && /* A */
489+ !((SingleHPred = getSingleHierarchicalPredecessor ()) &&
490+ SingleHPred->getExitingBasicBlock () == PrevVPBB &&
491+ PrevVPBB->getSingleHierarchicalSuccessor () &&
492+ (SingleHPred->getParent () == getEnclosingLoopRegion () &&
493+ !IsLoopRegion (SingleHPred))) && /* B */
494+ !(Replica && getPredecessors ().empty ())) { /* C */
495495 // The last IR basic block is reused, as an optimization, in three cases:
496496 // A. the first VPBB reuses the loop pre-header BB - when PrevVPBB is null;
497497 // B. when the current VPBB has a single (hierarchical) predecessor which
@@ -842,6 +842,19 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
842842 }
843843}
844844
845+ // / Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
846+ // / VPBB are moved to the newly created VPIRBasicBlock.
847+ static void replaceVPBBWithIRVPBB (VPBasicBlock *VPBB, BasicBlock *IRBB) {
848+ assert (VPBB->getNumSuccessors () == 0 && " VPBB must be a leave node"
849+ VPIRBasicBlock *IRMiddleVPBB = new VPIRBasicBlock (IRBB);
850+ for (auto &R : make_early_inc_range (*VPBB))
851+ R.moveBefore (*IRMiddleVPBB, IRMiddleVPBB->end ());
852+ VPBlockBase *PredVPBB = VPBB->getSinglePredecessor ();
853+ VPBlockUtils::disconnectBlocks (PredVPBB, VPBB);
854+ VPBlockUtils::connectBlocks (PredVPBB, IRMiddleVPBB);
855+ delete VPBB;
856+ }
857+
845858// / Generate the code inside the preheader and body of the vectorized loop.
846859// / Assumes a single pre-header basic-block was created for this. Introduce
847860// / additional basic-blocks as needed, and fill them all.
@@ -851,6 +864,9 @@ void VPlan::execute(VPTransformState *State) {
851864 State->CFG .ExitBB = State->CFG .PrevBB ->getSingleSuccessor ();
852865 BasicBlock *VectorPreHeader = State->CFG .PrevBB ;
853866 State->Builder .SetInsertPoint (VectorPreHeader->getTerminator ());
867+ replaceVPBBWithIRVPBB (
868+ cast<VPBasicBlock>(getVectorLoopRegion ()->getSingleSuccessor ()),
869+ State->CFG .ExitBB );
854870
855871 // Disconnect VectorPreHeader from ExitBB in both the CFG and DT.
856872 cast<BranchInst>(VectorPreHeader->getTerminator ())->setSuccessor (0 , nullptr );
0 commit comments