Skip to content

Commit 39ba9d3

Browse files
fhahnfhahn
committed
[VPlan] Model middle block via VPIRBasicBlock.
Use VPIRBasicBlock to wrap the middle block and implement patching up branches in predecessors in VPIRBasicBlock::execute. The IR middle block is only created after skeleton creation. Initially a regular VPBasicBlock is created, which will later be replaced by a VPIRBasicBlock once the middle IR basic block has been created. Note that this slightly changes the order of instructions created in the middle block; code generated by recipe execution in the middle block will now be inserted before the terminator (and in between the compare to used by the terminator). The original order will be restored in #92651.
1 parent f7336b9 commit 39ba9d3

File tree

4 files changed

+50
-35
lines changed

4 files changed

+50
-35
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 40 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -448,13 +448,29 @@ VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
448448
}
449449

450450
void VPIRBasicBlock::execute(VPTransformState *State) {
451-
assert(getHierarchicalPredecessors().empty() &&
452-
"VPIRBasicBlock cannot have predecessors at the moment");
453451
assert(getHierarchicalSuccessors().empty() &&
454452
"VPIRBasicBlock cannot have successors at the moment");
455453

456454
State->Builder.SetInsertPoint(getIRBasicBlock()->getTerminator());
457455
executeRecipes(State, getIRBasicBlock());
456+
457+
for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
458+
VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock();
459+
BasicBlock *PredBB = State->CFG.VPBB2IRBB[PredVPBB];
460+
assert(PredBB && "Predecessor basic-block not found building successor.");
461+
LLVM_DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n');
462+
463+
auto *PredBBTerminator = PredBB->getTerminator();
464+
auto *TermBr = cast<BranchInst>(PredBBTerminator);
465+
// Set each forward successor here when it is created, excluding
466+
// backedges. A backward successor is set when the branch is created.
467+
const auto &PredVPSuccessors = PredVPBB->getHierarchicalSuccessors();
468+
unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
469+
assert(!TermBr->getSuccessor(idx) &&
470+
"Trying to reset an existing successor block.");
471+
TermBr->setSuccessor(idx, IRBB);
472+
State->CFG.DTU.applyUpdates({{DominatorTree::Insert, PredBB, IRBB}});
473+
}
458474
}
459475

460476
void VPBasicBlock::execute(VPTransformState *State) {
@@ -469,29 +485,13 @@ void VPBasicBlock::execute(VPTransformState *State) {
469485
};
470486

471487
// 1. Create an IR basic block, or reuse the last one or ExitBB if possible.
472-
if (getPlan()->getVectorLoopRegion()->getSingleSuccessor() == this) {
473-
// ExitBB can be re-used for the exit block of the Plan.
474-
NewBB = State->CFG.ExitBB;
475-
State->CFG.PrevBB = NewBB;
476-
State->Builder.SetInsertPoint(NewBB->getFirstNonPHI());
477-
478-
// Update the branch instruction in the predecessor to branch to ExitBB.
479-
VPBlockBase *PredVPB = getSingleHierarchicalPredecessor();
480-
VPBasicBlock *ExitingVPBB = PredVPB->getExitingBasicBlock();
481-
assert(PredVPB->getSingleSuccessor() == this &&
482-
"predecessor must have the current block as only successor");
483-
BasicBlock *ExitingBB = State->CFG.VPBB2IRBB[ExitingVPBB];
484-
// The Exit block of a loop is always set to be successor 0 of the Exiting
485-
// block.
486-
cast<BranchInst>(ExitingBB->getTerminator())->setSuccessor(0, NewBB);
487-
State->CFG.DTU.applyUpdates({{DominatorTree::Insert, ExitingBB, NewBB}});
488-
} else if (PrevVPBB && /* A */
489-
!((SingleHPred = getSingleHierarchicalPredecessor()) &&
490-
SingleHPred->getExitingBasicBlock() == PrevVPBB &&
491-
PrevVPBB->getSingleHierarchicalSuccessor() &&
492-
(SingleHPred->getParent() == getEnclosingLoopRegion() &&
493-
!IsLoopRegion(SingleHPred))) && /* B */
494-
!(Replica && getPredecessors().empty())) { /* C */
488+
if (PrevVPBB && /* A */
489+
!((SingleHPred = getSingleHierarchicalPredecessor()) &&
490+
SingleHPred->getExitingBasicBlock() == PrevVPBB &&
491+
PrevVPBB->getSingleHierarchicalSuccessor() &&
492+
(SingleHPred->getParent() == getEnclosingLoopRegion() &&
493+
!IsLoopRegion(SingleHPred))) && /* B */
494+
!(Replica && getPredecessors().empty())) { /* C */
495495
// The last IR basic block is reused, as an optimization, in three cases:
496496
// A. the first VPBB reuses the loop pre-header BB - when PrevVPBB is null;
497497
// B. when the current VPBB has a single (hierarchical) predecessor which
@@ -842,6 +842,18 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
842842
}
843843
}
844844

845+
/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
846+
/// VPBB are moved to the newly created VPIRBasicBlock.
847+
static void replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB) {
848+
VPIRBasicBlock *IRMiddleVPBB = new VPIRBasicBlock(IRBB);
849+
for (auto &R : make_early_inc_range(*VPBB))
850+
R.moveBefore(*IRMiddleVPBB, IRMiddleVPBB->end());
851+
VPBlockBase *PredVPBB = VPBB->getSinglePredecessor();
852+
VPBlockUtils::disconnectBlocks(PredVPBB, VPBB);
853+
VPBlockUtils::connectBlocks(PredVPBB, IRMiddleVPBB);
854+
delete VPBB;
855+
}
856+
845857
/// Generate the code inside the preheader and body of the vectorized loop.
846858
/// Assumes a single pre-header basic-block was created for this. Introduce
847859
/// additional basic-blocks as needed, and fill them all.
@@ -851,6 +863,9 @@ void VPlan::execute(VPTransformState *State) {
851863
State->CFG.ExitBB = State->CFG.PrevBB->getSingleSuccessor();
852864
BasicBlock *VectorPreHeader = State->CFG.PrevBB;
853865
State->Builder.SetInsertPoint(VectorPreHeader->getTerminator());
866+
replaceVPBBWithIRVPBB(
867+
cast<VPBasicBlock>(getVectorLoopRegion()->getSingleSuccessor()),
868+
State->CFG.ExitBB);
854869

855870
// Disconnect VectorPreHeader from ExitBB in both the CFG and DT.
856871
cast<BranchInst>(VectorPreHeader->getTerminator())->setSuccessor(0, nullptr);

‎llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,9 @@ define i64 @pointer_induction_only(ptr %start, ptr %end) {
124124
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
125125
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
126126
; CHECK: middle.block:
127+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
127128
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
128129
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1
129-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
130130
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
131131
; CHECK: scalar.ph:
132132
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]

‎llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ define arm_aapcs_vfpcc i32 @minmaxval4(ptr nocapture readonly %x, ptr nocapture
2626
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
2727
; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
2828
; CHECK: middle.block:
29+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[N]]
2930
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP3]])
3031
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP2]])
31-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[N]]
3232
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
3333
; CHECK: scalar.ph:
3434
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]

‎llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
2828
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
2929
; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3030
; CHECK: middle.block:
31-
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])
3231
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
32+
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])
3333
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
3434
; CHECK: vec.epilog.iter.check:
3535
; CHECK-NEXT: [[N_VEC_REMAINING:%.*]] = sub i64 [[N]], [[N_VEC]]
@@ -54,8 +54,8 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
5454
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
5555
; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
5656
; CHECK: vec.epilog.middle.block:
57-
; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP10]])
5857
; CHECK-NEXT: [[CMP_N4:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
58+
; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP10]])
5959
; CHECK-NEXT: br i1 [[CMP_N4]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
6060
; CHECK: vec.epilog.scalar.ph:
6161
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
@@ -119,8 +119,8 @@ define float @fp_reduction_max(ptr noalias %a, i64 %N) {
119119
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
120120
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
121121
; CHECK: middle.block:
122-
; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP4]])
123122
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
123+
; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP4]])
124124
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
125125
; CHECK: vec.epilog.iter.check:
126126
; CHECK-NEXT: [[N_VEC_REMAINING:%.*]] = sub i64 [[N]], [[N_VEC]]
@@ -147,8 +147,8 @@ define float @fp_reduction_max(ptr noalias %a, i64 %N) {
147147
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
148148
; CHECK-NEXT: br i1 [[TMP12]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
149149
; CHECK: vec.epilog.middle.block:
150-
; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP11]])
151150
; CHECK-NEXT: [[CMP_N4:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
151+
; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP11]])
152152
; CHECK-NEXT: br i1 [[CMP_N4]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
153153
; CHECK: vec.epilog.scalar.ph:
154154
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
@@ -313,9 +313,9 @@ define float @multiple_fp_rdx(ptr %A, i64 %N) {
313313
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
314314
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
315315
; CHECK: middle.block:
316+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
316317
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP4]])
317318
; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP3]])
318-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
319319
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
320320
; CHECK: vec.epilog.iter.check:
321321
; CHECK-NEXT: [[N_VEC_REMAINING:%.*]] = sub i64 [[N]], [[N_VEC]]
@@ -344,9 +344,9 @@ define float @multiple_fp_rdx(ptr %A, i64 %N) {
344344
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC5]]
345345
; CHECK-NEXT: br i1 [[TMP15]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
346346
; CHECK: vec.epilog.middle.block:
347+
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
347348
; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP14]])
348349
; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP13]])
349-
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
350350
; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
351351
; CHECK: vec.epilog.scalar.ph:
352352
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
@@ -422,8 +422,8 @@ define i32 @reduction_phi_start_val(ptr %A, i64 %N) {
422422
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
423423
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
424424
; CHECK: middle.block:
425-
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
426425
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
426+
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
427427
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
428428
; CHECK: vec.epilog.iter.check:
429429
; CHECK-NEXT: [[N_VEC_REMAINING:%.*]] = sub i64 [[N]], [[N_VEC]]
@@ -448,8 +448,8 @@ define i32 @reduction_phi_start_val(ptr %A, i64 %N) {
448448
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
449449
; CHECK-NEXT: br i1 [[TMP12]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
450450
; CHECK: vec.epilog.middle.block:
451-
; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP11]])
452451
; CHECK-NEXT: [[CMP_N4:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
452+
; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP11]])
453453
; CHECK-NEXT: br i1 [[CMP_N4]], label [[FOR_COND]], label [[VEC_EPILOG_SCALAR_PH]]
454454
; CHECK: vec.epilog.scalar.ph:
455455
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]

0 commit comments

Comments
 (0)