Skip to content

Commit c66649c

Browse files
fhahnfhahn
committed
[VPlan] Add initial CFG simplification, removing BranchOnCond true.
Add an initial CFG simplification transform, which removes the dead edges for blocks terminated with BranchOnCond true. At the moment, this removes the edge between middle block and scalar preheader when folding the tail. Note: not all tests have been updated yet, but the current tests give a good idea of the change.
1 parent 206b5af commit c66649c

34 files changed

+141
-82
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1007,9 +1007,19 @@ void VPlan::execute(VPTransformState *State) {
10071007
"middle block has unexpected successors");
10081008
VPBasicBlock *ScalarPhVPBB = cast<VPBasicBlock>(
10091009
MiddleSuccs.size() == 1 ? MiddleSuccs[0] : MiddleSuccs[1]);
1010-
assert(!isa<VPIRBasicBlock>(ScalarPhVPBB) &&
1011-
"scalar preheader cannot be wrapped already");
1012-
replaceVPBBWithIRVPBB(ScalarPhVPBB, ScalarPh);
1010+
if (!isa<VPIRBasicBlock>(ScalarPhVPBB)) {
1011+
assert(!isa<VPIRBasicBlock>(ScalarPhVPBB) &&
1012+
"scalar preheader cannot be wrapped already");
1013+
replaceVPBBWithIRVPBB(ScalarPhVPBB, ScalarPh);
1014+
} else {
1015+
// There is no edge to the scalar pre-header in VPlan. Phis in ScalarPh have
1016+
// been created during skeleton construction, so remove the incoming values
1017+
// from the middle block here.
1018+
// TODO: Remove this once phis in the scalar preheader are managed in VPlan
1019+
// directly.
1020+
for (auto &Phi : ScalarPh->phis())
1021+
Phi.removeIncomingValue(MiddleBB);
1022+
}
10131023
replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB);
10141024

10151025
// Disconnect the middle block from its single successor (the scalar loop

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3496,6 +3496,11 @@ class VPlan {
34963496
return LiveOuts;
34973497
}
34983498

3499+
void removeLiveOut(PHINode *PN) {
3500+
delete LiveOuts[PN];
3501+
LiveOuts.erase(PN);
3502+
}
3503+
34993504
VPValue *getSCEVExpansion(const SCEV *S) const {
35003505
return SCEVToExpansion.lookup(S);
35013506
}

‎llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ inline specific_intval<0> m_SpecificInt(uint64_t V) {
8585
return specific_intval<0>(APInt(64, V));
8686
}
8787

88+
inline specific_intval<1> m_True() { return specific_intval<1>(APInt(64, 1)); }
8889
inline specific_intval<1> m_False() { return specific_intval<1>(APInt(64, 0)); }
8990

9091
/// Matching combinators

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1119,6 +1119,49 @@ void VPlanTransforms::truncateToMinimalBitwidths(
11191119
"some entries in MinBWs haven't been processed");
11201120
}
11211121

1122+
/// Remove BranchOnCond recipes with constant conditions together with removing
1123+
/// dead edges to their successors. Remove blocks that become dead (no remaining
1124+
/// predecessors())
1125+
static void simplifyCFG(VPlan &Plan) {
1126+
using namespace llvm::VPlanPatternMatch;
1127+
SmallVector<VPBasicBlock *> WorkList;
1128+
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1129+
vp_depth_first_deep(Plan.getEntry()))) {
1130+
VPRecipeBase *Term = VPBB->getTerminator();
1131+
if (!Term || !match(Term, m_BranchOnCond(m_True())))
1132+
continue;
1133+
WorkList.push_back(VPBB);
1134+
}
1135+
1136+
SetVector<VPBasicBlock *> PossiblyDeadBlocks;
1137+
for (VPBasicBlock *VPBB : WorkList) {
1138+
VPRecipeBase *Term = VPBB->getTerminator();
1139+
VPBasicBlock *DeadSucc = cast<VPBasicBlock>(VPBB->getSuccessors()[1]);
1140+
VPBlockUtils::disconnectBlocks(VPBB, DeadSucc);
1141+
PossiblyDeadBlocks.insert(DeadSucc);
1142+
Term->eraseFromParent();
1143+
}
1144+
for (VPBasicBlock *VPBB : PossiblyDeadBlocks) {
1145+
if (VPBB->getNumPredecessors() != 0)
1146+
continue;
1147+
// The block doesn't have any predecessors, remove it.
1148+
//
1149+
// To do so, first remove all recipes in the block. At the moment, recipes
1150+
// with users outside the block must be live-outs. Those are removed.
1151+
SmallVector<PHINode *> DeadLiveOuts;
1152+
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
1153+
if (auto *V = dyn_cast<VPSingleDefRecipe>(&R)) {
1154+
for (VPUser *U : to_vector(V->users())) {
1155+
auto *LO = cast<VPLiveOut>(U);
1156+
Plan.removeLiveOut(LO->getPhi());
1157+
}
1158+
}
1159+
R.eraseFromParent();
1160+
}
1161+
delete VPBB;
1162+
}
1163+
}
1164+
11221165
void VPlanTransforms::optimize(VPlan &Plan, ScalarEvolution &SE) {
11231166
removeRedundantCanonicalIVs(Plan);
11241167
removeRedundantInductionCasts(Plan);
@@ -1131,6 +1174,7 @@ void VPlanTransforms::optimize(VPlan &Plan, ScalarEvolution &SE) {
11311174

11321175
removeRedundantExpandSCEVRecipes(Plan);
11331176
mergeBlocksIntoPredecessors(Plan);
1177+
simplifyCFG(Plan);
11341178
}
11351179

11361180
// Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace

‎llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,13 @@ define i32 @test_phi_iterator_invalidation(ptr %A, ptr noalias %B) {
7171
; CHECK-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
7272
; CHECK: middle.block:
7373
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP24]], i32 3
74-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
74+
; CHECK-NEXT: br label [[EXIT:%.*]]
7575
; CHECK: scalar.ph:
76-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1004, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
77-
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
76+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
7877
; CHECK-NEXT: br label [[LOOP:%.*]]
7978
; CHECK: loop:
8079
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
81-
; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
80+
; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
8281
; CHECK-NEXT: [[SEXT:%.*]] = sext i16 [[SCALAR_RECUR]] to i32
8382
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
8483
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT]]

‎llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,10 @@ define i32 @pr70988(ptr %src, i32 %n) {
5454
; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
5555
; CHECK: middle.block:
5656
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP17]], i32 [[TMP18]])
57-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
57+
; CHECK-NEXT: br label [[EXIT:%.*]]
5858
; CHECK: scalar.ph:
59-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
60-
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_MINMAX]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
59+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
60+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
6161
; CHECK-NEXT: br label [[LOOP:%.*]]
6262
; CHECK: loop:
6363
; CHECK-NEXT: [[INDUC:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDUC_NEXT:%.*]], [[LOOP]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ define void @trip1025_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapt
3737
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <vscale x 2 x i1> [[TMP14]], i32 0
3838
; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3939
; CHECK: middle.block:
40-
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
40+
; CHECK-NEXT: br label [[FOR_END:%.*]]
4141
; CHECK: scalar.ph:
42-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
42+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
4343
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
4444
; CHECK: for.body:
4545
; CHECK-NEXT: [[I_06:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ define void @test(ptr %p, i64 %a, i8 %b) {
5858
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
5959
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
6060
; CHECK: middle.block:
61-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
61+
; CHECK-NEXT: br label [[EXIT:%.*]]
6262
; CHECK: scalar.ph:
63-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 4, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
63+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
6464
; CHECK-NEXT: br label [[FOR_COND:%.*]]
6565
; CHECK: for.cond:
6666
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY:%.*]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ define void @small_trip_count_min_vlen_128(ptr nocapture %a) nounwind vscale_ran
2525
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP2]]
2626
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
2727
; CHECK: middle.block:
28-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
28+
; CHECK-NEXT: br label [[EXIT:%.*]]
2929
; CHECK: scalar.ph:
30-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
30+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
3131
; CHECK-NEXT: br label [[LOOP:%.*]]
3232
; CHECK: loop:
3333
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -84,9 +84,9 @@ define void @small_trip_count_min_vlen_32(ptr nocapture %a) nounwind vscale_rang
8484
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]]
8585
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
8686
; CHECK: middle.block:
87-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
87+
; CHECK-NEXT: br label [[EXIT:%.*]]
8888
; CHECK: scalar.ph:
89-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
89+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
9090
; CHECK-NEXT: br label [[LOOP:%.*]]
9191
; CHECK: loop:
9292
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,9 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
7575
; IF-EVL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
7676
; IF-EVL-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
7777
; IF-EVL: middle.block:
78-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
78+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
7979
; IF-EVL: scalar.ph:
80-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
80+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
8181
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
8282
; IF-EVL: for.body:
8383
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
4444
; IF-EVL-NEXT: [[TMP18:%.*]] = icmp eq i32 [[IV_NEXT]], [[N_VEC]]
4545
; IF-EVL-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4646
; IF-EVL: middle.block:
47-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
47+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
4848
; IF-EVL: scalar.ph:
49-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
49+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY1:%.*]] ]
5050
; IF-EVL-NEXT: br label [[FOR_BODY1:%.*]]
5151
; IF-EVL: for.body:
5252
; IF-EVL-NEXT: [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
5858
; IF-EVL-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
5959
; IF-EVL-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
6060
; IF-EVL: middle.block:
61-
; IF-EVL-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
61+
; IF-EVL-NEXT: br label [[EXIT:%.*]]
6262
; IF-EVL: scalar.ph:
63-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
63+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
6464
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
6565
; IF-EVL: for.body:
6666
; IF-EVL-NEXT: [[I_011:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@ define float @fadd(ptr noalias nocapture readonly %a, i64 %n) {
4646
; IF-EVL-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4747
; IF-EVL-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4848
; IF-EVL: middle.block:
49-
; IF-EVL-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
49+
; IF-EVL-NEXT: br label [[FOR_END:%.*]]
5050
; IF-EVL: scalar.ph:
51-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
52-
; IF-EVL-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
51+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
52+
; IF-EVL-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
5353
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
5454
; IF-EVL: for.body:
5555
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
4949
; IF-EVL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
5050
; IF-EVL-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
5151
; IF-EVL: middle.block:
52-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
52+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
5353
; IF-EVL: scalar.ph:
54-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
54+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
5555
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
5656
; IF-EVL: for.body:
5757
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ define void @test_scalar_steps_target_instruction_cost(ptr %dst) {
3737
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10
3838
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3939
; CHECK: [[MIDDLE_BLOCK]]:
40-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
40+
; CHECK-NEXT: br label %[[EXIT:.*]]
4141
; CHECK: [[SCALAR_PH]]:
42-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 30, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
42+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
4343
; CHECK-NEXT: br label %[[LOOP:.*]]
4444
; CHECK: [[LOOP]]:
4545
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]

‎llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,9 @@ define void @test(ptr %p, i40 %a) {
139139
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 16
140140
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
141141
; CHECK: middle.block:
142-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
142+
; CHECK-NEXT: br label [[EXIT:%.*]]
143143
; CHECK: scalar.ph:
144-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
144+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
145145
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
146146
; CHECK: for.body:
147147
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
4444
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4545
; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4646
; CHECK: [[MIDDLE_BLOCK]]:
47-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
47+
; CHECK-NEXT: br label %[[EXIT:.*]]
4848
; CHECK: [[SCALAR_PH]]:
49-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
49+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
5050
; CHECK-NEXT: br label %[[LOOP:.*]]
5151
; CHECK: [[LOOP]]:
5252
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -163,9 +163,9 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
163163
; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
164164
; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
165165
; CHECK: [[MIDDLE_BLOCK]]:
166-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
166+
; CHECK-NEXT: br label %[[EXIT:.*]]
167167
; CHECK: [[SCALAR_PH]]:
168-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
168+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
169169
; CHECK-NEXT: br label %[[LOOP:.*]]
170170
; CHECK: [[LOOP]]:
171171
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]

‎llvm/test/Transforms/LoopVectorize/X86/fp64_to_uint32-cost-model.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ for.body: ; preds = %for.body.preheader,
2929
store i32 %conv, ptr %arrayidx2, align 4
3030
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
3131
%2 = trunc i64 %indvars.iv.next to i32
32-
%cmp = icmp ult i32 %2, %0
32+
%cmp = icmp ult i32 %2, 1000
3333
br i1 %cmp, label %for.body, label %for.end.loopexit
3434

3535
for.end.loopexit: ; preds = %for.body

‎llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,9 +136,9 @@ define void @vectorized1(ptr noalias nocapture %A, ptr noalias nocapture readonl
136136
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
137137
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
138138
; CHECK: middle.block:
139-
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
139+
; CHECK-NEXT: br label [[FOR_END:%.*]]
140140
; CHECK: scalar.ph:
141-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 24, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
141+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
142142
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
143143
; CHECK: for.body:
144144
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
4242
; IF-EVL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4343
; IF-EVL-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4444
; IF-EVL: middle.block:
45-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
45+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
4646
; IF-EVL: scalar.ph:
47-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
47+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
4848
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
4949
; IF-EVL: for.body:
5050
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,9 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
7777
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10008
7878
; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
7979
; CHECK: middle.block:
80-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
80+
; CHECK-NEXT: br label [[EXIT:%.*]]
8181
; CHECK: scalar.ph:
82-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 10008, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
82+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
8383
; CHECK-NEXT: br label [[LOOP:%.*]]
8484
; CHECK: loop:
8585
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]

‎llvm/test/Transforms/LoopVectorize/as_cast.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ loop:
2929

3030
; check that we branch to the exit block
3131
; CHECK: middle.block:
32-
; CHECK: br i1 true, label %exit, label %scalar.ph
32+
; CHECK: br label %exit
3333

3434
exit:
3535
ret void

0 commit comments

Comments
 (0)