Skip to content

Commit de6eba7

Browse files
fhahnfhahn
committed
[VPlan] Add initial CFG simplification, removing BranchOnCond true.
Add an initial CFG simplification transform, which removes the dead edges for blocks terminated with BranchOnCond true. At the moment, this removes the edge between middle block and scalar preheader when folding the tail. Note: not all tests have been updated yet, but the current tests give a good idea of the change.
1 parent 3726f9c commit de6eba7

34 files changed

+141
-82
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1007,9 +1007,19 @@ void VPlan::execute(VPTransformState *State) {
10071007
"middle block has unexpected successors");
10081008
VPBasicBlock *ScalarPhVPBB = cast<VPBasicBlock>(
10091009
MiddleSuccs.size() == 1 ? MiddleSuccs[0] : MiddleSuccs[1]);
1010-
assert(!isa<VPIRBasicBlock>(ScalarPhVPBB) &&
1011-
"scalar preheader cannot be wrapped already");
1012-
replaceVPBBWithIRVPBB(ScalarPhVPBB, ScalarPh);
1010+
if (!isa<VPIRBasicBlock>(ScalarPhVPBB)) {
1011+
assert(!isa<VPIRBasicBlock>(ScalarPhVPBB) &&
1012+
"scalar preheader cannot be wrapped already");
1013+
replaceVPBBWithIRVPBB(ScalarPhVPBB, ScalarPh);
1014+
} else {
1015+
// There is no edge to the scalar pre-header in VPlan. Phis in ScalarPh have
1016+
// been created during skeleton construction, so remove the incoming values
1017+
// from the middle block here.
1018+
// TODO: Remove this once phis in the scalar preheader are managed in VPlan
1019+
// directly.
1020+
for (auto &Phi : ScalarPh->phis())
1021+
Phi.removeIncomingValue(MiddleBB);
1022+
}
10131023
replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB);
10141024

10151025
// Disconnect the middle block from its single successor (the scalar loop

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3504,6 +3504,11 @@ class VPlan {
35043504
return LiveOuts;
35053505
}
35063506

3507+
void removeLiveOut(PHINode *PN) {
3508+
delete LiveOuts[PN];
3509+
LiveOuts.erase(PN);
3510+
}
3511+
35073512
VPValue *getSCEVExpansion(const SCEV *S) const {
35083513
return SCEVToExpansion.lookup(S);
35093514
}

‎llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ inline specific_intval<0> m_SpecificInt(uint64_t V) {
8585
return specific_intval<0>(APInt(64, V));
8686
}
8787

88+
inline specific_intval<1> m_True() { return specific_intval<1>(APInt(64, 1)); }
8889
inline specific_intval<1> m_False() { return specific_intval<1>(APInt(64, 0)); }
8990

9091
/// Matching combinators

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1120,6 +1120,49 @@ void VPlanTransforms::truncateToMinimalBitwidths(
11201120
"some entries in MinBWs haven't been processed");
11211121
}
11221122

1123+
/// Remove BranchOnCond recipes with constant conditions together with removing
1124+
/// dead edges to their successors. Remove blocks that become dead (no remaining
1125+
/// predecessors())
1126+
static void simplifyCFG(VPlan &Plan) {
1127+
using namespace llvm::VPlanPatternMatch;
1128+
SmallVector<VPBasicBlock *> WorkList;
1129+
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1130+
vp_depth_first_deep(Plan.getEntry()))) {
1131+
VPRecipeBase *Term = VPBB->getTerminator();
1132+
if (!Term || !match(Term, m_BranchOnCond(m_True())))
1133+
continue;
1134+
WorkList.push_back(VPBB);
1135+
}
1136+
1137+
SetVector<VPBasicBlock *> PossiblyDeadBlocks;
1138+
for (VPBasicBlock *VPBB : WorkList) {
1139+
VPRecipeBase *Term = VPBB->getTerminator();
1140+
VPBasicBlock *DeadSucc = cast<VPBasicBlock>(VPBB->getSuccessors()[1]);
1141+
VPBlockUtils::disconnectBlocks(VPBB, DeadSucc);
1142+
PossiblyDeadBlocks.insert(DeadSucc);
1143+
Term->eraseFromParent();
1144+
}
1145+
for (VPBasicBlock *VPBB : PossiblyDeadBlocks) {
1146+
if (VPBB->getNumPredecessors() != 0)
1147+
continue;
1148+
// The block doesn't have any predecessors, remove it.
1149+
//
1150+
// To do so, first remove all recipes in the block. At the moment, recipes
1151+
// with users outside the block must be live-outs. Those are removed.
1152+
SmallVector<PHINode *> DeadLiveOuts;
1153+
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
1154+
if (auto *V = dyn_cast<VPSingleDefRecipe>(&R)) {
1155+
for (VPUser *U : to_vector(V->users())) {
1156+
auto *LO = cast<VPLiveOut>(U);
1157+
Plan.removeLiveOut(LO->getPhi());
1158+
}
1159+
}
1160+
R.eraseFromParent();
1161+
}
1162+
delete VPBB;
1163+
}
1164+
}
1165+
11231166
void VPlanTransforms::optimize(VPlan &Plan, ScalarEvolution &SE) {
11241167
removeRedundantCanonicalIVs(Plan);
11251168
removeRedundantInductionCasts(Plan);
@@ -1132,6 +1175,7 @@ void VPlanTransforms::optimize(VPlan &Plan, ScalarEvolution &SE) {
11321175

11331176
removeRedundantExpandSCEVRecipes(Plan);
11341177
mergeBlocksIntoPredecessors(Plan);
1178+
simplifyCFG(Plan);
11351179
}
11361180

11371181
// Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace

‎llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,13 @@ define i32 @test_phi_iterator_invalidation(ptr %A, ptr noalias %B) {
7171
; CHECK-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
7272
; CHECK: middle.block:
7373
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP24]], i32 3
74-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
74+
; CHECK-NEXT: br label [[EXIT:%.*]]
7575
; CHECK: scalar.ph:
76-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1004, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
77-
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
76+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
7877
; CHECK-NEXT: br label [[LOOP:%.*]]
7978
; CHECK: loop:
8079
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
81-
; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
80+
; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
8281
; CHECK-NEXT: [[SEXT:%.*]] = sext i16 [[SCALAR_RECUR]] to i32
8382
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
8483
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT]]

‎llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,10 @@ define i32 @pr70988(ptr %src, i32 %n) {
5454
; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
5555
; CHECK: middle.block:
5656
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP17]], i32 [[TMP18]])
57-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
57+
; CHECK-NEXT: br label [[EXIT:%.*]]
5858
; CHECK: scalar.ph:
59-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
60-
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_MINMAX]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
59+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
60+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
6161
; CHECK-NEXT: br label [[LOOP:%.*]]
6262
; CHECK: loop:
6363
; CHECK-NEXT: [[INDUC:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDUC_NEXT:%.*]], [[LOOP]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ define void @trip1025_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapt
3737
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <vscale x 2 x i1> [[TMP14]], i32 0
3838
; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3939
; CHECK: middle.block:
40-
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
40+
; CHECK-NEXT: br label [[FOR_END:%.*]]
4141
; CHECK: scalar.ph:
42-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
42+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
4343
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
4444
; CHECK: for.body:
4545
; CHECK-NEXT: [[I_06:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ define void @test(ptr %p, i64 %a, i8 %b) {
5858
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
5959
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
6060
; CHECK: middle.block:
61-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
61+
; CHECK-NEXT: br label [[EXIT:%.*]]
6262
; CHECK: scalar.ph:
63-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 4, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
63+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
6464
; CHECK-NEXT: br label [[FOR_COND:%.*]]
6565
; CHECK: for.cond:
6666
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY:%.*]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ define void @small_trip_count_min_vlen_128(ptr nocapture %a) nounwind vscale_ran
2525
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP2]]
2626
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
2727
; CHECK: middle.block:
28-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
28+
; CHECK-NEXT: br label [[EXIT:%.*]]
2929
; CHECK: scalar.ph:
30-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
30+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
3131
; CHECK-NEXT: br label [[LOOP:%.*]]
3232
; CHECK: loop:
3333
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -84,9 +84,9 @@ define void @small_trip_count_min_vlen_32(ptr nocapture %a) nounwind vscale_rang
8484
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]]
8585
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
8686
; CHECK: middle.block:
87-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
87+
; CHECK-NEXT: br label [[EXIT:%.*]]
8888
; CHECK: scalar.ph:
89-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
89+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
9090
; CHECK-NEXT: br label [[LOOP:%.*]]
9191
; CHECK: loop:
9292
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,9 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
7575
; IF-EVL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
7676
; IF-EVL-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
7777
; IF-EVL: middle.block:
78-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
78+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
7979
; IF-EVL: scalar.ph:
80-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
80+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
8181
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
8282
; IF-EVL: for.body:
8383
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
4444
; IF-EVL-NEXT: [[TMP18:%.*]] = icmp eq i32 [[IV_NEXT]], [[N_VEC]]
4545
; IF-EVL-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4646
; IF-EVL: middle.block:
47-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
47+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
4848
; IF-EVL: scalar.ph:
49-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
49+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY1:%.*]] ]
5050
; IF-EVL-NEXT: br label [[FOR_BODY1:%.*]]
5151
; IF-EVL: for.body:
5252
; IF-EVL-NEXT: [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
5858
; IF-EVL-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
5959
; IF-EVL-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
6060
; IF-EVL: middle.block:
61-
; IF-EVL-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
61+
; IF-EVL-NEXT: br label [[EXIT:%.*]]
6262
; IF-EVL: scalar.ph:
63-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
63+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
6464
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
6565
; IF-EVL: for.body:
6666
; IF-EVL-NEXT: [[I_011:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@ define float @fadd(ptr noalias nocapture readonly %a, i64 %n) {
4646
; IF-EVL-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4747
; IF-EVL-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4848
; IF-EVL: middle.block:
49-
; IF-EVL-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
49+
; IF-EVL-NEXT: br label [[FOR_END:%.*]]
5050
; IF-EVL: scalar.ph:
51-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
52-
; IF-EVL-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
51+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
52+
; IF-EVL-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
5353
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
5454
; IF-EVL: for.body:
5555
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
4949
; IF-EVL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
5050
; IF-EVL-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
5151
; IF-EVL: middle.block:
52-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
52+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
5353
; IF-EVL: scalar.ph:
54-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
54+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
5555
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
5656
; IF-EVL: for.body:
5757
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ define void @test_scalar_steps_target_instruction_cost(ptr %dst) {
3737
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10
3838
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3939
; CHECK: [[MIDDLE_BLOCK]]:
40-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
40+
; CHECK-NEXT: br label %[[EXIT:.*]]
4141
; CHECK: [[SCALAR_PH]]:
42-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 30, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
42+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
4343
; CHECK-NEXT: br label %[[LOOP:.*]]
4444
; CHECK: [[LOOP]]:
4545
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]

‎llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,9 @@ define void @test(ptr %p, i40 %a) {
139139
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 16
140140
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
141141
; CHECK: middle.block:
142-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
142+
; CHECK-NEXT: br label [[EXIT:%.*]]
143143
; CHECK: scalar.ph:
144-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
144+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
145145
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
146146
; CHECK: for.body:
147147
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
4444
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4545
; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4646
; CHECK: [[MIDDLE_BLOCK]]:
47-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
47+
; CHECK-NEXT: br label %[[EXIT:.*]]
4848
; CHECK: [[SCALAR_PH]]:
49-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
49+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
5050
; CHECK-NEXT: br label %[[LOOP:.*]]
5151
; CHECK: [[LOOP]]:
5252
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -163,9 +163,9 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
163163
; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
164164
; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
165165
; CHECK: [[MIDDLE_BLOCK]]:
166-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
166+
; CHECK-NEXT: br label %[[EXIT:.*]]
167167
; CHECK: [[SCALAR_PH]]:
168-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
168+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
169169
; CHECK-NEXT: br label %[[LOOP:.*]]
170170
; CHECK: [[LOOP]]:
171171
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]

‎llvm/test/Transforms/LoopVectorize/X86/fp64_to_uint32-cost-model.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ for.body: ; preds = %for.body.preheader,
2929
store i32 %conv, ptr %arrayidx2, align 4
3030
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
3131
%2 = trunc i64 %indvars.iv.next to i32
32-
%cmp = icmp ult i32 %2, %0
32+
%cmp = icmp ult i32 %2, 1000
3333
br i1 %cmp, label %for.body, label %for.end.loopexit
3434

3535
for.end.loopexit: ; preds = %for.body

‎llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,9 +136,9 @@ define void @vectorized1(ptr noalias nocapture %A, ptr noalias nocapture readonl
136136
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
137137
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
138138
; CHECK: middle.block:
139-
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
139+
; CHECK-NEXT: br label [[FOR_END:%.*]]
140140
; CHECK: scalar.ph:
141-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 24, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
141+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
142142
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
143143
; CHECK: for.body:
144144
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
4242
; IF-EVL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4343
; IF-EVL-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4444
; IF-EVL: middle.block:
45-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
45+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
4646
; IF-EVL: scalar.ph:
47-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
47+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
4848
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
4949
; IF-EVL: for.body:
5050
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,9 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
7777
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10008
7878
; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
7979
; CHECK: middle.block:
80-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
80+
; CHECK-NEXT: br label [[EXIT:%.*]]
8181
; CHECK: scalar.ph:
82-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 10008, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
82+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
8383
; CHECK-NEXT: br label [[LOOP:%.*]]
8484
; CHECK: loop:
8585
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]

‎llvm/test/Transforms/LoopVectorize/as_cast.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ loop:
2929

3030
; check that we branch to the exit block
3131
; CHECK: middle.block:
32-
; CHECK: br i1 true, label %exit, label %scalar.ph
32+
; CHECK: br label %exit
3333

3434
exit:
3535
ret void

0 commit comments

Comments
 (0)