Skip to content

Commit 491b667

Browse files
fhahnfhahn
committed
[VPlan] Add initial CFG simplification, removing BranchOnCond true.
1 parent c36a512 commit 491b667

34 files changed

+140
-97
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2582,6 +2582,20 @@ void InnerLoopVectorizer::createInductionResumeValue(
25822582
ArrayRef<BasicBlock *> BypassBlocks,
25832583
std::pair<BasicBlock *, Value *> AdditionalBypass) {
25842584
Value *VectorTripCount = getOrCreateVectorTripCount(LoopVectorPreHeader);
2585+
VPBasicBlock *MiddleVPBB =
2586+
cast<VPBasicBlock>(Plan.getVectorLoopRegion()->getSingleSuccessor());
2587+
VPBasicBlock *ScalarPHVPBB = nullptr;
2588+
if (MiddleVPBB->getNumSuccessors() == 2) {
2589+
// Order is strict: first is the exit block, second is the scalar preheader.
2590+
ScalarPHVPBB = cast<VPBasicBlock>(MiddleVPBB->getSuccessors()[1]);
2591+
} else {
2592+
ScalarPHVPBB = cast<VPBasicBlock>(MiddleVPBB->getSingleSuccessor());
2593+
}
2594+
auto *ScalarLoopHeader =
2595+
cast_if_present<VPIRBasicBlock>(ScalarPHVPBB->getSingleSuccessor());
2596+
if (!ScalarLoopHeader)
2597+
return;
2598+
25852599
assert(VectorTripCount && "Expected valid arguments");
25862600

25872601
Instruction *OldInduction = Legal->getPrimaryInduction();
@@ -2612,25 +2626,12 @@ void InnerLoopVectorizer::createInductionResumeValue(
26122626
}
26132627
}
26142628

2615-
VPBasicBlock *MiddleVPBB =
2616-
cast<VPBasicBlock>(Plan.getVectorLoopRegion()->getSingleSuccessor());
2617-
2618-
VPBasicBlock *ScalarPHVPBB = nullptr;
2619-
if (MiddleVPBB->getNumSuccessors() == 2) {
2620-
// Order is strict: first is the exit block, second is the scalar preheader.
2621-
ScalarPHVPBB = cast<VPBasicBlock>(MiddleVPBB->getSuccessors()[1]);
2622-
} else {
2623-
ScalarPHVPBB = cast<VPBasicBlock>(MiddleVPBB->getSingleSuccessor());
2624-
}
2625-
26262629
VPBuilder ScalarPHBuilder(ScalarPHVPBB);
26272630
auto *ResumePhiRecipe = ScalarPHBuilder.createNaryOp(
26282631
VPInstruction::ResumePhi,
26292632
{Plan.getOrAddLiveIn(EndValue), Plan.getOrAddLiveIn(II.getStartValue())},
26302633
OrigPhi->getDebugLoc(), "bc.resume.val");
26312634

2632-
auto *ScalarLoopHeader =
2633-
cast<VPIRBasicBlock>(ScalarPHVPBB->getSingleSuccessor());
26342635
addOperandToPhiInVPIRBasicBlock(ScalarLoopHeader, OrigPhi, ResumePhiRecipe);
26352636
InductionBypassValues[OrigPhi] = {AdditionalBypass.first,
26362637
EndValueFromAdditionalBypass};

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1052,9 +1052,14 @@ void VPlan::execute(VPTransformState *State) {
10521052
"middle block has unexpected successors");
10531053
VPBasicBlock *ScalarPhVPBB = cast<VPBasicBlock>(
10541054
MiddleSuccs.size() == 1 ? MiddleSuccs[0] : MiddleSuccs[1]);
1055-
assert(!isa<VPIRBasicBlock>(ScalarPhVPBB) &&
1056-
"scalar preheader cannot be wrapped already");
1057-
replaceVPBBWithIRVPBB(ScalarPhVPBB, ScalarPh);
1055+
if (!isa<VPIRBasicBlock>(ScalarPhVPBB)) {
1056+
replaceVPBBWithIRVPBB(ScalarPhVPBB, ScalarPh);
1057+
// Only disconnect ScalarPh and the original loop header if ScalarPH is
1058+
// still present in VPlan. In that case, the branch and the DT update will
1059+
// be (re-)created during VPlan execution.
1060+
State->CFG.DTU.applyUpdates(
1061+
{{DominatorTree::Delete, ScalarPh, ScalarPh->getSingleSuccessor()}});
1062+
}
10581063
replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB);
10591064

10601065
// Disconnect the middle block from its single successor (the scalar loop
@@ -1064,8 +1069,6 @@ void VPlan::execute(VPTransformState *State) {
10641069
BrInst->insertBefore(MiddleBB->getTerminator());
10651070
MiddleBB->getTerminator()->eraseFromParent();
10661071
State->CFG.DTU.applyUpdates({{DominatorTree::Delete, MiddleBB, ScalarPh}});
1067-
State->CFG.DTU.applyUpdates(
1068-
{{DominatorTree::Delete, ScalarPh, ScalarPh->getSingleSuccessor()}});
10691072

10701073
// Generate code in the loop pre-header and body.
10711074
for (VPBlockBase *Block : vp_depth_first_shallow(Entry))

‎llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ inline specific_intval<0> m_SpecificInt(uint64_t V) {
8585
return specific_intval<0>(APInt(64, V));
8686
}
8787

88+
inline specific_intval<1> m_True() { return specific_intval<1>(APInt(64, 1)); }
8889
inline specific_intval<1> m_False() { return specific_intval<1>(APInt(64, 0)); }
8990

9091
/// Matching combinators

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,6 +1152,44 @@ void VPlanTransforms::truncateToMinimalBitwidths(
11521152
"some entries in MinBWs haven't been processed");
11531153
}
11541154

1155+
/// Remove BranchOnCond recipes with constant conditions together with removing
1156+
/// dead edges to their successors. Remove blocks that become dead (no remaining
1157+
/// predecessors())
1158+
static void simplifyCFG(VPlan &Plan) {
1159+
using namespace llvm::VPlanPatternMatch;
1160+
// Start by collecting successor blocks known to not be taken from
1161+
// branch-on-cond terminator in their predecessor.
1162+
SmallVector<VPBlockBase *> WorkList;
1163+
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1164+
vp_depth_first_deep(Plan.getEntry()))) {
1165+
VPRecipeBase *Term = VPBB->getTerminator();
1166+
if (!Term || !match(Term, m_BranchOnCond(m_True())))
1167+
continue;
1168+
VPBasicBlock *DeadSucc = cast<VPBasicBlock>(VPBB->getSuccessors()[1]);
1169+
VPBlockUtils::disconnectBlocks(VPBB, DeadSucc);
1170+
Term->eraseFromParent();
1171+
WorkList.push_back(DeadSucc);
1172+
}
1173+
1174+
VPValue Tmp;
1175+
// Remove any block in the worklist if it doesn't have any predecessors left.
1176+
// Disconnect the block from all successors and queue them for removal.
1177+
while (!WorkList.empty()) {
1178+
VPBlockBase *VPBB = WorkList.pop_back_val();
1179+
if (VPBB->getNumPredecessors() != 0)
1180+
continue;
1181+
for (VPBlockBase *Succ : to_vector(VPBB->getSuccessors())) {
1182+
VPBlockUtils::disconnectBlocks(VPBB, Succ);
1183+
WorkList.push_back(Succ);
1184+
}
1185+
// Replace all uses of values defined in VPB with a dummy VPValue to
1186+
// facilitate removal. All blocks with remaining uses of the dummy value
1187+
// must be removed by subsequent operation.
1188+
VPBB->dropAllReferences(&Tmp);
1189+
delete VPBB;
1190+
}
1191+
}
1192+
11551193
void VPlanTransforms::optimize(VPlan &Plan) {
11561194
removeRedundantCanonicalIVs(Plan);
11571195
removeRedundantInductionCasts(Plan);
@@ -1164,6 +1202,7 @@ void VPlanTransforms::optimize(VPlan &Plan) {
11641202

11651203
removeRedundantExpandSCEVRecipes(Plan);
11661204
mergeBlocksIntoPredecessors(Plan);
1205+
simplifyCFG(Plan);
11671206
licm(Plan);
11681207
}
11691208

‎llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,13 @@ define i32 @test_phi_iterator_invalidation(ptr %A, ptr noalias %B) {
7171
; CHECK-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
7272
; CHECK: middle.block:
7373
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP24]], i32 3
74-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
74+
; CHECK-NEXT: br label [[EXIT:%.*]]
7575
; CHECK: scalar.ph:
76-
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
77-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1004, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
76+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
7877
; CHECK-NEXT: br label [[LOOP:%.*]]
7978
; CHECK: loop:
8079
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
81-
; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
80+
; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
8281
; CHECK-NEXT: [[SEXT:%.*]] = sext i16 [[SCALAR_RECUR]] to i32
8382
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
8483
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT]]

‎llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,10 @@ define i32 @pr70988(ptr %src, i32 %n) {
5454
; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
5555
; CHECK: middle.block:
5656
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP17]], i32 [[TMP18]])
57-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
57+
; CHECK-NEXT: br label [[EXIT:%.*]]
5858
; CHECK: scalar.ph:
59-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
60-
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_MINMAX]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
59+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
60+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
6161
; CHECK-NEXT: br label [[LOOP:%.*]]
6262
; CHECK: loop:
6363
; CHECK-NEXT: [[INDUC:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDUC_NEXT:%.*]], [[LOOP]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ define void @trip1025_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapt
3737
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <vscale x 2 x i1> [[TMP14]], i32 0
3838
; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3939
; CHECK: middle.block:
40-
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
40+
; CHECK-NEXT: br label [[FOR_END:%.*]]
4141
; CHECK: scalar.ph:
42-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
42+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
4343
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
4444
; CHECK: for.body:
4545
; CHECK-NEXT: [[I_06:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ define void @test(ptr %p, i64 %a, i8 %b) {
5858
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
5959
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
6060
; CHECK: middle.block:
61-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
61+
; CHECK-NEXT: br label [[EXIT:%.*]]
6262
; CHECK: scalar.ph:
63-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 4, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
63+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
6464
; CHECK-NEXT: br label [[FOR_COND:%.*]]
6565
; CHECK: for.cond:
6666
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY:%.*]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ define void @small_trip_count_min_vlen_128(ptr nocapture %a) nounwind vscale_ran
2525
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP2]]
2626
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
2727
; CHECK: middle.block:
28-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
28+
; CHECK-NEXT: br label [[EXIT:%.*]]
2929
; CHECK: scalar.ph:
30-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
30+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
3131
; CHECK-NEXT: br label [[LOOP:%.*]]
3232
; CHECK: loop:
3333
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
@@ -84,9 +84,9 @@ define void @small_trip_count_min_vlen_32(ptr nocapture %a) nounwind vscale_rang
8484
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]]
8585
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
8686
; CHECK: middle.block:
87-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
87+
; CHECK-NEXT: br label [[EXIT:%.*]]
8888
; CHECK: scalar.ph:
89-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
89+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
9090
; CHECK-NEXT: br label [[LOOP:%.*]]
9191
; CHECK: loop:
9292
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,9 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
6767
; IF-EVL-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
6868
; IF-EVL-NEXT: br i1 [[TMP33]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
6969
; IF-EVL: middle.block:
70-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
70+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
7171
; IF-EVL: scalar.ph:
72-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
72+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
7373
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
7474
; IF-EVL: for.body:
7575
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
4444
; IF-EVL-NEXT: [[TMP18:%.*]] = icmp eq i32 [[IV_NEXT]], [[N_VEC]]
4545
; IF-EVL-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4646
; IF-EVL: middle.block:
47-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
47+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
4848
; IF-EVL: scalar.ph:
49-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
49+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY1:%.*]] ]
5050
; IF-EVL-NEXT: br label [[FOR_BODY1:%.*]]
5151
; IF-EVL: for.body:
5252
; IF-EVL-NEXT: [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
5858
; IF-EVL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
5959
; IF-EVL-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
6060
; IF-EVL: middle.block:
61-
; IF-EVL-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
61+
; IF-EVL-NEXT: br label [[EXIT:%.*]]
6262
; IF-EVL: scalar.ph:
63-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
63+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
6464
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
6565
; IF-EVL: for.body:
6666
; IF-EVL-NEXT: [[I_011:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@ define float @fadd(ptr noalias nocapture readonly %a, i64 %n) {
4646
; IF-EVL-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4747
; IF-EVL-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4848
; IF-EVL: middle.block:
49-
; IF-EVL-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
49+
; IF-EVL-NEXT: br label [[FOR_END:%.*]]
5050
; IF-EVL: scalar.ph:
51-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
52-
; IF-EVL-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
51+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
52+
; IF-EVL-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
5353
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
5454
; IF-EVL: for.body:
5555
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
4949
; IF-EVL-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
5050
; IF-EVL-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
5151
; IF-EVL: middle.block:
52-
; IF-EVL-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
52+
; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
5353
; IF-EVL: scalar.ph:
54-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
54+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
5555
; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]
5656
; IF-EVL: for.body:
5757
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ define void @test_scalar_steps_target_instruction_cost(ptr %dst) {
3737
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10
3838
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3939
; CHECK: [[MIDDLE_BLOCK]]:
40-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
40+
; CHECK-NEXT: br label %[[EXIT:.*]]
4141
; CHECK: [[SCALAR_PH]]:
42-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 30, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
42+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
4343
; CHECK-NEXT: br label %[[LOOP:.*]]
4444
; CHECK: [[LOOP]]:
4545
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]

‎llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,9 @@ define void @test(ptr %p, i40 %a) {
139139
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 16
140140
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
141141
; CHECK: middle.block:
142-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
142+
; CHECK-NEXT: br label [[EXIT:%.*]]
143143
; CHECK: scalar.ph:
144-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
144+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
145145
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
146146
; CHECK: for.body:
147147
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
4444
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4545
; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4646
; CHECK: [[MIDDLE_BLOCK]]:
47-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
47+
; CHECK-NEXT: br label %[[EXIT:.*]]
4848
; CHECK: [[SCALAR_PH]]:
49-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
49+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
5050
; CHECK-NEXT: br label %[[LOOP:.*]]
5151
; CHECK: [[LOOP]]:
5252
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -160,9 +160,9 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
160160
; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
161161
; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
162162
; CHECK: [[MIDDLE_BLOCK]]:
163-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
163+
; CHECK-NEXT: br label %[[EXIT:.*]]
164164
; CHECK: [[SCALAR_PH]]:
165-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
165+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
166166
; CHECK-NEXT: br label %[[LOOP:.*]]
167167
; CHECK: [[LOOP]]:
168168
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]

‎llvm/test/Transforms/LoopVectorize/X86/fp64_to_uint32-cost-model.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ for.body: ; preds = %for.body.preheader,
2929
store i32 %conv, ptr %arrayidx2, align 4
3030
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
3131
%2 = trunc i64 %indvars.iv.next to i32
32-
%cmp = icmp ult i32 %2, %0
32+
%cmp = icmp ult i32 %2, 1000
3333
br i1 %cmp, label %for.body, label %for.end.loopexit
3434

3535
for.end.loopexit: ; preds = %for.body

‎llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,9 +127,9 @@ define void @vectorized1(ptr noalias nocapture %A, ptr noalias nocapture readonl
127127
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
128128
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
129129
; CHECK: middle.block:
130-
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
130+
; CHECK-NEXT: br label [[FOR_END:%.*]]
131131
; CHECK: scalar.ph:
132-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 24, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
132+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
133133
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
134134
; CHECK: for.body:
135135
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]

0 commit comments

Comments
 (0)