Skip to content

Commit 19f1666

Browse files
fhahnfhahn
committed
!fixup update after recent changes.
1 parent 195677a commit 19f1666

8 files changed

+75
-60
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2502,13 +2502,13 @@ void InnerLoopVectorizer::introduceCheckBlockInVPlan(BasicBlock *CheckIRBB) {
25022502
PreVectorPH->swapSuccessors();
25032503

25042504
// We just connected a new block to the scalar preheader. Update all
2505-
// ResumePhis by adding an incoming value for it.
2505+
// ResumePhis by adding an incoming value for it, replacing the last value.
25062506
for (VPRecipeBase &R : *cast<VPBasicBlock>(ScalarPH)) {
25072507
auto *ResumePhi = dyn_cast<VPInstruction>(&R);
25082508
if (!ResumePhi || ResumePhi->getOpcode() != VPInstruction::ResumePhi)
25092509
continue;
25102510
ResumePhi->addOperand(
2511-
ResumePhi->getOperand(ScalarPH->getNumPredecessors() == 1 ? 0 : 1));
2511+
ResumePhi->getOperand(ResumePhi->getNumOperands() - 1));
25122512
}
25132513
}
25142514

@@ -7570,9 +7570,8 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
75707570
// properly model costs for such loops.
75717571
auto ExitBlocks = BestPlan.getExitBlocks();
75727572
bool PlanForEarlyExitLoop =
7573-
std::distance(ExitBlocks.begin(), ExitBlocks.end()) > 2 ||
7574-
(std::distance(ExitBlocks.begin(), ExitBlocks.end()) == 1 &&
7575-
(*ExitBlocks.begin())->getNumPredecessors() > 1);
7573+
ExitBlocks.size() > 1 ||
7574+
(ExitBlocks.size() == 1 && ExitBlocks[0]->getNumPredecessors() > 1);
75767575
assert((BestFactor.Width == LegacyVF.Width || PlanForEarlyExitLoop ||
75777576
planContainsAdditionalSimplifications(getPlanFor(BestFactor.Width),
75787577
CostCtx, OrigLoop) ||

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3529,12 +3529,18 @@ class VPlan {
35293529

35303530
/// Returns the 'middle' block of the plan, that is the block that selects
35313531
/// whether to execute the scalar tail loop or the exit block from the loop
3532-
/// latch.
3533-
const VPBasicBlock *getMiddleBlock() const {
3534-
return cast<VPBasicBlock>(getScalarPreheader()->getPredecessors().front());
3535-
}
3532+
/// latch. If the scalar tail loop or exit block are known to always execute,
3533+
/// the middle block may branch directly to the block.
35363534
VPBasicBlock *getMiddleBlock() {
3537-
return cast<VPBasicBlock>(getScalarPreheader()->getPredecessors().front());
3535+
if (!getScalarPreheader()->getPredecessors().empty())
3536+
return cast<VPBasicBlock>(
3537+
getScalarPreheader()->getPredecessors().front());
3538+
if (getExitBlocks().size() == 1)
3539+
return cast<VPBasicBlock>(getExitBlocks()[0]->getPredecessors().front());
3540+
return nullptr;
3541+
}
3542+
const VPBasicBlock *getMiddleBlock() const {
3543+
return const_cast<VPlan *>(this)->getMiddleBlock();
35383544
}
35393545

35403546
/// Return the VPBasicBlock for the preheader of the scalar loop.

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1467,45 +1467,45 @@ void VPlanTransforms::truncateToMinimalBitwidths(
14671467
"some entries in MinBWs haven't been processed");
14681468
}
14691469

1470-
/// Remove BranchOnCond recipes with constant conditions together with removing
1470+
/// Remove BranchOnCond recipes with true conditions together with removing
14711471
/// dead edges to their successors.
14721472
static void simplifyCFG(VPlan &Plan) {
14731473
using namespace llvm::VPlanPatternMatch;
14741474
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
14751475
vp_depth_first_deep(Plan.getEntry()))) {
1476-
if (VPBB->getNumSuccessors() != 2 || VPBB->begin() == VPBB->end() ||
1476+
if (VPBB->getNumSuccessors() != 2 ||
14771477
!match(&VPBB->back(), m_BranchOnCond(m_True())))
14781478
continue;
14791479

1480-
VPBasicBlock *DeadSucc = cast<VPBasicBlock>(VPBB->getSuccessors()[1]);
1481-
const auto &Preds = DeadSucc->getPredecessors();
1480+
VPBasicBlock *RemovedSucc = cast<VPBasicBlock>(VPBB->getSuccessors()[1]);
1481+
const auto &Preds = RemovedSucc->getPredecessors();
14821482
unsigned DeadIdx = std::distance(Preds.begin(), find(Preds, VPBB));
14831483

1484-
// Remove values coming from VPBB from phi-like recipes in DeadSucc.
1485-
for (VPRecipeBase &R : make_early_inc_range(*DeadSucc)) {
1484+
// Remove values coming from VPBB from phi-like recipes in RemovedSucc.
1485+
for (VPRecipeBase &R : make_early_inc_range(*RemovedSucc)) {
14861486
assert((!isa<VPIRInstruction>(&R) ||
14871487
!isa<PHINode>(cast<VPIRInstruction>(&R)->getInstruction())) &&
14881488
!isa<VPHeaderPHIRecipe>(&R) &&
14891489
"Cannot update VPIRInstructions wrapping phis or header phis yet");
14901490
auto *VPI = dyn_cast<VPInstruction>(&R);
1491-
if (VPI && VPI->getOpcode() == VPInstruction::ResumePhi) {
1492-
VPBuilder B(VPI);
1493-
SmallVector<VPValue *> NewOps;
1494-
// Create new operand list, with the dead incoming value filtered out.
1495-
for (const auto &[Idx, Op] : enumerate(VPI->operands())) {
1496-
if (Idx == DeadIdx)
1497-
continue;
1498-
NewOps.push_back(Op);
1499-
}
1500-
VPI->replaceAllUsesWith(B.createNaryOp(VPInstruction::ResumePhi, NewOps,
1501-
VPI->getDebugLoc(),
1502-
VPI->getName()));
1503-
VPI->eraseFromParent();
1491+
if (!VPI || VPI->getOpcode() != VPInstruction::ResumePhi)
1492+
break;
1493+
VPBuilder B(VPI);
1494+
SmallVector<VPValue *> NewOps;
1495+
// Create new operand list, with the dead incoming value filtered out.
1496+
for (const auto &[Idx, Op] : enumerate(VPI->operands())) {
1497+
if (Idx == DeadIdx)
1498+
continue;
1499+
NewOps.push_back(Op);
15041500
}
1501+
VPI->replaceAllUsesWith(B.createNaryOp(VPInstruction::ResumePhi, NewOps,
1502+
VPI->getDebugLoc(),
1503+
VPI->getName()));
1504+
VPI->eraseFromParent();
15051505
}
1506-
// Disconnect blocks and remove the terminator. DeadSucc will be deleted
1507-
// automatically on VPlan descruction.
1508-
VPBlockUtils::disconnectBlocks(VPBB, DeadSucc);
1506+
// Disconnect blocks and remove the terminator. RemovedSucc will be deleted
1507+
// automatically on VPlan destruction.
1508+
VPBlockUtils::disconnectBlocks(VPBB, RemovedSucc);
15091509
VPBB->back().eraseFromParent();
15101510
}
15111511
}
@@ -1519,7 +1519,7 @@ void VPlanTransforms::optimize(VPlan &Plan) {
15191519
runPass(legalizeAndOptimizeInductions, Plan);
15201520
runPass(removeRedundantExpandSCEVRecipes, Plan);
15211521
runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType());
1522-
simplifyCFG(Plan);
1522+
runPass(simplifyCFG, Plan);
15231523
runPass(removeDeadRecipes, Plan);
15241524

15251525
runPass(createAndOptimizeReplicateRegions, Plan);

‎llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -404,9 +404,9 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
404404
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
405405
; DEFAULT-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
406406
; DEFAULT: [[MIDDLE_BLOCK]]:
407-
; DEFAULT-NEXT: br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
407+
; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
408408
; DEFAULT: [[SCALAR_PH]]:
409-
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
409+
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
410410
; DEFAULT-NEXT: br label %[[FOR_BODY:.*]]
411411
; DEFAULT: [[FOR_BODY]]:
412412
; DEFAULT-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -599,9 +599,9 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
599599
; OPTSIZE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
600600
; OPTSIZE-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
601601
; OPTSIZE: [[MIDDLE_BLOCK]]:
602-
; OPTSIZE-NEXT: br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
602+
; OPTSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
603603
; OPTSIZE: [[SCALAR_PH]]:
604-
; OPTSIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
604+
; OPTSIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
605605
; OPTSIZE-NEXT: br label %[[FOR_BODY:.*]]
606606
; OPTSIZE: [[FOR_BODY]]:
607607
; OPTSIZE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -728,9 +728,9 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
728728
; DEFAULT-NEXT: [[TMP25:%.*]] = extractelement <vscale x 16 x i1> [[TMP24]], i32 0
729729
; DEFAULT-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
730730
; DEFAULT: [[MIDDLE_BLOCK]]:
731-
; DEFAULT-NEXT: br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
731+
; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
732732
; DEFAULT: [[SCALAR_PH]]:
733-
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
733+
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
734734
; DEFAULT-NEXT: br label %[[FOR_BODY:.*]]
735735
; DEFAULT: [[FOR_BODY]]:
736736
; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -805,9 +805,9 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
805805
; OPTSIZE-NEXT: [[TMP25:%.*]] = extractelement <vscale x 16 x i1> [[TMP24]], i32 0
806806
; OPTSIZE-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
807807
; OPTSIZE: [[MIDDLE_BLOCK]]:
808-
; OPTSIZE-NEXT: br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
808+
; OPTSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
809809
; OPTSIZE: [[SCALAR_PH]]:
810-
; OPTSIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
810+
; OPTSIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
811811
; OPTSIZE-NEXT: br label %[[FOR_BODY:.*]]
812812
; OPTSIZE: [[FOR_BODY]]:
813813
; OPTSIZE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -882,9 +882,9 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
882882
; MINSIZE-NEXT: [[TMP25:%.*]] = extractelement <vscale x 16 x i1> [[TMP24]], i32 0
883883
; MINSIZE-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
884884
; MINSIZE: [[MIDDLE_BLOCK]]:
885-
; MINSIZE-NEXT: br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
885+
; MINSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
886886
; MINSIZE: [[SCALAR_PH]]:
887-
; MINSIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
887+
; MINSIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
888888
; MINSIZE-NEXT: br label %[[FOR_BODY:.*]]
889889
; MINSIZE: [[FOR_BODY]]:
890890
; MINSIZE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -400,9 +400,9 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
400400
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
401401
; DEFAULT-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
402402
; DEFAULT: [[MIDDLE_BLOCK]]:
403-
; DEFAULT-NEXT: br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
403+
; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
404404
; DEFAULT: [[SCALAR_PH]]:
405-
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
405+
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
406406
; DEFAULT-NEXT: br label %[[FOR_BODY:.*]]
407407
; DEFAULT: [[FOR_BODY]]:
408408
; DEFAULT-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -595,9 +595,9 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
595595
; OPTSIZE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
596596
; OPTSIZE-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
597597
; OPTSIZE: [[MIDDLE_BLOCK]]:
598-
; OPTSIZE-NEXT: br i1 true, label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
598+
; OPTSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
599599
; OPTSIZE: [[SCALAR_PH]]:
600-
; OPTSIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
600+
; OPTSIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
601601
; OPTSIZE-NEXT: br label %[[FOR_BODY:.*]]
602602
; OPTSIZE: [[FOR_BODY]]:
603603
; OPTSIZE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-div.ll

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,9 @@ define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
4747
; IF-EVL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4848
; IF-EVL-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4949
; IF-EVL: [[MIDDLE_BLOCK]]:
50-
; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
50+
; IF-EVL-NEXT: br label %[[EXIT:.*]]
5151
; IF-EVL: [[SCALAR_PH]]:
52-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
52+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
5353
; IF-EVL-NEXT: br label %[[LOOP:.*]]
5454
; IF-EVL: [[LOOP]]:
5555
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -144,9 +144,9 @@ define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
144144
; IF-EVL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
145145
; IF-EVL-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
146146
; IF-EVL: [[MIDDLE_BLOCK]]:
147-
; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
147+
; IF-EVL-NEXT: br label %[[EXIT:.*]]
148148
; IF-EVL: [[SCALAR_PH]]:
149-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
149+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
150150
; IF-EVL-NEXT: br label %[[LOOP:.*]]
151151
; IF-EVL: [[LOOP]]:
152152
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -240,9 +240,9 @@ define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
240240
; IF-EVL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
241241
; IF-EVL-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
242242
; IF-EVL: [[MIDDLE_BLOCK]]:
243-
; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
243+
; IF-EVL-NEXT: br label %[[EXIT:.*]]
244244
; IF-EVL: [[SCALAR_PH]]:
245-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
245+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
246246
; IF-EVL-NEXT: br label %[[LOOP:.*]]
247247
; IF-EVL: [[LOOP]]:
248248
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -336,9 +336,9 @@ define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
336336
; IF-EVL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
337337
; IF-EVL-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
338338
; IF-EVL: [[MIDDLE_BLOCK]]:
339-
; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
339+
; IF-EVL-NEXT: br label %[[EXIT:.*]]
340340
; IF-EVL: [[SCALAR_PH]]:
341-
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]
341+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
342342
; IF-EVL-NEXT: br label %[[LOOP:.*]]
343343
; IF-EVL: [[LOOP]]:
344344
; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]

‎llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,14 @@ define void @second_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
208208
; IF-EVL: [[MIDDLE_BLOCK]]:
209209
; IF-EVL-NEXT: br label %[[FOR_END:.*]]
210210
; IF-EVL: [[SCALAR_PH]]:
211+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
212+
; IF-EVL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 33, %[[ENTRY]] ]
213+
; IF-EVL-NEXT: [[SCALAR_RECUR_INIT3:%.*]] = phi i32 [ 22, %[[ENTRY]] ]
214+
; IF-EVL-NEXT: br label %[[FOR_BODY:.*]]
215+
; IF-EVL: [[FOR_BODY]]:
216+
; IF-EVL-NEXT: [[INDVARS:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], %[[FOR_BODY]] ]
217+
; IF-EVL-NEXT: [[FOR1:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP31:%.*]], %[[FOR_BODY]] ]
218+
; IF-EVL-NEXT: [[FOR2:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT3]], %[[SCALAR_PH]] ], [ [[FOR1]], %[[FOR_BODY]] ]
211219
; IF-EVL-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS]]
212220
; IF-EVL-NEXT: [[TMP31]] = load i32, ptr [[ARRAYIDX]], align 4
213221
; IF-EVL-NEXT: [[ADD:%.*]] = add nsw i32 [[FOR1]], [[FOR2]]
@@ -371,12 +379,16 @@ define void @third_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
371379
; IF-EVL: [[MIDDLE_BLOCK]]:
372380
; IF-EVL-NEXT: br label %[[FOR_END:.*]]
373381
; IF-EVL: [[SCALAR_PH]]:
382+
; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
383+
; IF-EVL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 33, %[[ENTRY]] ]
384+
; IF-EVL-NEXT: [[SCALAR_RECUR_INIT5:%.*]] = phi i32 [ 22, %[[ENTRY]] ]
385+
; IF-EVL-NEXT: [[SCALAR_RECUR_INIT6:%.*]] = phi i32 [ 11, %[[ENTRY]] ]
374386
; IF-EVL-NEXT: br label %[[FOR_BODY:.*]]
375387
; IF-EVL: [[FOR_BODY]]:
376388
; IF-EVL-NEXT: [[INDVARS:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], %[[FOR_BODY]] ]
377389
; IF-EVL-NEXT: [[FOR1:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP38:%.*]], %[[FOR_BODY]] ]
378-
; IF-EVL-NEXT: [[FOR2:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT7]], %[[SCALAR_PH]] ], [ [[FOR1]], %[[FOR_BODY]] ]
379-
; IF-EVL-NEXT: [[FOR3:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT8]], %[[SCALAR_PH]] ], [ [[FOR2]], %[[FOR_BODY]] ]
390+
; IF-EVL-NEXT: [[FOR2:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT5]], %[[SCALAR_PH]] ], [ [[FOR1]], %[[FOR_BODY]] ]
391+
; IF-EVL-NEXT: [[FOR3:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT6]], %[[SCALAR_PH]] ], [ [[FOR2]], %[[FOR_BODY]] ]
380392
; IF-EVL-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS]]
381393
; IF-EVL-NEXT: [[TMP38]] = load i32, ptr [[ARRAYIDX]], align 4
382394
; IF-EVL-NEXT: [[ADD:%.*]] = add nsw i32 [[FOR2]], [[FOR3]]

‎llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,7 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
4848
; IF-EVL-NEXT: Successor(s): middle.block
4949
; IF-EVL-EMPTY:
5050
; IF-EVL: middle.block:
51-
; IF-EVL-NEXT: EMIT vp<[[RESUME_EXTRACT:%.+]]> = extract-from-end ir<[[LD]]>, ir<1>
52-
; IF-EVL-NEXT: EMIT branch-on-cond ir<true>
53-
; IF-EVL-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
51+
; IF-EVL-NEXT: Successor(s): ir-bb<for.end>
5452

5553
entry:
5654
br label %for.body

0 commit comments

Comments
 (0)