Skip to content

Commit 232fac0

Browse files
fhahnfhahn
committed
[VPlan] Introduce ExitPhi VPInstruction, use to create phi for FOR.
This patch introduces a new ExitPhi VPInstruction which creates a phi in a leaf block of a VPlan. The first use is to create the phi node for fixed-order recurrence resume values in the scalar preheader. The VPInstruction takes 2 operands: 1) the incoming value from the middle-block and a default value to be used for all other incoming blocks. In follow-up changes, it will also be used to create phis for reduction and induction resume values.
1 parent e1c1bd1 commit 232fac0

26 files changed

+336
-265
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 46 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -607,10 +607,6 @@ class InnerLoopVectorizer {
607607
BasicBlock *MiddleBlock, BasicBlock *VectorHeader,
608608
VPlan &Plan, VPTransformState &State);
609609

610-
/// Create the phi node for the resume value of first order recurrences in the
611-
/// scalar preheader and update the users in the scalar loop.
612-
void fixFixedOrderRecurrence(VPLiveOut *LO, VPTransformState &State);
613-
614610
/// Iteratively sink the scalarized operands of a predicated instruction into
615611
/// the block that was created for it.
616612
void sinkScalarOperands(Instruction *PredInst);
@@ -3339,8 +3335,6 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
33393335
for (const auto &[_, LO] : to_vector(Plan.getLiveOuts())) {
33403336
if (!Legal->isFixedOrderRecurrence(LO->getPhi()))
33413337
continue;
3342-
fixFixedOrderRecurrence(LO, State);
3343-
Plan.removeLiveOut(LO->getPhi());
33443338
}
33453339

33463340
// Forget the original basic block.
@@ -3422,35 +3416,6 @@ static void reorderIncomingBlocks(SmallVectorImpl<BasicBlock *> &Blocks,
34223416
std::swap(Blocks[0], Blocks[1]);
34233417
}
34243418

3425-
void InnerLoopVectorizer::fixFixedOrderRecurrence(VPLiveOut *LO,
3426-
VPTransformState &State) {
3427-
// Extract the last vector element in the middle block. This will be the
3428-
// initial value for the recurrence when jumping to the scalar loop.
3429-
VPValue *VPExtract = LO->getOperand(0);
3430-
using namespace llvm::VPlanPatternMatch;
3431-
assert(match(VPExtract, m_VPInstruction<VPInstruction::ExtractFromEnd>(
3432-
m_VPValue(), m_VPValue())) &&
3433-
"FOR LiveOut expects to use an extract from end.");
3434-
Value *ResumeScalarFOR = State.get(VPExtract, UF - 1, true);
3435-
3436-
// Fix the initial value of the original recurrence in the scalar loop.
3437-
PHINode *ScalarHeaderPhi = LO->getPhi();
3438-
auto *InitScalarFOR =
3439-
ScalarHeaderPhi->getIncomingValueForBlock(LoopScalarPreHeader);
3440-
Builder.SetInsertPoint(LoopScalarPreHeader, LoopScalarPreHeader->begin());
3441-
auto *ScalarPreheaderPhi =
3442-
Builder.CreatePHI(ScalarHeaderPhi->getType(), 2, "scalar.recur.init");
3443-
SmallVector<BasicBlock *> Blocks(predecessors(LoopScalarPreHeader));
3444-
reorderIncomingBlocks(Blocks, LoopMiddleBlock);
3445-
for (auto *BB : Blocks) {
3446-
auto *Incoming = BB == LoopMiddleBlock ? ResumeScalarFOR : InitScalarFOR;
3447-
ScalarPreheaderPhi->addIncoming(Incoming, BB);
3448-
}
3449-
ScalarHeaderPhi->setIncomingValueForBlock(LoopScalarPreHeader,
3450-
ScalarPreheaderPhi);
3451-
ScalarHeaderPhi->setName("scalar.recur");
3452-
}
3453-
34543419
void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {
34553420
// The basic block and loop containing the predicated instruction.
34563421
auto *PredBB = PredInst->getParent();
@@ -8501,7 +8466,9 @@ static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop,
85018466
Value *IncomingValue =
85028467
ExitPhi.getIncomingValueForBlock(ExitingBB);
85038468
VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue, Plan);
8504-
Plan.addLiveOut(&ExitPhi, V);
8469+
Plan.addLiveOut(
8470+
&ExitPhi, V,
8471+
cast<VPBasicBlock>(Plan.getVectorLoopRegion()->getSingleSuccessor()));
85058472
}
85068473
}
85078474

@@ -8681,6 +8648,49 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
86818648
"VPBasicBlock");
86828649
RecipeBuilder.fixHeaderPhis();
86838650

8651+
auto *MiddleVPBB =
8652+
cast<VPBasicBlock>(Plan->getVectorLoopRegion()->getSingleSuccessor());
8653+
8654+
VPBasicBlock *ScalarPH = nullptr;
8655+
for (VPBlockBase *Succ : MiddleVPBB->getSuccessors()) {
8656+
auto *VPIRBB = dyn_cast<VPIRBasicBlock>(Succ);
8657+
if (VPIRBB && VPIRBB->getIRBasicBlock() == OrigLoop->getHeader()) {
8658+
ScalarPH = VPIRBB;
8659+
break;
8660+
}
8661+
}
8662+
8663+
if (ScalarPH) {
8664+
for (auto &H : HeaderVPBB->phis()) {
8665+
auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&H);
8666+
if (!FOR)
8667+
continue;
8668+
VPBuilder B(ScalarPH);
8669+
VPBuilder MiddleBuilder;
8670+
// Set insert point so new recipes are inserted before terminator and
8671+
// condition, if there is either the former or both.
8672+
if (MiddleVPBB->getNumSuccessors() != 2)
8673+
MiddleBuilder.setInsertPoint(MiddleVPBB);
8674+
else if (isa<VPInstruction>(MiddleVPBB->getTerminator()->getOperand(0)))
8675+
MiddleBuilder.setInsertPoint(
8676+
&*std::prev(MiddleVPBB->getTerminator()->getIterator()));
8677+
else
8678+
MiddleBuilder.setInsertPoint(MiddleVPBB->getTerminator());
8679+
8680+
// Extract the resume value and create a new VPLiveOut for it.
8681+
auto *Resume = MiddleBuilder.createNaryOp(
8682+
VPInstruction::ExtractFromEnd,
8683+
{FOR->getBackedgeValue(),
8684+
Plan->getOrAddLiveIn(
8685+
ConstantInt::get(Plan->getCanonicalIV()->getScalarType(), 1))},
8686+
{}, "vector.recur.extract");
8687+
auto *R =
8688+
B.createNaryOp(VPInstruction::ExitPhi, {Resume, FOR->getStartValue()},
8689+
{}, "scalar.recur.init");
8690+
Plan->addLiveOut(cast<PHINode>(FOR->getUnderlyingInstr()), R, ScalarPH);
8691+
}
8692+
}
8693+
86848694
// ---------------------------------------------------------------------------
86858695
// Transform initial VPlan: Apply previously taken decisions, in order, to
86868696
// bring the VPlan to its final state.

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -446,9 +446,6 @@ void VPIRBasicBlock::execute(VPTransformState *State) {
446446
assert(getHierarchicalSuccessors().empty() &&
447447
"VPIRBasicBlock cannot have successors at the moment");
448448

449-
State->Builder.SetInsertPoint(getIRBasicBlock()->getTerminator());
450-
executeRecipes(State, getIRBasicBlock());
451-
452449
for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
453450
VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock();
454451
auto &PredVPSuccessors = PredVPBB->getHierarchicalSuccessors();
@@ -468,6 +465,9 @@ void VPIRBasicBlock::execute(VPTransformState *State) {
468465
TermBr->setSuccessor(idx, IRBB);
469466
}
470467
}
468+
469+
State->Builder.SetInsertPoint(getIRBasicBlock()->getTerminator());
470+
executeRecipes(State, getIRBasicBlock());
471471
}
472472

473473
void VPBasicBlock::execute(VPTransformState *State) {
@@ -1078,9 +1078,9 @@ LLVM_DUMP_METHOD
10781078
void VPlan::dump() const { print(dbgs()); }
10791079
#endif
10801080

1081-
void VPlan::addLiveOut(PHINode *PN, VPValue *V) {
1081+
void VPlan::addLiveOut(PHINode *PN, VPValue *V, VPBasicBlock *Pred) {
10821082
assert(LiveOuts.count(PN) == 0 && "an exit value for PN already exists");
1083-
LiveOuts.insert({PN, new VPLiveOut(PN, V)});
1083+
LiveOuts.insert({PN, new VPLiveOut(PN, V, Pred)});
10841084
}
10851085

10861086
static void remapOperands(VPBlockBase *Entry, VPBlockBase *NewEntry,
@@ -1149,9 +1149,18 @@ VPlan *VPlan::duplicate() {
11491149
remapOperands(Preheader, NewPreheader, Old2NewVPValues);
11501150
remapOperands(Entry, NewEntry, Old2NewVPValues);
11511151

1152+
DenseMap<VPBlockBase *, VPBlockBase *> Old2NewVPBlocks;
1153+
VPBlockBase *OldMiddle = getVectorLoopRegion()->getSingleSuccessor();
1154+
VPBlockBase *NewMiddle = NewPlan->getVectorLoopRegion()->getSingleSuccessor();
1155+
Old2NewVPBlocks[OldMiddle] = NewMiddle;
1156+
for (const auto &[Old, New] :
1157+
zip(OldMiddle->getSuccessors(), NewMiddle->getSuccessors()))
1158+
Old2NewVPBlocks[Old] = New;
1159+
11521160
// Clone live-outs.
11531161
for (const auto &[_, LO] : LiveOuts)
1154-
NewPlan->addLiveOut(LO->getPhi(), Old2NewVPValues[LO->getOperand(0)]);
1162+
NewPlan->addLiveOut(LO->getPhi(), Old2NewVPValues[LO->getOperand(0)],
1163+
cast<VPBasicBlock>(Old2NewVPBlocks[LO->getPred()]));
11551164

11561165
// Initialize remaining fields of cloned VPlan.
11571166
NewPlan->VFs = VFs;

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -683,9 +683,13 @@ class VPBlockBase {
683683
class VPLiveOut : public VPUser {
684684
PHINode *Phi;
685685

686+
/// Predecessor in VPlan of this live-out. Used to as block to set the
687+
/// incoming value for.
688+
VPBasicBlock *Pred;
689+
686690
public:
687-
VPLiveOut(PHINode *Phi, VPValue *Op)
688-
: VPUser({Op}, VPUser::VPUserID::LiveOut), Phi(Phi) {}
691+
VPLiveOut(PHINode *Phi, VPValue *Op, VPBasicBlock *Pred)
692+
: VPUser({Op}, VPUser::VPUserID::LiveOut), Phi(Phi), Pred(Pred) {}
689693

690694
static inline bool classof(const VPUser *U) {
691695
return U->getVPUserID() == VPUser::VPUserID::LiveOut;
@@ -707,6 +711,9 @@ class VPLiveOut : public VPUser {
707711

708712
PHINode *getPhi() const { return Phi; }
709713

714+
/// Returns to incoming block for which to set the value.
715+
VPBasicBlock *getPred() const { return Pred; }
716+
710717
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
711718
/// Print the VPLiveOut to \p O.
712719
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const;
@@ -1188,6 +1195,10 @@ class VPInstruction : public VPRecipeWithIRFlags {
11881195
SLPStore,
11891196
ActiveLaneMask,
11901197
ExplicitVectorLength,
1198+
/// Creates a scalar phi in a leaf VPBB with a single predecessor in VPlan.
1199+
/// The first operand is the incoming value from the predecessor in VPlan,
1200+
/// the second operand is the incoming value for all other predecessors.
1201+
ExitPhi,
11911202
CalculateTripCountMinusVF,
11921203
// Increment the canonical IV separately for each unrolled part.
11931204
CanonicalIVIncrementForPart,
@@ -3338,7 +3349,7 @@ class VPlan {
33383349
return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
33393350
}
33403351

3341-
void addLiveOut(PHINode *PN, VPValue *V);
3352+
void addLiveOut(PHINode *PN, VPValue *V, VPBasicBlock *Pred);
33423353

33433354
void removeLiveOut(PHINode *PN) {
33443355
delete LiveOuts[PN];

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -192,11 +192,12 @@ void VPLiveOut::fixPhi(VPlan &Plan, VPTransformState &State) {
192192
VPValue *ExitValue = getOperand(0);
193193
if (vputils::isUniformAfterVectorization(ExitValue))
194194
Lane = VPLane::getFirstLane();
195-
VPBasicBlock *MiddleVPBB =
196-
cast<VPBasicBlock>(Plan.getVectorLoopRegion()->getSingleSuccessor());
197-
BasicBlock *MiddleBB = State.CFG.VPBB2IRBB[MiddleVPBB];
198-
Phi->addIncoming(State.get(ExitValue, VPIteration(State.UF - 1, Lane)),
199-
MiddleBB);
195+
BasicBlock *PredBB = State.CFG.VPBB2IRBB[Pred];
196+
Value *V = State.get(ExitValue, VPIteration(State.UF - 1, Lane));
197+
if (Phi->getBasicBlockIndex(PredBB) != -1)
198+
Phi->setIncomingValueForBlock(PredBB, V);
199+
else
200+
Phi->addIncoming(V, PredBB);
200201
}
201202

202203
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -302,6 +303,7 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
302303
case VPInstruction::CanonicalIVIncrementForPart:
303304
case VPInstruction::PtrAdd:
304305
case VPInstruction::ExplicitVectorLength:
306+
case VPInstruction::ExitPhi:
305307
return true;
306308
default:
307309
return false;
@@ -318,6 +320,14 @@ Value *VPInstruction::generatePerLane(VPTransformState &State,
318320
State.get(getOperand(1), Lane), Name);
319321
}
320322

323+
static void reorderIncomingBlocks(SmallVectorImpl<BasicBlock *> &Blocks,
324+
BasicBlock *LoopMiddleBlock) {
325+
if (Blocks.front() == LoopMiddleBlock)
326+
std::swap(Blocks.front(), Blocks.back());
327+
if (Blocks.size() == 3)
328+
std::swap(Blocks[0], Blocks[1]);
329+
}
330+
321331
Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
322332
IRBuilderBase &Builder = State.Builder;
323333

@@ -597,13 +607,36 @@ Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
597607
Value *Addend = State.get(getOperand(1), Part, /* IsScalar */ true);
598608
return Builder.CreatePtrAdd(Ptr, Addend, Name);
599609
}
610+
case VPInstruction::ExitPhi: {
611+
if (Part != 0)
612+
return State.get(this, 0, /*IsScalar*/ true);
613+
Value *IncomingFromVPlanPred =
614+
State.get(getOperand(0), Part, /* IsScalar */ true);
615+
Value *IncomingForOtherPredecessors =
616+
State.get(getOperand(1), Part, /* IsScalar */ true);
617+
auto *NewPhi =
618+
Builder.CreatePHI(IncomingForOtherPredecessors->getType(), 2, Name);
619+
SmallVector<BasicBlock *> Blocks(predecessors(Builder.GetInsertBlock()));
620+
BasicBlock *VPlanPred =
621+
State.CFG
622+
.VPBB2IRBB[cast<VPBasicBlock>(getParent()->getSinglePredecessor())];
623+
reorderIncomingBlocks(Blocks, VPlanPred);
624+
for (auto *BB : Blocks) {
625+
auto *Incoming = BB == VPlanPred ? IncomingFromVPlanPred
626+
: IncomingForOtherPredecessors;
627+
NewPhi->addIncoming(Incoming, BB);
628+
}
629+
return NewPhi;
630+
}
631+
600632
default:
601633
llvm_unreachable("Unsupported opcode for instruction");
602634
}
603635
}
604636

605637
bool VPInstruction::isVectorToScalar() const {
606638
return getOpcode() == VPInstruction::ExtractFromEnd ||
639+
getOpcode() == VPInstruction::ExitPhi ||
607640
getOpcode() == VPInstruction::ComputeReductionResult;
608641
}
609642

@@ -706,6 +739,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
706739
case VPInstruction::ActiveLaneMask:
707740
O << "active lane mask";
708741
break;
742+
case VPInstruction::ExitPhi:
743+
O << "exit-phi";
744+
break;
709745
case VPInstruction::ExplicitVectorLength:
710746
O << "EXPLICIT-VECTOR-LENGTH";
711747
break;

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -939,14 +939,6 @@ bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan,
939939
{}, "vector.recur.extract.for.phi"));
940940
RecurSplice->replaceUsesWithIf(
941941
Penultimate, [](VPUser &U, unsigned) { return isa<VPLiveOut>(&U); });
942-
943-
// Extract the resume value and create a new VPLiveOut for it.
944-
auto *Resume = MiddleBuilder.createNaryOp(
945-
VPInstruction::ExtractFromEnd,
946-
{FOR->getBackedgeValue(),
947-
Plan.getOrAddLiveIn(ConstantInt::get(IntTy, 1))},
948-
{}, "vector.recur.extract");
949-
Plan.addLiveOut(cast<PHINode>(FOR->getUnderlyingInstr()), Resume);
950942
}
951943
return true;
952944
}

‎llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@ define i32 @test_phi_iterator_invalidation(ptr %A, ptr noalias %B) {
7373
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP24]], i32 3
7474
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
7575
; CHECK: scalar.ph:
76-
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
77-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1004, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
76+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1004, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
77+
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
7878
; CHECK-NEXT: br label [[LOOP:%.*]]
7979
; CHECK: loop:
8080
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/fixed-order-recurrence.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ define void @firstorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt
5151
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
5252
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
5353
; CHECK: scalar.ph:
54-
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
5554
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_BODY_PREHEADER]] ]
55+
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
5656
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
5757
; CHECK: for.cond.cleanup.loopexit:
5858
; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
@@ -160,10 +160,10 @@ define void @thirdorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt
160160
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
161161
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
162162
; CHECK: scalar.ph:
163-
; CHECK-NEXT: [[SCALAR_RECUR_INIT10:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT9]], [[MIDDLE_BLOCK]] ]
164-
; CHECK-NEXT: [[SCALAR_RECUR_INIT7:%.*]] = phi i8 [ [[DOTPRE44]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT6]], [[MIDDLE_BLOCK]] ]
165-
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[DOTPRE45]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
166163
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 3, [[FOR_BODY_PREHEADER]] ]
164+
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[DOTPRE45]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
165+
; CHECK-NEXT: [[SCALAR_RECUR_INIT7:%.*]] = phi i8 [ [[DOTPRE44]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT6]], [[MIDDLE_BLOCK]] ]
166+
; CHECK-NEXT: [[SCALAR_RECUR_INIT10:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT9]], [[MIDDLE_BLOCK]] ]
167167
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
168168
; CHECK: for.cond.cleanup.loopexit:
169169
; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]

‎llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,13 +126,13 @@ define i64 @pointer_induction_only(ptr %start, ptr %end) {
126126
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
127127
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
128128
; CHECK: middle.block:
129-
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
130129
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1
130+
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
131131
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
132132
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
133133
; CHECK: scalar.ph:
134-
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
135134
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY]] ]
135+
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
136136
; CHECK-NEXT: br label [[LOOP:%.*]]
137137
; CHECK: loop:
138138
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -191,13 +191,13 @@ define i64 @int_and_pointer_iv(ptr %start, i32 %N) {
191191
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
192192
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
193193
; CHECK: middle.block:
194-
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i64> [[TMP5]], i32 2
195194
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP5]], i32 3
195+
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i64> [[TMP5]], i32 2
196196
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
197197
; CHECK: scalar.ph:
198-
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
199198
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
200199
; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY]] ]
200+
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
201201
; CHECK-NEXT: br label [[LOOP:%.*]]
202202
; CHECK: loop:
203203
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]

0 commit comments

Comments
 (0)