Skip to content

Commit e8bac66

Browse files
committed
[VPlan] Introduce multi-branch recipe, use for multi-exit loops (WIP).
This patch introduces a new BranchMultipleConds VPInstruction that takes multiple conditions and branches to the first successor if the first operand is true, to the second successor if the second condition is true and to the region header if neither is true. At the moment it only supports 2 conditions, but it can be extended in the future. This may serve as an alternative to changing VPRegionBlock to allow multiple exiting blocks and keep it single-entry-single-exit. With BranchMultipleConds, we still leave a region via a single exiting block, but can have more than 2 destinations (similar idea to switch in LLVM IR). The new recipe allows to precisely model edges and conditions leaving the vector loop region. BranchMultipleConds also allows predicating instructions in blocks after any early exit, i.e. also allows later stores. See llvm/test/Transforms/LoopVectorize/X86/multi-exit-vplan.ll for an example VPlan and llvm/test/Transforms/LoopVectorize/X86/multi-exit-codegen.ll for example predicated codegen. The patch also contains logic to construct VPlans using BranchMultipleConds for simple loops with 2 exit blocks instead of requiring a scalar tail. To logic to detect such cases is a bit rough around the edges and mainly to test the new recipes end-to-end. This may serve as an alternative to #108563 that would allow us to keep the single-entry-single-exit property and support predication between early exits and latches.
1 parent 3cfe25b commit e8bac66

File tree

12 files changed

+536
-65
lines changed

12 files changed

+536
-65
lines changed

llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,8 @@ class LoopVectorizationLegality {
287287
/// we can use in-order reductions.
288288
bool canVectorizeFPMath(bool EnableStrictReductions);
289289

290+
bool canVectorizeMultiCond() const;
291+
290292
/// Return true if we can vectorize this loop while folding its tail by
291293
/// masking.
292294
bool canFoldTailByMasking() const;

llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ AllowStridedPointerIVs("lv-strided-pointer-ivs", cl::init(false), cl::Hidden,
4343
cl::desc("Enable recognition of non-constant strided "
4444
"pointer induction variables."));
4545

46+
static cl::opt<bool> EnableMultiCond("enable-multi-cond-vectorization",
47+
cl::init(false), cl::Hidden, cl::desc(""));
48+
4649
namespace llvm {
4750
cl::opt<bool>
4851
HintsAllowReordering("hints-allow-reordering", cl::init(true), cl::Hidden,
@@ -1378,6 +1381,8 @@ bool LoopVectorizationLegality::isFixedOrderRecurrence(
13781381
}
13791382

13801383
bool LoopVectorizationLegality::blockNeedsPredication(BasicBlock *BB) const {
1384+
if (canVectorizeMultiCond() && BB != TheLoop->getHeader())
1385+
return true;
13811386
return LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT);
13821387
}
13831388

@@ -1514,6 +1519,35 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
15141519
return true;
15151520
}
15161521

1522+
bool LoopVectorizationLegality::canVectorizeMultiCond() const {
1523+
if (!EnableMultiCond)
1524+
return false;
1525+
SmallVector<BasicBlock *> Exiting;
1526+
TheLoop->getExitingBlocks(Exiting);
1527+
if (Exiting.size() != 2 || Exiting[0] != TheLoop->getHeader() ||
1528+
Exiting[1] != TheLoop->getLoopLatch() ||
1529+
any_of(*TheLoop->getHeader(), [](Instruction &I) {
1530+
return I.mayReadFromMemory() || I.mayHaveSideEffects();
1531+
}))
1532+
return false;
1533+
CmpInst::Predicate Pred;
1534+
Value *A, *B;
1535+
if (!match(
1536+
TheLoop->getHeader()->getTerminator(),
1537+
m_Br(m_ICmp(Pred, m_Value(A), m_Value(B)), m_Value(), m_Value())) ||
1538+
Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE)
1539+
return false;
1540+
if (any_of(TheLoop->getBlocks(), [this](BasicBlock *BB) {
1541+
return any_of(*BB, [this](Instruction &I) {
1542+
return any_of(I.users(), [this](User *U) {
1543+
return !TheLoop->contains(cast<Instruction>(U)->getParent());
1544+
});
1545+
});
1546+
}))
1547+
return false;
1548+
return true;
1549+
}
1550+
15171551
// Helper function to canVectorizeLoopNestCFG.
15181552
bool LoopVectorizationLegality::canVectorizeLoopCFG(Loop *Lp,
15191553
bool UseVPlanNativePath) {

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 57 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1363,9 +1363,11 @@ class LoopVectorizationCostModel {
13631363
// If we might exit from anywhere but the latch, must run the exiting
13641364
// iteration in scalar form.
13651365
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
1366-
LLVM_DEBUG(
1367-
dbgs() << "LV: Loop requires scalar epilogue: multiple exits\n");
1368-
return true;
1366+
if (!Legal->canVectorizeMultiCond()) {
1367+
LLVM_DEBUG(
1368+
dbgs() << "LV: Loop requires scalar epilogue: multiple exits\n");
1369+
return true;
1370+
}
13691371
}
13701372
if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
13711373
LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
@@ -2544,8 +2546,17 @@ void InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
25442546
LoopVectorPreHeader = OrigLoop->getLoopPreheader();
25452547
assert(LoopVectorPreHeader && "Invalid loop structure");
25462548
LoopExitBlock = OrigLoop->getUniqueExitBlock(); // may be nullptr
2547-
assert((LoopExitBlock || Cost->requiresScalarEpilogue(VF.isVector())) &&
2548-
"multiple exit loop without required epilogue?");
2549+
if (Legal->canVectorizeMultiCond()) {
2550+
BasicBlock *Latch = OrigLoop->getLoopLatch();
2551+
BasicBlock *TrueSucc =
2552+
cast<BranchInst>(Latch->getTerminator())->getSuccessor(0);
2553+
BasicBlock *FalseSucc =
2554+
cast<BranchInst>(Latch->getTerminator())->getSuccessor(1);
2555+
LoopExitBlock = OrigLoop->contains(TrueSucc) ? FalseSucc : TrueSucc;
2556+
} else {
2557+
assert((LoopExitBlock || Cost->requiresScalarEpilogue(VF.isVector())) &&
2558+
"multiple exit loop without required epilogue?");
2559+
}
25492560

25502561
LoopMiddleBlock =
25512562
SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
@@ -2912,7 +2923,8 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
29122923
for (PHINode &PN : Exit->phis())
29132924
PSE.getSE()->forgetLcssaPhiWithNewPredecessor(OrigLoop, &PN);
29142925

2915-
if (Cost->requiresScalarEpilogue(VF.isVector())) {
2926+
if (Legal->canVectorizeMultiCond() ||
2927+
Cost->requiresScalarEpilogue(VF.isVector())) {
29162928
// No edge from the middle block to the unique exit block has been inserted
29172929
// and there is nothing to fix from vector loop; phis should have incoming
29182930
// from scalar loop only.
@@ -3557,7 +3569,8 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
35573569
TheLoop->getExitingBlocks(Exiting);
35583570
for (BasicBlock *E : Exiting) {
35593571
auto *Cmp = dyn_cast<Instruction>(E->getTerminator()->getOperand(0));
3560-
if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse())
3572+
if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse() &&
3573+
(TheLoop->getLoopLatch() == E || !Legal->canVectorizeMultiCond()))
35613574
AddToWorklistIfAllowed(Cmp);
35623575
}
35633576

@@ -7522,7 +7535,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
75227535
LLVM_DEBUG(BestVPlan.dump());
75237536

75247537
// Perform the actual loop transformation.
7525-
VPTransformState State(BestVF, BestUF, LI, DT, ILV.Builder, &ILV, &BestVPlan);
7538+
VPTransformState State(BestVF, BestUF, LI, DT, ILV.Builder, &ILV, &BestVPlan,
7539+
OrigLoop);
75267540

75277541
// 0. Generate SCEV-dependent code into the preheader, including TripCount,
75287542
// before making any changes to the CFG.
@@ -7583,12 +7597,15 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
75837597
BestVPlan.execute(&State);
75847598

75857599
// 2.5 Collect reduction resume values.
7586-
auto *ExitVPBB =
7587-
cast<VPBasicBlock>(BestVPlan.getVectorLoopRegion()->getSingleSuccessor());
7588-
for (VPRecipeBase &R : *ExitVPBB) {
7589-
createAndCollectMergePhiForReduction(
7590-
dyn_cast<VPInstruction>(&R), State, OrigLoop,
7591-
State.CFG.VPBB2IRBB[ExitVPBB], ExpandedSCEVs);
7600+
VPBasicBlock *ExitVPBB = nullptr;
7601+
if (BestVPlan.getVectorLoopRegion()->getSingleSuccessor()) {
7602+
ExitVPBB = cast<VPBasicBlock>(
7603+
BestVPlan.getVectorLoopRegion()->getSingleSuccessor());
7604+
for (VPRecipeBase &R : *ExitVPBB) {
7605+
createAndCollectMergePhiForReduction(
7606+
dyn_cast<VPInstruction>(&R), State, OrigLoop,
7607+
State.CFG.VPBB2IRBB[ExitVPBB], ExpandedSCEVs);
7608+
}
75927609
}
75937610

75947611
// 2.6. Maintain Loop Hints
@@ -7614,6 +7631,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
76147631
LoopVectorizeHints Hints(L, true, *ORE);
76157632
Hints.setAlreadyVectorized();
76167633
}
7634+
76177635
TargetTransformInfo::UnrollingPreferences UP;
76187636
TTI.getUnrollingPreferences(L, *PSE.getSE(), UP, ORE);
76197637
if (!UP.UnrollVectorizedLoop || CanonicalIVStartValue)
@@ -7626,15 +7644,17 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
76267644
ILV.printDebugTracesAtEnd();
76277645

76287646
// 4. Adjust branch weight of the branch in the middle block.
7629-
auto *MiddleTerm =
7630-
cast<BranchInst>(State.CFG.VPBB2IRBB[ExitVPBB]->getTerminator());
7631-
if (MiddleTerm->isConditional() &&
7632-
hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) {
7633-
// Assume that `Count % VectorTripCount` is equally distributed.
7634-
unsigned TripCount = BestVPlan.getUF() * State.VF.getKnownMinValue();
7635-
assert(TripCount > 0 && "trip count should not be zero");
7636-
const uint32_t Weights[] = {1, TripCount - 1};
7637-
setBranchWeights(*MiddleTerm, Weights, /*IsExpected=*/false);
7647+
if (ExitVPBB) {
7648+
auto *MiddleTerm =
7649+
cast<BranchInst>(State.CFG.VPBB2IRBB[ExitVPBB]->getTerminator());
7650+
if (MiddleTerm->isConditional() &&
7651+
hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) {
7652+
// Assume that `Count % VectorTripCount` is equally distributed.
7653+
unsigned TripCount = BestVPlan.getUF() * State.VF.getKnownMinValue();
7654+
assert(TripCount > 0 && "trip count should not be zero");
7655+
const uint32_t Weights[] = {1, TripCount - 1};
7656+
setBranchWeights(*MiddleTerm, Weights, /*IsExpected=*/false);
7657+
}
76387658
}
76397659

76407660
return State.ExpandedSCEVs;
@@ -8019,7 +8039,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) {
80198039
// If source is an exiting block, we know the exit edge is dynamically dead
80208040
// in the vector loop, and thus we don't need to restrict the mask. Avoid
80218041
// adding uses of an otherwise potentially dead instruction.
8022-
if (OrigLoop->isLoopExiting(Src))
8042+
if (!Legal->canVectorizeMultiCond() && OrigLoop->isLoopExiting(Src))
80238043
return EdgeMaskCache[Edge] = SrcMask;
80248044

80258045
VPValue *EdgeMask = getVPValueOrAddLiveIn(BI->getCondition());
@@ -8664,6 +8684,8 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
86648684
static SetVector<VPIRInstruction *> collectUsersInExitBlock(
86658685
Loop *OrigLoop, VPRecipeBuilder &Builder, VPlan &Plan,
86668686
const MapVector<PHINode *, InductionDescriptor> &Inductions) {
8687+
if (!Plan.getVectorLoopRegion()->getSingleSuccessor())
8688+
return {};
86678689
auto *MiddleVPBB =
86688690
cast<VPBasicBlock>(Plan.getVectorLoopRegion()->getSingleSuccessor());
86698691
// No edge from the middle block to the unique exit block has been inserted
@@ -8751,6 +8773,8 @@ static void addLiveOutsForFirstOrderRecurrences(
87518773
// TODO: Should be replaced by
87528774
// Plan->getScalarLoopRegion()->getSinglePredecessor() in the future once the
87538775
// scalar region is modeled as well.
8776+
if (!VectorRegion->getSingleSuccessor())
8777+
return;
87548778
auto *MiddleVPBB = cast<VPBasicBlock>(VectorRegion->getSingleSuccessor());
87558779
VPBasicBlock *ScalarPHVPBB = nullptr;
87568780
if (MiddleVPBB->getNumSuccessors() == 2) {
@@ -9037,6 +9061,11 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
90379061
"VPBasicBlock");
90389062
RecipeBuilder.fixHeaderPhis();
90399063

9064+
if (Legal->canVectorizeMultiCond()) {
9065+
VPlanTransforms::convertToMultiCond(*Plan, *PSE.getSE(), OrigLoop,
9066+
RecipeBuilder);
9067+
}
9068+
90409069
SetVector<VPIRInstruction *> ExitUsersToFix = collectUsersInExitBlock(
90419070
OrigLoop, RecipeBuilder, *Plan, Legal->getInductionVars());
90429071
addLiveOutsForFirstOrderRecurrences(*Plan, ExitUsersToFix);
@@ -9168,8 +9197,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
91689197
using namespace VPlanPatternMatch;
91699198
VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
91709199
VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock();
9171-
VPBasicBlock *MiddleVPBB =
9172-
cast<VPBasicBlock>(VectorLoopRegion->getSingleSuccessor());
91739200
for (VPRecipeBase &R : Header->phis()) {
91749201
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
91759202
if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
@@ -9188,8 +9215,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
91889215
for (VPUser *U : Cur->users()) {
91899216
auto *UserRecipe = cast<VPSingleDefRecipe>(U);
91909217
if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
9191-
assert(UserRecipe->getParent() == MiddleVPBB &&
9192-
"U must be either in the loop region or the middle block.");
91939218
continue;
91949219
}
91959220
Worklist.insert(UserRecipe);
@@ -9294,6 +9319,10 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92949319
}
92959320
VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
92969321
Builder.setInsertPoint(&*LatchVPBB->begin());
9322+
if (!VectorLoopRegion->getSingleSuccessor())
9323+
return;
9324+
VPBasicBlock *MiddleVPBB =
9325+
cast<VPBasicBlock>(VectorLoopRegion->getSingleSuccessor());
92979326
VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
92989327
for (VPRecipeBase &R :
92999328
Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {

llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -474,6 +474,14 @@ void VPIRBasicBlock::execute(VPTransformState *State) {
474474
// backedges. A backward successor is set when the branch is created.
475475
const auto &PredVPSuccessors = PredVPBB->getHierarchicalSuccessors();
476476
unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
477+
if (TermBr->getSuccessor(idx) &&
478+
PredVPBlock == getPlan()->getVectorLoopRegion() &&
479+
PredVPBlock->getNumSuccessors()) {
480+
// Update PRedBB and TermBr for BranchOnMultiCond in predecessor.
481+
PredBB = TermBr->getSuccessor(1);
482+
TermBr = cast<BranchInst>(PredBB->getTerminator());
483+
idx = 0;
484+
}
477485
assert(!TermBr->getSuccessor(idx) &&
478486
"Trying to reset an existing successor block.");
479487
TermBr->setSuccessor(idx, IRBB);
@@ -601,9 +609,11 @@ static bool hasConditionalTerminator(const VPBasicBlock *VPBB) {
601609
}
602610

603611
const VPRecipeBase *R = &VPBB->back();
604-
bool IsCondBranch = isa<VPBranchOnMaskRecipe>(R) ||
605-
match(R, m_BranchOnCond(m_VPValue())) ||
606-
match(R, m_BranchOnCount(m_VPValue(), m_VPValue()));
612+
bool IsCondBranch =
613+
isa<VPBranchOnMaskRecipe>(R) || match(R, m_BranchOnCond(m_VPValue())) ||
614+
match(R, m_BranchOnCount(m_VPValue(), m_VPValue())) ||
615+
(isa<VPInstruction>(R) && cast<VPInstruction>(R)->getOpcode() ==
616+
VPInstruction::BranchMultipleConds);
607617
(void)IsCondBranch;
608618

609619
if (VPBB->getNumSuccessors() >= 2 ||
@@ -901,8 +911,8 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
901911
VPBasicBlock *MiddleVPBB = new VPBasicBlock("middle.block");
902912
VPBlockUtils::insertBlockAfter(MiddleVPBB, TopRegion);
903913

904-
VPBasicBlock *ScalarPH = new VPBasicBlock("scalar.ph");
905914
if (!RequiresScalarEpilogueCheck) {
915+
VPBasicBlock *ScalarPH = new VPBasicBlock("scalar.ph");
906916
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
907917
return Plan;
908918
}
@@ -916,10 +926,14 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
916926
// we unconditionally branch to the scalar preheader. Do nothing.
917927
// 3) Otherwise, construct a runtime check.
918928
BasicBlock *IRExitBlock = TheLoop->getUniqueExitBlock();
919-
auto *VPExitBlock = VPIRBasicBlock::fromBasicBlock(IRExitBlock);
920-
// The connection order corresponds to the operands of the conditional branch.
921-
VPBlockUtils::insertBlockAfter(VPExitBlock, MiddleVPBB);
922-
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
929+
if (IRExitBlock) {
930+
auto *VPExitBlock = VPIRBasicBlock::fromBasicBlock(IRExitBlock);
931+
// The connection order corresponds to the operands of the conditional
932+
// branch.
933+
VPBlockUtils::insertBlockAfter(VPExitBlock, MiddleVPBB);
934+
VPBasicBlock *ScalarPH = new VPBasicBlock("scalar.ph");
935+
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
936+
}
923937

924938
auto *ScalarLatchTerm = TheLoop->getLoopLatch()->getTerminator();
925939
// Here we use the same DebugLoc as the scalar loop latch terminator instead
@@ -1028,7 +1042,9 @@ void VPlan::execute(VPTransformState *State) {
10281042
// VPlan execution rather than earlier during VPlan construction.
10291043
BasicBlock *MiddleBB = State->CFG.ExitBB;
10301044
VPBasicBlock *MiddleVPBB =
1031-
cast<VPBasicBlock>(getVectorLoopRegion()->getSingleSuccessor());
1045+
getVectorLoopRegion()->getNumSuccessors() == 1
1046+
? cast<VPBasicBlock>(getVectorLoopRegion()->getSuccessors()[0])
1047+
: cast<VPBasicBlock>(getVectorLoopRegion()->getSuccessors()[1]);
10321048
// Find the VPBB for the scalar preheader, relying on the current structure
10331049
// when creating the middle block and its successrs: if there's a single
10341050
// predecessor, it must be the scalar preheader. Otherwise, the second
@@ -1062,6 +1078,10 @@ void VPlan::execute(VPTransformState *State) {
10621078
VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock();
10631079
BasicBlock *VectorLatchBB = State->CFG.VPBB2IRBB[LatchVPBB];
10641080

1081+
if (!getVectorLoopRegion()->getSingleSuccessor())
1082+
VectorLatchBB =
1083+
cast<BranchInst>(VectorLatchBB->getTerminator())->getSuccessor(1);
1084+
10651085
// Fix the latch value of canonical, reduction and first-order recurrences
10661086
// phis in the vector loop.
10671087
VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
@@ -1088,7 +1108,10 @@ void VPlan::execute(VPTransformState *State) {
10881108
// Move the last step to the end of the latch block. This ensures
10891109
// consistent placement of all induction updates.
10901110
Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
1091-
Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
1111+
if (VectorLatchBB->getTerminator() == &*VectorLatchBB->getFirstNonPHI())
1112+
Inc->moveBefore(VectorLatchBB->getTerminator());
1113+
else
1114+
Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
10921115

10931116
// Use the steps for the last part as backedge value for the induction.
10941117
if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))

llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ class VPLane {
236236
struct VPTransformState {
237237
VPTransformState(ElementCount VF, unsigned UF, LoopInfo *LI,
238238
DominatorTree *DT, IRBuilderBase &Builder,
239-
InnerLoopVectorizer *ILV, VPlan *Plan);
239+
InnerLoopVectorizer *ILV, VPlan *Plan, Loop *OrigLoop);
240240

241241
/// The chosen Vectorization Factor of the loop being vectorized.
242242
ElementCount VF;
@@ -1235,6 +1235,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
12351235
CanonicalIVIncrementForPart,
12361236
BranchOnCount,
12371237
BranchOnCond,
1238+
BranchMultipleConds,
12381239
ComputeReductionResult,
12391240
// Takes the VPValue to extract from as first operand and the lane or part
12401241
// to extract as second operand, counting from the end starting with 1 for
@@ -1245,6 +1246,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
12451246
// operand). Only generates scalar values (either for the first lane only or
12461247
// for all lanes, depending on its uses).
12471248
PtrAdd,
1249+
AnyOf,
12481250
};
12491251

12501252
private:
@@ -1360,6 +1362,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
13601362
case Instruction::AtomicRMW:
13611363
case VPInstruction::BranchOnCond:
13621364
case VPInstruction::BranchOnCount:
1365+
case VPInstruction::BranchMultipleConds:
13631366
return false;
13641367
default:
13651368
return true;

0 commit comments

Comments
 (0)