Skip to content

Commit 0fc05ca

Browse files
fhahnfhahn
committed
[VPlan] Add new VPIRPhi overlay for VPIRInsts wrapping phi nodes (NFC).
Add a new VPIRPhi subclass of VPIRInstruction, that purely serves as an overlay, to provide more convenient checking (via directly doing isa/dyn_cast/cast) and specialied execute/print implementations. Both VPIRInstruction and VPIRPhi share the same VPDefID, and are differentiated by the backing IR instruction. This pattern could alos be used to provide more specialized interfaces for some VPInstructions ocpodes, without introducing new, completely spearate recipes. An example would be modeling VPWidenPHIRecipe & VPScalarPHIRecip using VPInstructions opcodes and providing an interface to retrieve incoming blocks and values through a VPInstruction subclass similar to VPIRPhi.
1 parent 369c0a7 commit 0fc05ca

File tree

6 files changed

+92
-53
lines changed

6 files changed

+92
-53
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9074,14 +9074,14 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
90749074
VPValue *OneVPV = Plan.getOrAddLiveIn(
90759075
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
90769076
for (VPRecipeBase &ScalarPhiR : *Plan.getScalarHeader()) {
9077-
auto *ScalarPhiIRI = cast<VPIRInstruction>(&ScalarPhiR);
9078-
auto *ScalarPhiI = dyn_cast<PHINode>(&ScalarPhiIRI->getInstruction());
9079-
if (!ScalarPhiI)
9077+
auto *ScalarPhiIRI = dyn_cast<VPIRPhi>(&ScalarPhiR);
9078+
if (!ScalarPhiIRI)
90809079
break;
90819080

90829081
// TODO: Extract final value from induction recipe initially, optimize to
90839082
// pre-computed end value together in optimizeInductionExitUsers.
9084-
auto *VectorPhiR = cast<VPHeaderPHIRecipe>(Builder.getRecipe(ScalarPhiI));
9083+
auto *VectorPhiR =
9084+
cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi()));
90859085
if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
90869086
if (VPInstruction *ResumePhi = addResumePhiRecipeForInduction(
90879087
WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
@@ -9131,20 +9131,19 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
91319131
continue;
91329132

91339133
for (VPRecipeBase &R : *ExitVPBB) {
9134-
auto *ExitIRI = dyn_cast<VPIRInstruction>(&R);
9134+
auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
91359135
if (!ExitIRI)
9136-
continue;
9137-
auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
9138-
if (!ExitPhi)
91399136
break;
91409137
if (ExitVPBB->getSinglePredecessor() != Plan.getMiddleBlock()) {
91419138
assert(ExitIRI->getNumOperands() ==
91429139
ExitVPBB->getPredecessors().size() &&
91439140
"early-exit must update exit values on construction");
91449141
continue;
91459142
}
9143+
9144+
PHINode &ExitPhi = ExitIRI->getIRPhi();
91469145
BasicBlock *ExitingBB = OrigLoop->getLoopLatch();
9147-
Value *IncomingValue = ExitPhi->getIncomingValueForBlock(ExitingBB);
9146+
Value *IncomingValue = ExitPhi.getIncomingValueForBlock(ExitingBB);
91489147
VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue);
91499148
ExitIRI->addOperand(V);
91509149
if (V->isLiveIn())
@@ -10325,11 +10324,10 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
1032510324
cast<PHINode>(R.getVPSingleValue()->getUnderlyingValue()));
1032610325
}
1032710326
for (VPRecipeBase &R : make_early_inc_range(*MainPlan.getScalarHeader())) {
10328-
auto *VPIRInst = cast<VPIRInstruction>(&R);
10329-
auto *IRI = dyn_cast<PHINode>(&VPIRInst->getInstruction());
10330-
if (!IRI)
10327+
auto *VPIRInst = dyn_cast<VPIRPhi>(&R);
10328+
if (!VPIRInst)
1033110329
break;
10332-
if (EpiWidenedPhis.contains(IRI))
10330+
if (EpiWidenedPhis.contains(&VPIRInst->getIRPhi()))
1033310331
continue;
1033410332
// There is no corresponding wide induction in the epilogue plan that would
1033510333
// need a resume value. Remove the VPIRInst wrapping the scalar header phi

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1227,7 +1227,7 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) {
12271227
auto *VPIRBB = createEmptyVPIRBasicBlock(IRBB);
12281228
for (Instruction &I :
12291229
make_range(IRBB->begin(), IRBB->getTerminator()->getIterator()))
1230-
VPIRBB->appendRecipe(new VPIRInstruction(I));
1230+
VPIRBB->appendRecipe(VPIRInstruction::create(I));
12311231
return VPIRBB;
12321232
}
12331233

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1033,16 +1033,19 @@ class VPInstruction : public VPRecipeWithIRFlags,
10331033
class VPIRInstruction : public VPRecipeBase {
10341034
Instruction &I;
10351035

1036-
public:
1036+
protected:
10371037
VPIRInstruction(Instruction &I)
10381038
: VPRecipeBase(VPDef::VPIRInstructionSC, ArrayRef<VPValue *>()), I(I) {}
10391039

1040+
public:
10401041
~VPIRInstruction() override = default;
10411042

1043+
static VPIRInstruction *create(Instruction &I);
1044+
10421045
VP_CLASSOF_IMPL(VPDef::VPIRInstructionSC)
10431046

10441047
VPIRInstruction *clone() override {
1045-
auto *R = new VPIRInstruction(I);
1048+
auto *R = create(I);
10461049
for (auto *Op : operands())
10471050
R->addOperand(Op);
10481051
return R;
@@ -1086,6 +1089,27 @@ class VPIRInstruction : public VPRecipeBase {
10861089
void extractLastLaneOfOperand(VPBuilder &Builder);
10871090
};
10881091

1092+
/// An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use
1093+
/// cast/dyn_cast/isa and execute() implementation.
1094+
struct VPIRPhi : public VPIRInstruction {
1095+
VPIRPhi(PHINode &PN) : VPIRInstruction(PN) {}
1096+
1097+
static inline bool classof(const VPRecipeBase *U) {
1098+
auto *R = dyn_cast<VPIRInstruction>(U);
1099+
return R && isa<PHINode>(R->getInstruction());
1100+
}
1101+
1102+
PHINode &getIRPhi() { return cast<PHINode>(getInstruction()); }
1103+
1104+
void execute(VPTransformState &State) override;
1105+
1106+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1107+
/// Print the recipe.
1108+
void print(raw_ostream &O, const Twine &Indent,
1109+
VPSlotTracker &SlotTracker) const override;
1110+
#endif
1111+
};
1112+
10891113
/// VPWidenRecipe is a recipe for producing a widened instruction using the
10901114
/// opcode and operands of the recipe. This recipe covers most of the
10911115
/// traditional vectorization cases where each recipe transforms into a

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 42 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -972,30 +972,15 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
972972
}
973973
#endif
974974

975-
void VPIRInstruction::execute(VPTransformState &State) {
976-
assert((isa<PHINode>(&I) || getNumOperands() == 0) &&
977-
"Only PHINodes can have extra operands");
978-
for (const auto &[Idx, Op] : enumerate(operands())) {
979-
VPValue *ExitValue = Op;
980-
auto Lane = vputils::isUniformAfterVectorization(ExitValue)
981-
? VPLane::getFirstLane()
982-
: VPLane::getLastLaneForVF(State.VF);
983-
VPBlockBase *Pred = getParent()->getPredecessors()[Idx];
984-
auto *PredVPBB = Pred->getExitingBasicBlock();
985-
BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];
986-
// Set insertion point in PredBB in case an extract needs to be generated.
987-
// TODO: Model extracts explicitly.
988-
State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt());
989-
Value *V = State.get(ExitValue, VPLane(Lane));
990-
auto *Phi = cast<PHINode>(&I);
991-
// If there is no existing block for PredBB in the phi, add a new incoming
992-
// value. Otherwise update the existing incoming value for PredBB.
993-
if (Phi->getBasicBlockIndex(PredBB) == -1)
994-
Phi->addIncoming(V, PredBB);
995-
else
996-
Phi->setIncomingValueForBlock(PredBB, V);
997-
}
975+
VPIRInstruction *VPIRInstruction ::create(Instruction &I) {
976+
if (auto *Phi = dyn_cast<PHINode>(&I))
977+
return new VPIRPhi(*Phi);
978+
return new VPIRInstruction(I);
979+
}
998980

981+
void VPIRInstruction::execute(VPTransformState &State) {
982+
assert(!isa<VPIRPhi>(this) && getNumOperands() == 0 &&
983+
"PHINodes must be handled by VPIRPhi");
999984
// Advance the insert point after the wrapped IR instruction. This allows
1000985
// interleaving VPIRInstructions and other recipes.
1001986
State.Builder.SetInsertPoint(I.getParent(), std::next(I.getIterator()));
@@ -1028,6 +1013,40 @@ void VPIRInstruction::extractLastLaneOfOperand(VPBuilder &Builder) {
10281013
void VPIRInstruction::print(raw_ostream &O, const Twine &Indent,
10291014
VPSlotTracker &SlotTracker) const {
10301015
O << Indent << "IR " << I;
1016+
}
1017+
#endif
1018+
1019+
void VPIRPhi::execute(VPTransformState &State) {
1020+
PHINode *Phi = &getIRPhi();
1021+
for (const auto &[Idx, Op] : enumerate(operands())) {
1022+
VPValue *ExitValue = Op;
1023+
auto Lane = vputils::isUniformAfterVectorization(ExitValue)
1024+
? VPLane::getFirstLane()
1025+
: VPLane::getLastLaneForVF(State.VF);
1026+
VPBlockBase *Pred = getParent()->getPredecessors()[Idx];
1027+
auto *PredVPBB = Pred->getExitingBasicBlock();
1028+
BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];
1029+
// Set insertion point in PredBB in case an extract needs to be generated.
1030+
// TODO: Model extracts explicitly.
1031+
State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt());
1032+
Value *V = State.get(ExitValue, VPLane(Lane));
1033+
// If there is no existing block for PredBB in the phi, add a new incoming
1034+
// value. Otherwise update the existing incoming value for PredBB.
1035+
if (Phi->getBasicBlockIndex(PredBB) == -1)
1036+
Phi->addIncoming(V, PredBB);
1037+
else
1038+
Phi->setIncomingValueForBlock(PredBB, V);
1039+
}
1040+
1041+
// Advance the insert point after the wrapped IR instruction. This allows
1042+
// interleaving VPIRInstructions and other recipes.
1043+
State.Builder.SetInsertPoint(Phi->getParent(), std::next(Phi->getIterator()));
1044+
}
1045+
1046+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1047+
void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
1048+
VPSlotTracker &SlotTracker) const {
1049+
VPIRInstruction::print(O, Indent, SlotTracker);
10311050

10321051
if (getNumOperands() != 0) {
10331052
O << " (extra operand" << (getNumOperands() > 1 ? "s" : "") << ": ";

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -792,11 +792,11 @@ void VPlanTransforms::optimizeInductionExitUsers(
792792
VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues) {
793793
VPBlockBase *MiddleVPBB = Plan.getMiddleBlock();
794794
VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
795-
for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
796-
for (VPRecipeBase &R : *ExitVPBB) {
797-
auto *ExitIRI = cast<VPIRInstruction>(&R);
798-
if (!isa<PHINode>(ExitIRI->getInstruction()))
799-
break;
795+
VPBuilder B(Plan.getMiddleBlock()->getTerminator());
796+
for (VPRecipeBase &R : *ExitVPBB) {
797+
auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
798+
if (!ExitIRI)
799+
break;
800800

801801
for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
802802
if (PredVPBB == MiddleVPBB)
@@ -2139,20 +2139,20 @@ void VPlanTransforms::handleUncountableEarlyExit(
21392139
VPBuilder MiddleBuilder(NewMiddle);
21402140
VPBuilder EarlyExitB(VectorEarlyExitVPBB);
21412141
for (VPRecipeBase &R : *VPEarlyExitBlock) {
2142-
auto *ExitIRI = cast<VPIRInstruction>(&R);
2143-
auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
2144-
if (!ExitPhi)
2142+
auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
2143+
if (!ExitIRI)
21452144
break;
21462145

2146+
PHINode &ExitPhi = ExitIRI->getIRPhi();
21472147
VPValue *IncomingFromEarlyExit = RecipeBuilder.getVPValueOrAddLiveIn(
2148-
ExitPhi->getIncomingValueForBlock(UncountableExitingBlock));
2148+
ExitPhi.getIncomingValueForBlock(UncountableExitingBlock));
21492149

21502150
if (OrigLoop->getUniqueExitBlock()) {
21512151
// If there's a unique exit block, VPEarlyExitBlock has 2 predecessors
21522152
// (MiddleVPBB and NewMiddle). Add the incoming value from MiddleVPBB
21532153
// which is coming from the original latch.
21542154
VPValue *IncomingFromLatch = RecipeBuilder.getVPValueOrAddLiveIn(
2155-
ExitPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
2155+
ExitPhi.getIncomingValueForBlock(OrigLoop->getLoopLatch()));
21562156
ExitIRI->addOperand(IncomingFromLatch);
21572157
ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
21582158
}

‎llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -205,10 +205,8 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
205205
for (const VPUser *U : V->users()) {
206206
auto *UI = cast<VPRecipeBase>(U);
207207
// TODO: check dominance of incoming values for phis properly.
208-
if (!UI ||
209-
isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI) ||
210-
(isa<VPIRInstruction>(UI) &&
211-
isa<PHINode>(cast<VPIRInstruction>(UI)->getInstruction())))
208+
if (!UI || isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe,
209+
VPIRPhi>(UI))
212210
continue;
213211

214212
// If the user is in the same block, check it comes after R in the

0 commit comments

Comments
 (0)