@@ -8630,11 +8630,12 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
8630
8630
{CanonicalIVIncrement, &Plan.getVectorTripCount()}, DL);
8631
8631
}
8632
8632
8633
- // Collect (ExitPhi, ExitingValue) pairs phis in the original exit block that
8634
- // are modeled in VPlan. Some exiting values are not modeled explicitly yet and
8635
- // won't be included. Those are un-truncated VPWidenIntOrFpInductionRecipe,
8636
- // VPWidenPointerInductionRecipe and induction increments.
8637
- static MapVector<PHINode *, VPValue *> collectUsersInExitBlock(
8633
+ // Collect VPIRInstructions for phis in the original exit block that are modeled
8634
+ // in VPlan and add the exiting VPValue as operand. Some exiting values are not
8635
+ // modeled explicitly yet and won't be included. Those are un-truncated
8636
+ // VPWidenIntOrFpInductionRecipe, VPWidenPointerInductionRecipe and induction
8637
+ // increments.
8638
+ static SetVector<VPIRInstruction *> collectUsersInExitBlock(
8638
8639
Loop *OrigLoop, VPRecipeBuilder &Builder, VPlan &Plan,
8639
8640
const MapVector<PHINode *, InductionDescriptor> &Inductions) {
8640
8641
auto *MiddleVPBB =
@@ -8644,13 +8645,17 @@ static MapVector<PHINode *, VPValue *> collectUsersInExitBlock(
8644
8645
// from scalar loop only.
8645
8646
if (MiddleVPBB->getNumSuccessors() != 2)
8646
8647
return {};
8647
- MapVector<PHINode *, VPValue *> ExitingValuesToFix;
8648
- BasicBlock *ExitBB =
8649
- cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0])->getIRBasicBlock();
8648
+ SetVector<VPIRInstruction *> ExitUsersToFix;
8649
+ VPBasicBlock *ExitVPBB = cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0]);
8650
8650
BasicBlock *ExitingBB = OrigLoop->getExitingBlock();
8651
- for (PHINode &ExitPhi : ExitBB->phis()) {
8652
- Value *IncomingValue =
8653
- ExitPhi.getIncomingValueForBlock(ExitingBB);
8651
+ for (VPRecipeBase &R : *ExitVPBB) {
8652
+ auto *ExitIRI = dyn_cast<VPIRInstruction>(&R);
8653
+ if (!ExitIRI)
8654
+ continue;
8655
+ auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
8656
+ if (!ExitPhi)
8657
+ break;
8658
+ Value *IncomingValue = ExitPhi->getIncomingValueForBlock(ExitingBB);
8654
8659
VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue);
8655
8660
// Exit values for inductions are computed and updated outside of VPlan and
8656
8661
// independent of induction recipes.
@@ -8666,17 +8671,18 @@ static MapVector<PHINode *, VPValue *> collectUsersInExitBlock(
8666
8671
return P && Inductions.contains(P);
8667
8672
})))
8668
8673
continue;
8669
- ExitingValuesToFix.insert({&ExitPhi, V});
8674
+ ExitUsersToFix.insert(ExitIRI);
8675
+ ExitIRI->addOperand(V);
8670
8676
}
8671
- return ExitingValuesToFix ;
8677
+ return ExitUsersToFix ;
8672
8678
}
8673
8679
8674
- // Add exit values to \p Plan. Extracts and VPLiveOuts are added for each entry
8675
- // in \p ExitingValuesToFix .
8680
+ // Add exit values to \p Plan. Extracts are added for each entry in \p
8681
+ // ExitUsersToFix if needed and their operands are updated .
8676
8682
static void
8677
8683
addUsersInExitBlock(VPlan &Plan,
8678
- MapVector<PHINode *, VPValue * > &ExitingValuesToFix ) {
8679
- if (ExitingValuesToFix .empty())
8684
+ const SetVector<VPIRInstruction * > &ExitUsersToFix ) {
8685
+ if (ExitUsersToFix .empty())
8680
8686
return;
8681
8687
8682
8688
auto *MiddleVPBB =
@@ -8685,18 +8691,19 @@ addUsersInExitBlock(VPlan &Plan,
8685
8691
cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0])->getIRBasicBlock();
8686
8692
VPBuilder B(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
8687
8693
8688
- // Introduce VPUsers modeling the exit values.
8689
- for (const auto &[ExitPhi, V] : ExitingValuesToFix) {
8694
+ // Introduce extract for exiting values and update the VPIRInstructions
8695
+ // modeling the corresponding LCSSA phis.
8696
+ for (VPIRInstruction *ExitIRI : ExitUsersToFix) {
8697
+ VPValue *V = ExitIRI->getOperand(0);
8690
8698
// Pass live-in values used by exit phis directly through to the live-out.
8691
- if (V->isLiveIn()) {
8692
- Plan.addLiveOut(ExitPhi, V);
8699
+ if (V->isLiveIn())
8693
8700
continue;
8694
- }
8701
+
8695
8702
VPValue *Ext = B.createNaryOp(
8696
8703
VPInstruction::ExtractFromEnd,
8697
8704
{V, Plan.getOrAddLiveIn(ConstantInt::get(
8698
8705
IntegerType::get(ExitBB->getContext(), 32), 1))});
8699
- Plan.addLiveOut(ExitPhi , Ext);
8706
+ ExitIRI->setOperand(0 , Ext);
8700
8707
}
8701
8708
}
8702
8709
@@ -8709,7 +8716,7 @@ addUsersInExitBlock(VPlan &Plan,
8709
8716
/// 2. Feed the penultimate value of recurrences to their LCSSA phi users in
8710
8717
/// the original exit block using a VPLiveOut.
8711
8718
static void addLiveOutsForFirstOrderRecurrences(
8712
- VPlan &Plan, MapVector<PHINode *, VPValue * > &ExitingValuesToFix ) {
8719
+ VPlan &Plan, SetVector<VPIRInstruction * > &ExitUsersToFix ) {
8713
8720
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
8714
8721
8715
8722
// Start by finding out if middle block branches to scalar preheader, which is
@@ -8726,14 +8733,14 @@ static void addLiveOutsForFirstOrderRecurrences(
8726
8733
ExitBB =
8727
8734
cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0])->getIRBasicBlock();
8728
8735
ScalarPHVPBB = cast<VPBasicBlock>(MiddleVPBB->getSuccessors()[1]);
8729
- } else if (ExitingValuesToFix .empty()) {
8736
+ } else if (ExitUsersToFix .empty()) {
8730
8737
ScalarPHVPBB = cast<VPBasicBlock>(MiddleVPBB->getSingleSuccessor());
8731
8738
} else {
8732
8739
ExitBB = cast<VPIRBasicBlock>(MiddleVPBB->getSingleSuccessor())
8733
8740
->getIRBasicBlock();
8734
8741
}
8735
8742
if (!ScalarPHVPBB) {
8736
- assert(ExitingValuesToFix .empty() &&
8743
+ assert(ExitUsersToFix .empty() &&
8737
8744
"missed inserting extracts for exiting values");
8738
8745
return;
8739
8746
}
@@ -8827,24 +8834,17 @@ static void addLiveOutsForFirstOrderRecurrences(
8827
8834
auto *FORPhi = cast<PHINode>(FOR->getUnderlyingInstr());
8828
8835
Plan.addLiveOut(FORPhi, ResumePhiRecipe);
8829
8836
8830
- // Now create VPLiveOuts for users in the exit block.
8831
- // Extract the penultimate value of the recurrence and add VPLiveOut
8832
- // users of the recurrence splice.
8833
-
8834
- // No edge from the middle block to the unique exit block has been inserted
8835
- // and there is nothing to fix from vector loop; phis should have incoming
8836
- // from scalar loop only.
8837
- if (ExitingValuesToFix.empty())
8838
- continue;
8839
- for (User *U : FORPhi->users()) {
8840
- auto *UI = cast<Instruction>(U);
8841
- if (UI->getParent() != ExitBB)
8837
+ // Now update VPIRInstructions modeling LCSSA phis in the exit block.
8838
+ // Extract the penultimate value of the recurrence and use it as operand for
8839
+ // the VPIRInstruction modeling the phi.
8840
+ for (VPIRInstruction *ExitIRI : ExitUsersToFix) {
8841
+ if (ExitIRI->getOperand(0) != FOR)
8842
8842
continue;
8843
8843
VPValue *Ext = MiddleBuilder.createNaryOp(
8844
8844
VPInstruction::ExtractFromEnd, {FOR->getBackedgeValue(), TwoVPV}, {},
8845
8845
"vector.recur.extract.for.phi");
8846
- Plan.addLiveOut(cast<PHINode>(UI) , Ext);
8847
- ExitingValuesToFix.erase(cast<PHINode>(UI) );
8846
+ ExitIRI->setOperand(0 , Ext);
8847
+ ExitUsersToFix.remove(ExitIRI );
8848
8848
}
8849
8849
}
8850
8850
}
@@ -9006,11 +9006,10 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
9006
9006
"VPBasicBlock");
9007
9007
RecipeBuilder.fixHeaderPhis();
9008
9008
9009
- MapVector<PHINode *, VPValue *> ExitingValuesToFix = collectUsersInExitBlock(
9009
+ SetVector<VPIRInstruction *> ExitUsersToFix = collectUsersInExitBlock(
9010
9010
OrigLoop, RecipeBuilder, *Plan, Legal->getInductionVars());
9011
-
9012
- addLiveOutsForFirstOrderRecurrences(*Plan, ExitingValuesToFix);
9013
- addUsersInExitBlock(*Plan, ExitingValuesToFix);
9011
+ addLiveOutsForFirstOrderRecurrences(*Plan, ExitUsersToFix);
9012
+ addUsersInExitBlock(*Plan, ExitUsersToFix);
9014
9013
9015
9014
// ---------------------------------------------------------------------------
9016
9015
// Transform initial VPlan: Apply previously taken decisions, in order, to
@@ -10128,7 +10127,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
10128
10127
// directly in VPlan.
10129
10128
EpilogILV.setTripCount(MainILV.getTripCount());
10130
10129
for (auto &R : make_early_inc_range(*BestEpiPlan.getPreheader())) {
10131
- auto *ExpandR = cast<VPExpandSCEVRecipe>(&R);
10130
+ auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
10131
+ if (!ExpandR)
10132
+ continue;
10132
10133
auto *ExpandedVal = BestEpiPlan.getOrAddLiveIn(
10133
10134
ExpandedSCEVs.find(ExpandR->getSCEV())->second);
10134
10135
ExpandR->replaceAllUsesWith(ExpandedVal);
0 commit comments