@@ -7659,14 +7659,17 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
76597659 } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
76607660 RdxDesc.getRecurrenceKind())) {
76617661 using namespace llvm::PatternMatch;
7662- Value *Cmp, *OrigResumeV;
7662+ Value *Cmp, *OrigResumeV, *CmpOp ;
76637663 bool IsExpectedPattern =
76647664 match(MainResumeValue, m_Select(m_OneUse(m_Value(Cmp)),
76657665 m_Specific(RdxDesc.getSentinelValue()),
76667666 m_Value(OrigResumeV))) &&
7667- match(Cmp,
7668- m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(OrigResumeV),
7669- m_Specific(RdxDesc.getRecurrenceStartValue())));
7667+ (match(Cmp, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(OrigResumeV),
7668+ m_Value(CmpOp))) &&
7669+ (match(CmpOp,
7670+ m_Freeze(m_Specific(RdxDesc.getRecurrenceStartValue()))) ||
7671+ (CmpOp == RdxDesc.getRecurrenceStartValue() &&
7672+ isGuaranteedNotToBeUndefOrPoison(CmpOp))));
76707673 assert(IsExpectedPattern && "Unexpected reduction resume pattern");
76717674 (void)IsExpectedPattern;
76727675 MainResumeValue = OrigResumeV;
@@ -10374,6 +10377,36 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
1037410377 VPlanTransforms::runPass(VPlanTransforms::removeDeadRecipes, MainPlan);
1037510378
1037610379 using namespace VPlanPatternMatch;
10380+ // When vectorizing the epilogue, FindLastIV reductions can introduce multiple
10381+ // uses of undef/poison. If the reduction start value may be undef or poison
10382+ // it needs to be frozen and the frozen start has to be used when computing
10383+ // the reduction result. We also need to use the frozen value in the resume
10384+ // phi generated by the main vector loop, as this is also used to compute the
10385+ // reduction result after the epilogue vector loop.
10386+ auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
10387+ bool UpdateResumePhis) {
10388+ VPBuilder Builder(Plan.getEntry());
10389+ for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
10390+ auto *VPI = dyn_cast<VPInstruction>(&R);
10391+ if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindLastIVResult)
10392+ continue;
10393+ VPValue *OrigStart = VPI->getOperand(1);
10394+ if (isGuaranteedNotToBeUndefOrPoison(OrigStart->getLiveInIRValue()))
10395+ continue;
10396+ VPInstruction *Freeze =
10397+ Builder.createNaryOp(Instruction::Freeze, {OrigStart}, {}, "fr");
10398+ VPI->setOperand(1, Freeze);
10399+ if (UpdateResumePhis)
10400+ OrigStart->replaceUsesWithIf(Freeze, [Freeze](VPUser &U, unsigned) {
10401+ return Freeze != &U && isa<VPInstruction>(&U) &&
10402+ cast<VPInstruction>(&U)->getOpcode() ==
10403+ VPInstruction::ResumePhi;
10404+ });
10405+ }
10406+ };
10407+ AddFreezeForFindLastIVReductions(MainPlan, true);
10408+ AddFreezeForFindLastIVReductions(EpiPlan, false);
10409+
1037710410 VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
1037810411 VPValue *VectorTC = &MainPlan.getVectorTripCount();
1037910412 // If there is a suitable resume value for the canonical induction in the
@@ -10401,24 +10434,7 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
1040110434 VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
1040210435 Header->setName("vec.epilog.vector.body");
1040310436
10404- // Re-use the trip count and steps expanded for the main loop, as
10405- // skeleton creation needs it as a value that dominates both the scalar
10406- // and vector epilogue loops
10407- // TODO: This is a workaround needed for epilogue vectorization and it
10408- // should be removed once induction resume value creation is done
10409- // directly in VPlan.
10410- for (auto &R : make_early_inc_range(*Plan.getEntry())) {
10411- auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
10412- if (!ExpandR)
10413- continue;
10414- auto *ExpandedVal =
10415- Plan.getOrAddLiveIn(ExpandedSCEVs.find(ExpandR->getSCEV())->second);
10416- ExpandR->replaceAllUsesWith(ExpandedVal);
10417- if (Plan.getTripCount() == ExpandR)
10418- Plan.resetTripCount(ExpandedVal);
10419- ExpandR->eraseFromParent();
10420- }
10421-
10437+ DenseMap<Value *, Value *> ToFrozen;
1042210438 // Ensure that the start values for all header phi recipes are updated before
1042310439 // vectorizing the epilogue loop.
1042410440 for (VPRecipeBase &R : Header->phis()) {
@@ -10484,6 +10500,10 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
1048410500 ResumeV =
1048510501 Builder.CreateICmpNE(ResumeV, RdxDesc.getRecurrenceStartValue());
1048610502 } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
10503+ ToFrozen[RdxDesc.getRecurrenceStartValue()] =
10504+ cast<PHINode>(ResumeV)->getIncomingValueForBlock(
10505+ EPI.MainLoopIterationCountCheck);
10506+
1048710507 // VPReductionPHIRecipe for FindLastIV reductions requires an adjustment
1048810508 // to the resume value. The resume value is adjusted to the sentinel
1048910509 // value when the final value from the main vector loop equals the start
@@ -10492,8 +10512,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
1049210512 // variable.
1049310513 BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
1049410514 IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
10495- Value *Cmp =
10496- Builder.CreateICmpEQ( ResumeV, RdxDesc.getRecurrenceStartValue());
10515+ Value *Cmp = Builder.CreateICmpEQ(
10516+ ResumeV, ToFrozen[ RdxDesc.getRecurrenceStartValue()] );
1049710517 ResumeV =
1049810518 Builder.CreateSelect(Cmp, RdxDesc.getSentinelValue(), ResumeV);
1049910519 }
@@ -10509,6 +10529,35 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
1050910529 VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
1051010530 cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
1051110531 }
10532+
10533+ // For some VPValues in the epilogue plan we must re-use the generated IR
10534+ // values from the main plan. Replace them with live-in VPValues.
10535+ // TODO: This is a workaround needed for epilogue vectorization and it
10536+ // should be removed once induction resume value creation is done
10537+ // directly in VPlan.
10538+ for (auto &R : make_early_inc_range(*Plan.getEntry())) {
10539+ // Re-use frozen values from the main plan for Freeze VPInstructions in the
10540+ // epilogue plan. This ensures all users use the same frozen value.
10541+ auto *VPI = dyn_cast<VPInstruction>(&R);
10542+ if (VPI && VPI->getOpcode() == Instruction::Freeze) {
10543+ VPI->replaceAllUsesWith(Plan.getOrAddLiveIn(
10544+ ToFrozen.lookup(VPI->getOperand(0)->getLiveInIRValue())));
10545+ continue;
10546+ }
10547+
10548+ // Re-use the trip count and steps expanded for the main loop, as
10549+ // skeleton creation needs it as a value that dominates both the scalar
10550+ // and vector epilogue loops
10551+ auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
10552+ if (!ExpandR)
10553+ continue;
10554+ auto *ExpandedVal =
10555+ Plan.getOrAddLiveIn(ExpandedSCEVs.find(ExpandR->getSCEV())->second);
10556+ ExpandR->replaceAllUsesWith(ExpandedVal);
10557+ if (Plan.getTripCount() == ExpandR)
10558+ Plan.resetTripCount(ExpandedVal);
10559+ ExpandR->eraseFromParent();
10560+ }
1051210561}
1051310562
1051410563// Generate bypass values from the additional bypass block. Note that when the
0 commit comments