@@ -8682,12 +8682,12 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I,
8682
8682
/// are valid so recipes can be formed later.
8683
8683
void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
8684
8684
// Find all possible partial reductions.
8685
- SmallVector<std::pair<PartialReductionChain, unsigned>>
8685
+ SmallVector<std::pair<PartialReductionChain, unsigned>, 1 >
8686
8686
PartialReductionChains;
8687
- for (const auto &[Phi, RdxDesc] : Legal->getReductionVars()) {
8688
- if (auto SR = getScaledReduction(Phi, RdxDesc.getLoopExitInstr(), Range))
8689
- PartialReductionChains.append(*SR);
8690
- }
8687
+ for (const auto &[Phi, RdxDesc] : Legal->getReductionVars())
8688
+ if (std::optional<std::pair<PartialReductionChain, unsigned>> Pair =
8689
+ getScaledReduction(Phi, RdxDesc, Range))
8690
+ PartialReductionChains.push_back(*Pair);
8691
8691
8692
8692
// A partial reduction is invalid if any of its extends are used by
8693
8693
// something that isn't another partial reduction. This is because the
@@ -8715,44 +8715,26 @@ void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
8715
8715
}
8716
8716
}
8717
8717
8718
- std::optional<SmallVector<std::pair<PartialReductionChain, unsigned>>>
8719
- VPRecipeBuilder::getScaledReduction(Instruction *PHI, Instruction *RdxExitInstr,
8718
+ std::optional<std::pair<PartialReductionChain, unsigned>>
8719
+ VPRecipeBuilder::getScaledReduction(PHINode *PHI,
8720
+ const RecurrenceDescriptor &Rdx,
8720
8721
VFRange &Range) {
8721
-
8722
- if (!CM.TheLoop->contains(RdxExitInstr))
8723
- return std::nullopt;
8724
-
8725
8722
// TODO: Allow scaling reductions when predicating. The select at
8726
8723
// the end of the loop chooses between the phi value and most recent
8727
8724
// reduction result, both of which have different VFs to the active lane
8728
8725
// mask when scaling.
8729
- if (CM.blockNeedsPredicationForAnyReason(RdxExitInstr ->getParent()))
8726
+ if (CM.blockNeedsPredicationForAnyReason(Rdx.getLoopExitInstr() ->getParent()))
8730
8727
return std::nullopt;
8731
8728
8732
- auto *Update = dyn_cast<BinaryOperator>(RdxExitInstr );
8729
+ auto *Update = dyn_cast<BinaryOperator>(Rdx.getLoopExitInstr() );
8733
8730
if (!Update)
8734
8731
return std::nullopt;
8735
8732
8736
8733
Value *Op = Update->getOperand(0);
8737
8734
Value *PhiOp = Update->getOperand(1);
8738
- if (Op == PHI)
8739
- std::swap(Op, PhiOp);
8740
-
8741
- SmallVector<std::pair<PartialReductionChain, unsigned>> Chains;
8742
-
8743
- // Try and get a scaled reduction from the first non-phi operand.
8744
- // If one is found, we use the discovered reduction instruction in
8745
- // place of the accumulator for costing.
8746
- if (auto *OpInst = dyn_cast<Instruction>(Op)) {
8747
- if (auto SR0 = getScaledReduction(PHI, OpInst, Range)) {
8748
- Chains.append(*SR0);
8749
- PHI = SR0->rbegin()->first.Reduction;
8750
-
8751
- Op = Update->getOperand(0);
8752
- PhiOp = Update->getOperand(1);
8753
- if (Op == PHI)
8754
- std::swap(Op, PhiOp);
8755
- }
8735
+ if (Op == PHI) {
8736
+ Op = Update->getOperand(1);
8737
+ PhiOp = Update->getOperand(0);
8756
8738
}
8757
8739
if (PhiOp != PHI)
8758
8740
return std::nullopt;
@@ -8775,7 +8757,7 @@ VPRecipeBuilder::getScaledReduction(Instruction *PHI, Instruction *RdxExitInstr,
8775
8757
TTI::PartialReductionExtendKind OpBExtend =
8776
8758
TargetTransformInfo::getPartialReductionExtendKind(ExtB);
8777
8759
8778
- PartialReductionChain Chain(RdxExitInstr , ExtA, ExtB, BinOp);
8760
+ PartialReductionChain Chain(Rdx.getLoopExitInstr() , ExtA, ExtB, BinOp);
8779
8761
8780
8762
unsigned TargetScaleFactor =
8781
8763
PHI->getType()->getPrimitiveSizeInBits().getKnownScalarFactor(
@@ -8790,9 +8772,9 @@ VPRecipeBuilder::getScaledReduction(Instruction *PHI, Instruction *RdxExitInstr,
8790
8772
return Cost.isValid();
8791
8773
},
8792
8774
Range))
8793
- Chains.push_back( std::make_pair(Chain, TargetScaleFactor) );
8775
+ return std::make_pair(Chain, TargetScaleFactor);
8794
8776
8795
- return Chains ;
8777
+ return std::nullopt ;
8796
8778
}
8797
8779
8798
8780
VPRecipeBase *
@@ -8887,14 +8869,12 @@ VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
8887
8869
"Unexpected number of operands for partial reduction");
8888
8870
8889
8871
VPValue *BinOp = Operands[0];
8890
- VPValue *Accumulator = Operands[1];
8891
- VPRecipeBase *BinOpRecipe = BinOp->getDefiningRecipe();
8892
- if (isa<VPReductionPHIRecipe>(BinOpRecipe) ||
8893
- isa<VPPartialReductionRecipe>(BinOpRecipe))
8894
- std::swap(BinOp, Accumulator);
8895
-
8896
- return new VPPartialReductionRecipe(Reduction->getOpcode(), BinOp,
8897
- Accumulator, Reduction);
8872
+ VPValue *Phi = Operands[1];
8873
+ if (isa<VPReductionPHIRecipe>(BinOp->getDefiningRecipe()))
8874
+ std::swap(BinOp, Phi);
8875
+
8876
+ return new VPPartialReductionRecipe(Reduction->getOpcode(), BinOp, Phi,
8877
+ Reduction);
8898
8878
}
8899
8879
8900
8880
void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
0 commit comments