@@ -8703,6 +8703,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
8703
8703
Value *V1 = P1.dyn_cast<Value *>(), *V2 = P2.dyn_cast<Value *>();
8704
8704
unsigned CommonVF = Mask.size();
8705
8705
InstructionCost ExtraCost = 0;
8706
+ unsigned ScalarTyNumElements = getNumElements(ScalarTy);
8706
8707
auto GetNodeMinBWAffectedCost = [&](const TreeEntry &E,
8707
8708
unsigned VF) -> InstructionCost {
8708
8709
if (E.isGather() && allConstant(E.Scalars))
@@ -8743,6 +8744,15 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
8743
8744
}
8744
8745
return TTI::TCC_Free;
8745
8746
};
8747
+ auto GetVF = [&](Value *V) {
8748
+ unsigned VNumElements =
8749
+ cast<FixedVectorType>(V->getType())->getNumElements();
8750
+ assert(VNumElements > ScalarTyNumElements &&
8751
+ "the number of elements of V is not large enough");
8752
+ assert(VNumElements % ScalarTyNumElements == 0 &&
8753
+ "the number of elements of V is not a vectorized value");
8754
+ return VNumElements / ScalarTyNumElements;
8755
+ };
8746
8756
if (!V1 && !V2 && !P2.isNull()) {
8747
8757
// Shuffle 2 entry nodes.
8748
8758
const TreeEntry *E = P1.get<const TreeEntry *>();
@@ -8814,14 +8824,14 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
8814
8824
} else if (V1 && P2.isNull()) {
8815
8825
// Shuffle single vector.
8816
8826
ExtraCost += GetValueMinBWAffectedCost(V1);
8817
- CommonVF = cast<FixedVectorType> (V1->getType())->getNumElements( );
8827
+ CommonVF = GetVF (V1);
8818
8828
assert(
8819
8829
all_of(Mask,
8820
8830
[=](int Idx) { return Idx < static_cast<int>(CommonVF); }) &&
8821
8831
"All elements in mask must be less than CommonVF.");
8822
8832
} else if (V1 && !V2) {
8823
8833
// Shuffle vector and tree node.
8824
- unsigned VF = cast<FixedVectorType> (V1->getType())->getNumElements( );
8834
+ unsigned VF = GetVF (V1);
8825
8835
const TreeEntry *E2 = P2.get<const TreeEntry *>();
8826
8836
CommonVF = std::max(VF, E2->getVectorFactor());
8827
8837
assert(all_of(Mask,
@@ -8847,7 +8857,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
8847
8857
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
8848
8858
} else if (!V1 && V2) {
8849
8859
// Shuffle vector and tree node.
8850
- unsigned VF = cast<FixedVectorType> (V2->getType())->getNumElements( );
8860
+ unsigned VF = GetVF (V2);
8851
8861
const TreeEntry *E1 = P1.get<const TreeEntry *>();
8852
8862
CommonVF = std::max(VF, E1->getVectorFactor());
8853
8863
assert(all_of(Mask,
@@ -8875,9 +8885,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
8875
8885
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
8876
8886
} else {
8877
8887
assert(V1 && V2 && "Expected both vectors.");
8878
- unsigned VF = cast<FixedVectorType>(V1->getType())->getNumElements();
8879
- CommonVF =
8880
- std::max(VF, cast<FixedVectorType>(V2->getType())->getNumElements());
8888
+ unsigned VF = GetVF(V1);
8889
+ CommonVF = std::max(VF, GetVF(V2));
8881
8890
assert(all_of(Mask,
8882
8891
[=](int Idx) {
8883
8892
return Idx < 2 * static_cast<int>(CommonVF);
@@ -8895,6 +8904,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
8895
8904
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
8896
8905
}
8897
8906
}
8907
+ if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy))
8908
+ transformScalarShuffleIndiciesToVector(VecTy->getNumElements(),
8909
+ CommonMask);
8898
8910
InVectors.front() =
8899
8911
Constant::getNullValue(getWidenedType(ScalarTy, CommonMask.size()));
8900
8912
if (InVectors.size() == 2)
0 commit comments