Skip to content

Commit c88c281

Browse files
alexey-bataevalexey-bataev
committed
[IR]Add NumSrcElts param to is..Mask static function in ShuffleVectorInst.
Need to add NumSrcElts param to is..Mask functions in ShuffleVectorInstruction class for better mask analysis. Mask.size() not always matches the sizes of the permuted vector(s). Allows to better estimate the cost in SLP and fix uses of the functions in other cases. Differential Revision: https://reviews.llvm.org/D158449
1 parent 6ca47eb commit c88c281

File tree

14 files changed

+325
-218
lines changed

14 files changed

+325
-218
lines changed

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -935,31 +935,28 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
935935
ArrayRef<int> Mask,
936936
VectorType *Ty, int &Index,
937937
VectorType *&SubTy) const {
938-
int Limit = Mask.size() * 2;
939-
if (Mask.empty() ||
940-
// Extra check required by isSingleSourceMaskImpl function (called by
941-
// ShuffleVectorInst::isSingleSourceMask).
942-
any_of(Mask, [Limit](int I) { return I >= Limit; }))
938+
if (Mask.empty())
943939
return Kind;
940+
int NumSrcElts = Ty->getElementCount().getKnownMinValue();
944941
switch (Kind) {
945942
case TTI::SK_PermuteSingleSrc:
946-
if (ShuffleVectorInst::isReverseMask(Mask))
943+
if (ShuffleVectorInst::isReverseMask(Mask, NumSrcElts))
947944
return TTI::SK_Reverse;
948-
if (ShuffleVectorInst::isZeroEltSplatMask(Mask))
945+
if (ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts))
949946
return TTI::SK_Broadcast;
950947
break;
951948
case TTI::SK_PermuteTwoSrc: {
952949
int NumSubElts;
953950
if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
954-
Mask, Mask.size(), NumSubElts, Index)) {
951+
Mask, NumSrcElts, NumSubElts, Index)) {
955952
SubTy = FixedVectorType::get(Ty->getElementType(), NumSubElts);
956953
return TTI::SK_InsertSubvector;
957954
}
958-
if (ShuffleVectorInst::isSelectMask(Mask))
955+
if (ShuffleVectorInst::isSelectMask(Mask, NumSrcElts))
959956
return TTI::SK_Select;
960-
if (ShuffleVectorInst::isTransposeMask(Mask))
957+
if (ShuffleVectorInst::isTransposeMask(Mask, NumSrcElts))
961958
return TTI::SK_Transpose;
962-
if (ShuffleVectorInst::isSpliceMask(Mask, Index))
959+
if (ShuffleVectorInst::isSpliceMask(Mask, NumSrcElts, Index))
963960
return TTI::SK_Splice;
964961
break;
965962
}

‎llvm/include/llvm/IR/Instructions.h

Lines changed: 39 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2068,30 +2068,32 @@ class ShuffleVectorInst : public Instruction {
20682068
/// Return true if this shuffle mask chooses elements from exactly one source
20692069
/// vector.
20702070
/// Example: <7,5,undef,7>
2071-
/// This assumes that vector operands are the same length as the mask.
2072-
static bool isSingleSourceMask(ArrayRef<int> Mask);
2073-
static bool isSingleSourceMask(const Constant *Mask) {
2071+
/// This assumes that vector operands (of length \p NumSrcElts) are the same
2072+
/// length as the mask.
2073+
static bool isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts);
2074+
static bool isSingleSourceMask(const Constant *Mask, int NumSrcElts) {
20742075
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
20752076
SmallVector<int, 16> MaskAsInts;
20762077
getShuffleMask(Mask, MaskAsInts);
2077-
return isSingleSourceMask(MaskAsInts);
2078+
return isSingleSourceMask(MaskAsInts, NumSrcElts);
20782079
}
20792080

20802081
/// Return true if this shuffle chooses elements from exactly one source
20812082
/// vector without changing the length of that vector.
20822083
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
20832084
/// TODO: Optionally allow length-changing shuffles.
20842085
bool isSingleSource() const {
2085-
return !changesLength() && isSingleSourceMask(ShuffleMask);
2086+
return !changesLength() &&
2087+
isSingleSourceMask(ShuffleMask, ShuffleMask.size());
20862088
}
20872089

20882090
/// Return true if this shuffle mask chooses elements from exactly one source
20892091
/// vector without lane crossings. A shuffle using this mask is not
20902092
/// necessarily a no-op because it may change the number of elements from its
20912093
/// input vectors or it may provide demanded bits knowledge via undef lanes.
20922094
/// Example: <undef,undef,2,3>
2093-
static bool isIdentityMask(ArrayRef<int> Mask);
2094-
static bool isIdentityMask(const Constant *Mask) {
2095+
static bool isIdentityMask(ArrayRef<int> Mask, int NumSrcElts);
2096+
static bool isIdentityMask(const Constant *Mask, int NumSrcElts) {
20952097
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
20962098

20972099
// Not possible to express a shuffle mask for a scalable vector for this
@@ -2101,7 +2103,7 @@ class ShuffleVectorInst : public Instruction {
21012103

21022104
SmallVector<int, 16> MaskAsInts;
21032105
getShuffleMask(Mask, MaskAsInts);
2104-
return isIdentityMask(MaskAsInts);
2106+
return isIdentityMask(MaskAsInts, NumSrcElts);
21052107
}
21062108

21072109
/// Return true if this shuffle chooses elements from exactly one source
@@ -2114,7 +2116,7 @@ class ShuffleVectorInst : public Instruction {
21142116
if (isa<ScalableVectorType>(getType()))
21152117
return false;
21162118

2117-
return !changesLength() && isIdentityMask(ShuffleMask);
2119+
return !changesLength() && isIdentityMask(ShuffleMask, ShuffleMask.size());
21182120
}
21192121

21202122
/// Return true if this shuffle lengthens exactly one source vector with
@@ -2138,12 +2140,12 @@ class ShuffleVectorInst : public Instruction {
21382140
/// In that case, the shuffle is better classified as an identity shuffle.
21392141
/// This assumes that vector operands are the same length as the mask
21402142
/// (a length-changing shuffle can never be equivalent to a vector select).
2141-
static bool isSelectMask(ArrayRef<int> Mask);
2142-
static bool isSelectMask(const Constant *Mask) {
2143+
static bool isSelectMask(ArrayRef<int> Mask, int NumSrcElts);
2144+
static bool isSelectMask(const Constant *Mask, int NumSrcElts) {
21432145
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
21442146
SmallVector<int, 16> MaskAsInts;
21452147
getShuffleMask(Mask, MaskAsInts);
2146-
return isSelectMask(MaskAsInts);
2148+
return isSelectMask(MaskAsInts, NumSrcElts);
21472149
}
21482150

21492151
/// Return true if this shuffle chooses elements from its source vectors
@@ -2155,39 +2157,41 @@ class ShuffleVectorInst : public Instruction {
21552157
/// In that case, the shuffle is better classified as an identity shuffle.
21562158
/// TODO: Optionally allow length-changing shuffles.
21572159
bool isSelect() const {
2158-
return !changesLength() && isSelectMask(ShuffleMask);
2160+
return !changesLength() && isSelectMask(ShuffleMask, ShuffleMask.size());
21592161
}
21602162

21612163
/// Return true if this shuffle mask swaps the order of elements from exactly
21622164
/// one source vector.
21632165
/// Example: <7,6,undef,4>
2164-
/// This assumes that vector operands are the same length as the mask.
2165-
static bool isReverseMask(ArrayRef<int> Mask);
2166-
static bool isReverseMask(const Constant *Mask) {
2166+
/// This assumes that vector operands (of length \p NumSrcElts) are the same
2167+
/// length as the mask.
2168+
static bool isReverseMask(ArrayRef<int> Mask, int NumSrcElts);
2169+
static bool isReverseMask(const Constant *Mask, int NumSrcElts) {
21672170
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
21682171
SmallVector<int, 16> MaskAsInts;
21692172
getShuffleMask(Mask, MaskAsInts);
2170-
return isReverseMask(MaskAsInts);
2173+
return isReverseMask(MaskAsInts, NumSrcElts);
21712174
}
21722175

21732176
/// Return true if this shuffle swaps the order of elements from exactly
21742177
/// one source vector.
21752178
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
21762179
/// TODO: Optionally allow length-changing shuffles.
21772180
bool isReverse() const {
2178-
return !changesLength() && isReverseMask(ShuffleMask);
2181+
return !changesLength() && isReverseMask(ShuffleMask, ShuffleMask.size());
21792182
}
21802183

21812184
/// Return true if this shuffle mask chooses all elements with the same value
21822185
/// as the first element of exactly one source vector.
21832186
/// Example: <4,undef,undef,4>
2184-
/// This assumes that vector operands are the same length as the mask.
2185-
static bool isZeroEltSplatMask(ArrayRef<int> Mask);
2186-
static bool isZeroEltSplatMask(const Constant *Mask) {
2187+
/// This assumes that vector operands (of length \p NumSrcElts) are the same
2188+
/// length as the mask.
2189+
static bool isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts);
2190+
static bool isZeroEltSplatMask(const Constant *Mask, int NumSrcElts) {
21872191
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
21882192
SmallVector<int, 16> MaskAsInts;
21892193
getShuffleMask(Mask, MaskAsInts);
2190-
return isZeroEltSplatMask(MaskAsInts);
2194+
return isZeroEltSplatMask(MaskAsInts, NumSrcElts);
21912195
}
21922196

21932197
/// Return true if all elements of this shuffle are the same value as the
@@ -2197,7 +2201,8 @@ class ShuffleVectorInst : public Instruction {
21972201
/// TODO: Optionally allow length-changing shuffles.
21982202
/// TODO: Optionally allow splats from other elements.
21992203
bool isZeroEltSplat() const {
2200-
return !changesLength() && isZeroEltSplatMask(ShuffleMask);
2204+
return !changesLength() &&
2205+
isZeroEltSplatMask(ShuffleMask, ShuffleMask.size());
22012206
}
22022207

22032208
/// Return true if this shuffle mask is a transpose mask.
@@ -2232,12 +2237,12 @@ class ShuffleVectorInst : public Instruction {
22322237
/// ; Transposed matrix
22332238
/// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 >
22342239
/// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 >
2235-
static bool isTransposeMask(ArrayRef<int> Mask);
2236-
static bool isTransposeMask(const Constant *Mask) {
2240+
static bool isTransposeMask(ArrayRef<int> Mask, int NumSrcElts);
2241+
static bool isTransposeMask(const Constant *Mask, int NumSrcElts) {
22372242
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
22382243
SmallVector<int, 16> MaskAsInts;
22392244
getShuffleMask(Mask, MaskAsInts);
2240-
return isTransposeMask(MaskAsInts);
2245+
return isTransposeMask(MaskAsInts, NumSrcElts);
22412246
}
22422247

22432248
/// Return true if this shuffle transposes the elements of its inputs without
@@ -2246,27 +2251,30 @@ class ShuffleVectorInst : public Instruction {
22462251
/// exact specification.
22472252
/// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
22482253
bool isTranspose() const {
2249-
return !changesLength() && isTransposeMask(ShuffleMask);
2254+
return !changesLength() && isTransposeMask(ShuffleMask, ShuffleMask.size());
22502255
}
22512256

22522257
/// Return true if this shuffle mask is a splice mask, concatenating the two
22532258
/// inputs together and then extracts an original width vector starting from
22542259
/// the splice index.
22552260
/// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
2256-
static bool isSpliceMask(ArrayRef<int> Mask, int &Index);
2257-
static bool isSpliceMask(const Constant *Mask, int &Index) {
2261+
/// This assumes that vector operands (of length \p NumSrcElts) are the same
2262+
/// length as the mask.
2263+
static bool isSpliceMask(ArrayRef<int> Mask, int NumSrcElts, int &Index);
2264+
static bool isSpliceMask(const Constant *Mask, int NumSrcElts, int &Index) {
22582265
assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
22592266
SmallVector<int, 16> MaskAsInts;
22602267
getShuffleMask(Mask, MaskAsInts);
2261-
return isSpliceMask(MaskAsInts, Index);
2268+
return isSpliceMask(MaskAsInts, NumSrcElts, Index);
22622269
}
22632270

22642271
/// Return true if this shuffle splices two inputs without changing the length
22652272
/// of the vectors. This operation concatenates the two inputs together and
22662273
/// then extracts an original width vector starting from the splice index.
22672274
/// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
22682275
bool isSplice(int &Index) const {
2269-
return !changesLength() && isSpliceMask(ShuffleMask, Index);
2276+
return !changesLength() &&
2277+
isSpliceMask(ShuffleMask, ShuffleMask.size(), Index);
22702278
}
22712279

22722280
/// Return true if this shuffle mask is an extract subvector mask.

‎llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24119,7 +24119,7 @@ static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N,
2411924119

2412024120
// Profitability check: only deal with extractions from the first subvector
2412124121
// unless the mask becomes an identity mask.
24122-
if (!ShuffleVectorInst::isIdentityMask(NewMask) ||
24122+
if (!ShuffleVectorInst::isIdentityMask(NewMask, NewMask.size()) ||
2412324123
any_of(NewMask, [](int M) { return M < 0; }))
2412424124
for (auto &DemandedSubvector : DemandedSubvectors)
2412524125
if (DemandedSubvector.second != 0)

‎llvm/lib/IR/Instructions.cpp

Lines changed: 44 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2136,10 +2136,10 @@ static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
21362136
return UsesLHS || UsesRHS;
21372137
}
21382138

2139-
bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask) {
2139+
bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts) {
21402140
// We don't have vector operand size information, so assume operands are the
21412141
// same size as the mask.
2142-
return isSingleSourceMaskImpl(Mask, Mask.size());
2142+
return isSingleSourceMaskImpl(Mask, NumSrcElts);
21432143
}
21442144

21452145
static bool isIdentityMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
@@ -2154,65 +2154,75 @@ static bool isIdentityMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
21542154
return true;
21552155
}
21562156

2157-
bool ShuffleVectorInst::isIdentityMask(ArrayRef<int> Mask) {
2157+
bool ShuffleVectorInst::isIdentityMask(ArrayRef<int> Mask, int NumSrcElts) {
2158+
if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2159+
return false;
21582160
// We don't have vector operand size information, so assume operands are the
21592161
// same size as the mask.
2160-
return isIdentityMaskImpl(Mask, Mask.size());
2162+
return isIdentityMaskImpl(Mask, NumSrcElts);
21612163
}
21622164

2163-
bool ShuffleVectorInst::isReverseMask(ArrayRef<int> Mask) {
2164-
if (!isSingleSourceMask(Mask))
2165+
bool ShuffleVectorInst::isReverseMask(ArrayRef<int> Mask, int NumSrcElts) {
2166+
if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2167+
return false;
2168+
if (!isSingleSourceMask(Mask, NumSrcElts))
21652169
return false;
21662170

21672171
// The number of elements in the mask must be at least 2.
2168-
int NumElts = Mask.size();
2169-
if (NumElts < 2)
2172+
if (NumSrcElts < 2)
21702173
return false;
21712174

2172-
for (int i = 0; i < NumElts; ++i) {
2173-
if (Mask[i] == -1)
2175+
for (int I = 0, E = Mask.size(); I < E; ++I) {
2176+
if (Mask[I] == -1)
21742177
continue;
2175-
if (Mask[i] != (NumElts - 1 - i) && Mask[i] != (NumElts + NumElts - 1 - i))
2178+
if (Mask[I] != (NumSrcElts - 1 - I) &&
2179+
Mask[I] != (NumSrcElts + NumSrcElts - 1 - I))
21762180
return false;
21772181
}
21782182
return true;
21792183
}
21802184

2181-
bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef<int> Mask) {
2182-
if (!isSingleSourceMask(Mask))
2185+
bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts) {
2186+
if (Mask.size() != static_cast<unsigned>(NumSrcElts))
21832187
return false;
2184-
for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) {
2185-
if (Mask[i] == -1)
2188+
if (!isSingleSourceMask(Mask, NumSrcElts))
2189+
return false;
2190+
for (int I = 0, E = Mask.size(); I < E; ++I) {
2191+
if (Mask[I] == -1)
21862192
continue;
2187-
if (Mask[i] != 0 && Mask[i] != NumElts)
2193+
if (Mask[I] != 0 && Mask[I] != NumSrcElts)
21882194
return false;
21892195
}
21902196
return true;
21912197
}
21922198

2193-
bool ShuffleVectorInst::isSelectMask(ArrayRef<int> Mask) {
2199+
bool ShuffleVectorInst::isSelectMask(ArrayRef<int> Mask, int NumSrcElts) {
2200+
if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2201+
return false;
21942202
// Select is differentiated from identity. It requires using both sources.
2195-
if (isSingleSourceMask(Mask))
2203+
if (isSingleSourceMask(Mask, NumSrcElts))
21962204
return false;
2197-
for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) {
2198-
if (Mask[i] == -1)
2205+
for (int I = 0, E = Mask.size(); I < E; ++I) {
2206+
if (Mask[I] == -1)
21992207
continue;
2200-
if (Mask[i] != i && Mask[i] != (NumElts + i))
2208+
if (Mask[I] != I && Mask[I] != (NumSrcElts + I))
22012209
return false;
22022210
}
22032211
return true;
22042212
}
22052213

2206-
bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask) {
2214+
bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask, int NumSrcElts) {
22072215
// Example masks that will return true:
22082216
// v1 = <a, b, c, d>
22092217
// v2 = <e, f, g, h>
22102218
// trn1 = shufflevector v1, v2 <0, 4, 2, 6> = <a, e, c, g>
22112219
// trn2 = shufflevector v1, v2 <1, 5, 3, 7> = <b, f, d, h>
22122220

2221+
if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2222+
return false;
22132223
// 1. The number of elements in the mask must be a power-of-2 and at least 2.
2214-
int NumElts = Mask.size();
2215-
if (NumElts < 2 || !isPowerOf2_32(NumElts))
2224+
int Sz = Mask.size();
2225+
if (Sz < 2 || !isPowerOf2_32(Sz))
22162226
return false;
22172227

22182228
// 2. The first element of the mask must be either a 0 or a 1.
@@ -2221,23 +2231,26 @@ bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask) {
22212231

22222232
// 3. The difference between the first 2 elements must be equal to the
22232233
// number of elements in the mask.
2224-
if ((Mask[1] - Mask[0]) != NumElts)
2234+
if ((Mask[1] - Mask[0]) != NumSrcElts)
22252235
return false;
22262236

22272237
// 4. The difference between consecutive even-numbered and odd-numbered
22282238
// elements must be equal to 2.
2229-
for (int i = 2; i < NumElts; ++i) {
2230-
int MaskEltVal = Mask[i];
2239+
for (int I = 2; I < Sz; ++I) {
2240+
int MaskEltVal = Mask[I];
22312241
if (MaskEltVal == -1)
22322242
return false;
2233-
int MaskEltPrevVal = Mask[i - 2];
2243+
int MaskEltPrevVal = Mask[I - 2];
22342244
if (MaskEltVal - MaskEltPrevVal != 2)
22352245
return false;
22362246
}
22372247
return true;
22382248
}
22392249

2240-
bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int &Index) {
2250+
bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int NumSrcElts,
2251+
int &Index) {
2252+
if (Mask.size() != static_cast<unsigned>(NumSrcElts))
2253+
return false;
22412254
// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>
22422255
int StartIndex = -1;
22432256
for (int I = 0, E = Mask.size(); I != E; ++I) {
@@ -2248,7 +2261,7 @@ bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int &Index) {
22482261
if (StartIndex == -1) {
22492262
// Don't support a StartIndex that begins in the second input, or if the
22502263
// first non-undef index would access below the StartIndex.
2251-
if (MaskEltVal < I || E <= (MaskEltVal - I))
2264+
if (MaskEltVal < I || NumSrcElts <= (MaskEltVal - I))
22522265
return false;
22532266

22542267
StartIndex = MaskEltVal - I;
@@ -2543,7 +2556,7 @@ bool ShuffleVectorInst::isOneUseSingleSourceMask(int VF) const {
25432556
// case.
25442557
if (isa<ScalableVectorType>(getType()))
25452558
return false;
2546-
if (!isSingleSourceMask(ShuffleMask))
2559+
if (!isSingleSourceMask(ShuffleMask, VF))
25472560
return false;
25482561

25492562
return isOneUseSingleSourceMask(ShuffleMask, VF);

0 commit comments

Comments
 (0)