Skip to content

Commit 8be5726

Browse files
kunalspathakkunalspathak
committed
Revert "Arm64 SVE: Optimise zero/allbits vectors the same as masks (dotnet#115566)"
This reverts commit 9fe86ca.
1 parent f124c0e commit 8be5726

18 files changed

+172
-1216
lines changed

‎src/coreclr/jit/codegenarm64.cpp‎

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2338,50 +2338,6 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre
23382338

23392339
break;
23402340
}
2341-
2342-
case GT_CNS_MSK:
2343-
{
2344-
GenTreeMskCon* mask = tree->AsMskCon();
2345-
emitter* emit = GetEmitter();
2346-
2347-
// Try every type until a match is found
2348-
2349-
if (mask->IsZero())
2350-
{
2351-
emit->emitInsSve_R(INS_sve_pfalse, EA_SCALABLE, targetReg, INS_OPTS_SCALABLE_B);
2352-
break;
2353-
}
2354-
2355-
insOpts opt = INS_OPTS_SCALABLE_B;
2356-
SveMaskPattern pat = EvaluateSimdMaskToPattern<simd16_t>(TYP_BYTE, mask->gtSimdMaskVal);
2357-
2358-
if (pat == SveMaskPatternNone)
2359-
{
2360-
opt = INS_OPTS_SCALABLE_H;
2361-
pat = EvaluateSimdMaskToPattern<simd16_t>(TYP_SHORT, mask->gtSimdMaskVal);
2362-
}
2363-
2364-
if (pat == SveMaskPatternNone)
2365-
{
2366-
opt = INS_OPTS_SCALABLE_S;
2367-
pat = EvaluateSimdMaskToPattern<simd16_t>(TYP_INT, mask->gtSimdMaskVal);
2368-
}
2369-
2370-
if (pat == SveMaskPatternNone)
2371-
{
2372-
opt = INS_OPTS_SCALABLE_D;
2373-
pat = EvaluateSimdMaskToPattern<simd16_t>(TYP_LONG, mask->gtSimdMaskVal);
2374-
}
2375-
2376-
// Should only ever create constant masks for valid patterns.
2377-
if (pat == SveMaskPatternNone)
2378-
{
2379-
unreached();
2380-
}
2381-
2382-
emit->emitIns_R_PATTERN(INS_sve_ptrue, EA_SCALABLE, targetReg, opt, (insSvePattern)pat);
2383-
break;
2384-
}
23852341
#endif // FEATURE_SIMD
23862342

23872343
default:

‎src/coreclr/jit/compiler.h‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3146,8 +3146,8 @@ class Compiler
31463146
var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
31473147

31483148
#if defined(TARGET_ARM64)
3149-
GenTree* gtNewSimdAllTrueMaskNode(CorInfoType simdBaseJitType);
3150-
GenTree* gtNewSimdFalseMaskByteNode();
3149+
GenTree* gtNewSimdAllTrueMaskNode(CorInfoType simdBaseJitType, unsigned simdSize);
3150+
GenTree* gtNewSimdFalseMaskByteNode(unsigned simdSize);
31513151
#endif
31523152

31533153
GenTree* gtNewSimdBinOpNode(genTreeOps op,
@@ -3707,7 +3707,6 @@ class Compiler
37073707

37083708
#if defined(FEATURE_HW_INTRINSICS)
37093709
GenTree* gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree);
3710-
GenTreeMskCon* gtFoldExprConvertVecCnsToMask(GenTreeHWIntrinsic* tree, GenTreeVecCon* vecCon);
37113710
#endif // FEATURE_HW_INTRINSICS
37123711

37133712
// Options to control behavior of gtTryRemoveBoxUpstreamEffects

‎src/coreclr/jit/gentree.cpp‎

Lines changed: 53 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -21945,8 +21945,8 @@ GenTree* Compiler::gtNewSimdCvtVectorToMaskNode(var_types type,
2194521945
#if defined(TARGET_XARCH)
2194621946
return gtNewSimdHWIntrinsicNode(TYP_MASK, op1, NI_AVX512_ConvertVectorToMask, simdBaseJitType, simdSize);
2194721947
#elif defined(TARGET_ARM64)
21948-
// ConvertVectorToMask uses cmpne which requires an embedded mask.
21949-
GenTree* trueMask = gtNewSimdHWIntrinsicNode(TYP_MASK, NI_Sve_ConversionTrueMask, simdBaseJitType, simdSize);
21948+
// We use cmpne which requires an embedded mask.
21949+
GenTree* trueMask = gtNewSimdAllTrueMaskNode(simdBaseJitType, simdSize);
2195021950
return gtNewSimdHWIntrinsicNode(TYP_MASK, trueMask, op1, NI_Sve_ConvertVectorToMask, simdBaseJitType, simdSize);
2195121951
#else
2195221952
#error Unsupported platform
@@ -32028,7 +32028,6 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3202832028
}
3202932029

3203032030
#if defined(FEATURE_MASKED_HW_INTRINSICS)
32031-
// Fold ConvertMaskToVector(ConvertVectorToMask(vec)) to vec
3203232031
if (tree->OperIsConvertMaskToVector())
3203332032
{
3203432033
GenTree* op = op1;
@@ -32061,7 +32060,6 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3206132060
}
3206232061
}
3206332062

32064-
// Fold ConvertVectorToMask(ConvertMaskToVector(mask)) to mask
3206532063
if (tree->OperIsConvertVectorToMask())
3206632064
{
3206732065
GenTree* op = op1;
@@ -32070,9 +32068,11 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3207032068
#if defined(TARGET_XARCH)
3207132069
tryHandle = op->OperIsHWIntrinsic();
3207232070
#elif defined(TARGET_ARM64)
32073-
assert(op->OperIsHWIntrinsic(NI_Sve_ConversionTrueMask));
32074-
op = op2;
32075-
tryHandle = op->OperIsHWIntrinsic();
32071+
if (op->OperIsHWIntrinsic(NI_Sve_CreateTrueMaskAll))
32072+
{
32073+
op = op2;
32074+
tryHandle = op->OperIsHWIntrinsic();
32075+
}
3207632076
#endif // TARGET_ARM64
3207732077

3207832078
if (tryHandle)
@@ -32158,12 +32158,53 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3215832158

3215932159
resultNode = gtNewVconNode(retType, &simdVal);
3216032160
}
32161-
#if defined(TARGET_XARCH)
3216232161
else if (tree->OperIsConvertVectorToMask())
3216332162
{
32164-
resultNode = gtFoldExprConvertVecCnsToMask(tree, cnsNode->AsVecCon());
32165-
}
32163+
GenTreeVecCon* vecCon = cnsNode->AsVecCon();
32164+
GenTreeMskCon* mskCon = gtNewMskConNode(retType);
32165+
32166+
switch (vecCon->TypeGet())
32167+
{
32168+
case TYP_SIMD8:
32169+
{
32170+
EvaluateSimdCvtVectorToMask<simd8_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd8Val);
32171+
break;
32172+
}
32173+
32174+
case TYP_SIMD12:
32175+
{
32176+
EvaluateSimdCvtVectorToMask<simd12_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd12Val);
32177+
break;
32178+
}
32179+
32180+
case TYP_SIMD16:
32181+
{
32182+
EvaluateSimdCvtVectorToMask<simd16_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd16Val);
32183+
break;
32184+
}
32185+
32186+
#if defined(TARGET_XARCH)
32187+
case TYP_SIMD32:
32188+
{
32189+
EvaluateSimdCvtVectorToMask<simd32_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd32Val);
32190+
break;
32191+
}
32192+
32193+
case TYP_SIMD64:
32194+
{
32195+
EvaluateSimdCvtVectorToMask<simd64_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd64Val);
32196+
break;
32197+
}
3216632198
#endif // TARGET_XARCH
32199+
32200+
default:
32201+
{
32202+
unreached();
32203+
}
32204+
}
32205+
32206+
resultNode = mskCon;
32207+
}
3216732208
#endif // FEATURE_MASKED_HW_INTRINSICS
3216832209
else
3216932210
{
@@ -33006,10 +33047,6 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3300633047
switch (ni)
3300733048
{
3300833049
#ifdef TARGET_ARM64
33009-
case NI_Sve_ConvertVectorToMask:
33010-
resultNode = gtFoldExprConvertVecCnsToMask(tree, cnsNode->AsVecCon());
33011-
break;
33012-
3301333050
case NI_AdvSimd_MultiplyByScalar:
3301433051
case NI_AdvSimd_Arm64_MultiplyByScalar:
3301533052
{
@@ -33151,18 +33188,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3315133188
break;
3315233189
}
3315333190

33154-
#if defined(TARGET_ARM64)
33155-
if (ni == NI_Sve_ConditionalSelect)
33156-
{
33157-
assert(!op1->IsVectorAllBitsSet() && !op1->IsVectorZero());
33158-
}
33159-
else
33160-
{
33161-
assert(!op1->IsTrueMask(simdBaseType) && !op1->IsFalseMask());
33162-
}
33163-
#endif
33164-
33165-
if (op1->IsVectorAllBitsSet() || op1->IsTrueMask(simdBaseType))
33191+
if (op1->IsVectorAllBitsSet() || op1->IsMaskAllBitsSet())
3316633192
{
3316733193
if ((op3->gtFlags & GTF_SIDE_EFFECT) != 0)
3316833194
{
@@ -33176,7 +33202,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3317633202
return op2;
3317733203
}
3317833204

33179-
if (op1->IsVectorZero() || op1->IsFalseMask())
33205+
if (op1->IsVectorZero())
3318033206
{
3318133207
return gtWrapWithSideEffects(op3, op2, GTF_ALL_EFFECT);
3318233208
}
@@ -33228,70 +33254,6 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3322833254
}
3322933255
return resultNode;
3323033256
}
33231-
33232-
//------------------------------------------------------------------------------
33233-
// gtFoldExprConvertVecCnsToMask: Folds a constant vector plus conversion to
33234-
// mask into a constant mask.
33235-
//
33236-
// Arguments:
33237-
// tree - The convert vector to mask node
33238-
// vecCon - The vector constant converted by the convert
33239-
//
33240-
// Return Value:
33241-
// Returns a constant mask
33242-
//
33243-
GenTreeMskCon* Compiler::gtFoldExprConvertVecCnsToMask(GenTreeHWIntrinsic* tree, GenTreeVecCon* vecCon)
33244-
{
33245-
assert(tree->OperIsConvertVectorToMask());
33246-
assert(vecCon == tree->Op(1) || vecCon == tree->Op(2));
33247-
33248-
var_types retType = tree->TypeGet();
33249-
var_types simdBaseType = tree->GetSimdBaseType();
33250-
GenTreeMskCon* mskCon = gtNewMskConNode(retType);
33251-
33252-
switch (vecCon->TypeGet())
33253-
{
33254-
case TYP_SIMD8:
33255-
{
33256-
EvaluateSimdCvtVectorToMask<simd8_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd8Val);
33257-
break;
33258-
}
33259-
33260-
case TYP_SIMD12:
33261-
{
33262-
EvaluateSimdCvtVectorToMask<simd12_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd12Val);
33263-
break;
33264-
}
33265-
33266-
case TYP_SIMD16:
33267-
{
33268-
EvaluateSimdCvtVectorToMask<simd16_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd16Val);
33269-
break;
33270-
}
33271-
33272-
#if defined(TARGET_XARCH)
33273-
case TYP_SIMD32:
33274-
{
33275-
EvaluateSimdCvtVectorToMask<simd32_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd32Val);
33276-
break;
33277-
}
33278-
33279-
case TYP_SIMD64:
33280-
{
33281-
EvaluateSimdCvtVectorToMask<simd64_t>(simdBaseType, &mskCon->gtSimdMaskVal, vecCon->gtSimd64Val);
33282-
break;
33283-
}
33284-
#endif // TARGET_XARCH
33285-
33286-
default:
33287-
{
33288-
unreached();
33289-
}
33290-
}
33291-
33292-
return mskCon;
33293-
}
33294-
3329533257
#endif // FEATURE_HW_INTRINSICS
3329633258

3329733259
//------------------------------------------------------------------------

‎src/coreclr/jit/gentree.h‎

Lines changed: 36 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1802,8 +1802,8 @@ struct GenTree
18021802
inline bool IsVectorCreate() const;
18031803
inline bool IsVectorAllBitsSet() const;
18041804
inline bool IsVectorBroadcast(var_types simdBaseType) const;
1805-
inline bool IsTrueMask(var_types simdBaseType) const;
1806-
inline bool IsFalseMask() const;
1805+
inline bool IsMaskAllBitsSet() const;
1806+
inline bool IsMaskZero() const;
18071807

18081808
inline uint64_t GetIntegralVectorConstElement(size_t index, var_types simdBaseType);
18091809

@@ -9550,46 +9550,54 @@ inline bool GenTree::IsVectorBroadcast(var_types simdBaseType) const
95509550
return false;
95519551
}
95529552

9553-
//------------------------------------------------------------------------
9554-
// IsTrueMask: Is the given node a true mask
9555-
//
9556-
// Arguments:
9557-
// simdBaseType - the base type of the mask
9558-
//
9559-
// Returns true if the node is a true mask for the given simdBaseType.
9560-
//
9561-
// Note that a byte true mask (1111...) is different to an int true mask
9562-
// (10001000...), therefore the simdBaseType of the mask needs to be
9563-
// taken into account.
9564-
//
9565-
inline bool GenTree::IsTrueMask(var_types simdBaseType) const
9553+
inline bool GenTree::IsMaskAllBitsSet() const
95669554
{
95679555
#ifdef TARGET_ARM64
9568-
// TODO-SVE: For agnostic VL, vector type may not be simd16_t
9556+
static_assert_no_msg(AreContiguous(NI_Sve_CreateTrueMaskByte, NI_Sve_CreateTrueMaskDouble,
9557+
NI_Sve_CreateTrueMaskInt16, NI_Sve_CreateTrueMaskInt32,
9558+
NI_Sve_CreateTrueMaskInt64, NI_Sve_CreateTrueMaskSByte,
9559+
NI_Sve_CreateTrueMaskSingle, NI_Sve_CreateTrueMaskUInt16,
9560+
NI_Sve_CreateTrueMaskUInt32, NI_Sve_CreateTrueMaskUInt64));
95699561

9570-
if (IsCnsMsk())
9562+
if (OperIsHWIntrinsic())
95719563
{
9572-
return SveMaskPatternAll == EvaluateSimdMaskToPattern<simd16_t>(simdBaseType, AsMskCon()->gtSimdMaskVal);
9564+
NamedIntrinsic id = AsHWIntrinsic()->GetHWIntrinsicId();
9565+
if (id == NI_Sve_ConvertMaskToVector)
9566+
{
9567+
GenTree* op1 = AsHWIntrinsic()->Op(1);
9568+
assert(op1->OperIsHWIntrinsic());
9569+
id = op1->AsHWIntrinsic()->GetHWIntrinsicId();
9570+
}
9571+
return ((id == NI_Sve_CreateTrueMaskAll) ||
9572+
((id >= NI_Sve_CreateTrueMaskByte) && (id <= NI_Sve_CreateTrueMaskUInt64)));
95739573
}
9574-
#endif
95759574

9575+
#endif
95769576
return false;
95779577
}
95789578

9579-
//------------------------------------------------------------------------
9580-
// IsFalseMask: Is the given node a false mask
9581-
//
9582-
// Returns true if the node is a false mask, ie all zeros
9583-
//
9584-
inline bool GenTree::IsFalseMask() const
9579+
inline bool GenTree::IsMaskZero() const
95859580
{
95869581
#ifdef TARGET_ARM64
9587-
if (IsCnsMsk())
9582+
static_assert_no_msg(AreContiguous(NI_Sve_CreateFalseMaskByte, NI_Sve_CreateFalseMaskDouble,
9583+
NI_Sve_CreateFalseMaskInt16, NI_Sve_CreateFalseMaskInt32,
9584+
NI_Sve_CreateFalseMaskInt64, NI_Sve_CreateFalseMaskSByte,
9585+
NI_Sve_CreateFalseMaskSingle, NI_Sve_CreateFalseMaskUInt16,
9586+
NI_Sve_CreateFalseMaskUInt32, NI_Sve_CreateFalseMaskUInt64));
9587+
9588+
if (OperIsHWIntrinsic())
95889589
{
9589-
return AsMskCon()->IsZero();
9590+
NamedIntrinsic id = AsHWIntrinsic()->GetHWIntrinsicId();
9591+
if (id == NI_Sve_ConvertMaskToVector)
9592+
{
9593+
GenTree* op1 = AsHWIntrinsic()->Op(1);
9594+
assert(op1->OperIsHWIntrinsic());
9595+
id = op1->AsHWIntrinsic()->GetHWIntrinsicId();
9596+
}
9597+
return ((id >= NI_Sve_CreateFalseMaskByte) && (id <= NI_Sve_CreateFalseMaskUInt64));
95909598
}
9591-
#endif
95929599

9600+
#endif
95939601
return false;
95949602
}
95959603

0 commit comments

Comments
 (0)