Skip to content

Removed usage of BMI2 pdep with SSE2 alternative #19009

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
5 commits merged into from
Feb 26, 2020
Merged

Removed usage of BMI2 pdep with SSE2 alternative #19009

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2821a19
SSE2 alternative to BMI2 ParallelBitDeposit
gfoidl Feb 11, 2020
2aee840
Codegen tuning
gfoidl Feb 11, 2020
f14d3b0
Keep zero vector in register
gfoidl Feb 11, 2020
d77a1de
Better 64-bit BMI2 alternative
gfoidl Feb 11, 2020
8f9fe41
Removed BMI2
gfoidl Feb 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 45 additions & 42 deletions src/Shared/ServerInfrastructure/StringUtilities.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,25 @@ public static unsafe bool TryGetAsciiString(byte* input, char* output, int count

Debug.Assert((long)end >= Vector256<sbyte>.Count);

// PERF: so the JIT can reuse the zero from a register
Vector128<sbyte> zero = Vector128<sbyte>.Zero;

if (Sse2.IsSupported)
{
if (Avx2.IsSupported && input <= end - Vector256<sbyte>.Count)
{
Vector256<sbyte> zero = Vector256<sbyte>.Zero;
Vector256<sbyte> avxZero = Vector256<sbyte>.Zero;

do
{
var vector = Avx.LoadVector256(input).AsSByte();
if (!CheckBytesInAsciiRange(vector, zero))
if (!CheckBytesInAsciiRange(vector, avxZero))
{
return false;
}

var tmp0 = Avx2.UnpackLow(vector, zero);
var tmp1 = Avx2.UnpackHigh(vector, zero);
var tmp0 = Avx2.UnpackLow(vector, avxZero);
var tmp1 = Avx2.UnpackHigh(vector, avxZero);

// Bring into the right order
var out0 = Avx2.Permute2x128(tmp0, tmp1, 0x20);
Expand All @@ -60,8 +63,6 @@ public static unsafe bool TryGetAsciiString(byte* input, char* output, int count

if (input <= end - Vector128<sbyte>.Count)
{
Vector128<sbyte> zero = Vector128<sbyte>.Zero;

do
{
var vector = Sse2.LoadVector128(input).AsSByte();
Expand Down Expand Up @@ -122,11 +123,12 @@ out Unsafe.AsRef<Vector<short>>(output),
return false;
}

if (Bmi2.X64.IsSupported)
// BMI2 could be used, but this variant is faster on both Intel and AMD.
if (Sse2.X64.IsSupported)
{
// BMI2 will work regardless of the processor's endianness.
((ulong*)output)[0] = Bmi2.X64.ParallelBitDeposit((ulong)value, 0x00FF00FF_00FF00FFul);
((ulong*)output)[1] = Bmi2.X64.ParallelBitDeposit((ulong)(value >> 32), 0x00FF00FF_00FF00FFul);
Vector128<sbyte> vecNarrow = Sse2.X64.ConvertScalarToVector128Int64(value).AsSByte();
Vector128<ulong> vecWide = Sse2.UnpackLow(vecNarrow, zero).AsUInt64();
Sse2.Store((ulong*)output, vecWide);
}
else
{
Expand All @@ -152,19 +154,7 @@ out Unsafe.AsRef<Vector<short>>(output),
return false;
}

if (Bmi2.IsSupported)
{
// BMI2 will work regardless of the processor's endianness.
((uint*)output)[0] = Bmi2.ParallelBitDeposit((uint)value, 0x00FF00FFu);
((uint*)output)[1] = Bmi2.ParallelBitDeposit((uint)(value >> 16), 0x00FF00FFu);
}
else
{
output[0] = (char)input[0];
output[1] = (char)input[1];
output[2] = (char)input[2];
output[3] = (char)input[3];
}
WidenFourAsciiBytesToUtf16AndWriteToBuffer(output, input, value, zero);

input += sizeof(int);
output += sizeof(int);
Expand All @@ -181,19 +171,7 @@ out Unsafe.AsRef<Vector<short>>(output),
return false;
}

if (Bmi2.IsSupported)
{
// BMI2 will work regardless of the processor's endianness.
((uint*)output)[0] = Bmi2.ParallelBitDeposit((uint)value, 0x00FF00FFu);
((uint*)output)[1] = Bmi2.ParallelBitDeposit((uint)(value >> 16), 0x00FF00FFu);
}
else
{
output[0] = (char)input[0];
output[1] = (char)input[1];
output[2] = (char)input[2];
output[3] = (char)input[3];
}
WidenFourAsciiBytesToUtf16AndWriteToBuffer(output, input, value, zero);

input += sizeof(int);
output += sizeof(int);
Expand Down Expand Up @@ -378,6 +356,25 @@ ref Unsafe.Add(ref str, offset),
return false;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void WidenFourAsciiBytesToUtf16AndWriteToBuffer(char* output, byte* input, int value, Vector128<sbyte> zero)
{
// BMI2 could be used, but this variant is faster on both Intel and AMD.
if (Sse2.X64.IsSupported)
{
Vector128<sbyte> vecNarrow = Sse2.ConvertScalarToVector128Int32(value).AsSByte();
Vector128<ulong> vecWide = Sse2.UnpackLow(vecNarrow, zero).AsUInt64();
Unsafe.WriteUnaligned(output, Sse2.X64.ConvertToUInt64(vecWide));
}
else
{
output[0] = (char)input[0];
output[1] = (char)input[1];
output[2] = (char)input[2];
output[3] = (char)input[3];
}
}

/// <summary>
/// Given a DWORD which represents a buffer of 4 bytes, widens the buffer into 4 WORDs and
/// compares them to the WORD buffer with machine endianness.
Expand All @@ -390,11 +387,13 @@ private static bool WidenFourAsciiBytesToUtf16AndCompareToChars(ref char charSta
return false;
}

if (Bmi2.X64.IsSupported)
// BMI2 could be used, but this variant is faster on both Intel and AMD.
if (Sse2.X64.IsSupported)
{
// BMI2 will work regardless of the processor's endianness.
Vector128<byte> vecNarrow = Sse2.ConvertScalarToVector128UInt32(value).AsByte();
Vector128<ulong> vecWide = Sse2.UnpackLow(vecNarrow, Vector128<byte>.Zero).AsUInt64();
return Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<char, byte>(ref charStart)) ==
Bmi2.X64.ParallelBitDeposit(value, 0x00FF00FF_00FF00FFul);
Sse2.X64.ConvertToUInt64(vecWide);
}
else
{
Expand Down Expand Up @@ -427,11 +426,13 @@ private static bool WidenTwoAsciiBytesToUtf16AndCompareToChars(ref char charStar
return false;
}

if (Bmi2.IsSupported)
// BMI2 could be used, but this variant is faster on both Intel and AMD.
if (Sse2.IsSupported)
{
// BMI2 will work regardless of the processor's endianness.
Vector128<byte> vecNarrow = Sse2.ConvertScalarToVector128UInt32(value).AsByte();
Vector128<uint> vecWide = Sse2.UnpackLow(vecNarrow, Vector128<byte>.Zero).AsUInt32();
return Unsafe.ReadUnaligned<uint>(ref Unsafe.As<char, byte>(ref charStart)) ==
Bmi2.ParallelBitDeposit(value, 0x00FF00FFu);
Sse2.ConvertToUInt32(vecWide);
}
else
{
Expand Down Expand Up @@ -560,12 +561,14 @@ private static bool CheckBytesInAsciiRange(long check)
return (((check - 0x0101010101010101L) | check) & HighBits) == 0;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This didn't inline always in some constellations. So I'll force the JIT to do so 😉

private static bool CheckBytesInAsciiRange(int check)
{
const int HighBits = unchecked((int)0x80808080);
return (((check - 0x01010101) | check) & HighBits) == 0;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool CheckBytesInAsciiRange(short check)
{
const short HighBits = unchecked((short)0x8080);
Expand Down