Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Commit 177e7d9

Browse files
benaadamsbenaadams
committed
Faster CopyFromAscii
Copy ulong per loop, less variables, split multiblock into own path
1 parent f32058c commit 177e7d9

File tree

2 files changed

+148
-43
lines changed

2 files changed

+148
-43
lines changed

‎src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs

Lines changed: 72 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ public struct MemoryPoolIterator
2121
0x02ul << 40 |
2222
0x01ul << 48 ) + 1;
2323

24+
const int Shift16Shift24 = 256 * 256 * 256 + 256 * 256;
25+
const int Shift8Identity = 256 + 1;
26+
2427
private static readonly int _vectorSpan = Vector<byte>.Count;
2528

2629
private MemoryPoolBlock _block;
@@ -1020,64 +1023,90 @@ public void CopyFrom(byte[] data, int offset, int count)
10201023
public unsafe void CopyFromAscii(string data)
10211024
{
10221025
var block = _block;
1023-
if (block == null)
1026+
if (block != null)
10241027
{
1025-
return;
1026-
}
1027-
1028-
Debug.Assert(block.Next == null);
1029-
Debug.Assert(block.End == _index);
1028+
Debug.Assert(block.Next == null);
1029+
Debug.Assert(block.End == _index);
10301030

1031-
var pool = block.Pool;
1032-
var blockIndex = _index;
1033-
var length = data.Length;
1031+
fixed (char* pData = data)
1032+
{
1033+
var input = pData;
1034+
var blockIndex = _index;
1035+
var length = data.Length;
10341036

1035-
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
1036-
var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
1037+
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
1038+
var toCopy = Math.Min(length, bytesLeftInBlock);
1039+
var toCopyULong = toCopy & ~0x3;
1040+
var output = (block.DataFixedPtr + block.End);
10371041

1038-
fixed (char* pData = data)
1039-
{
1040-
var input = pData;
1041-
var inputEnd = pData + length;
1042-
var inputEndMinusSpan = inputEnd - 3;
1042+
blockIndex += toCopy;
10431043

1044-
while (input < inputEnd)
1045-
{
1046-
if (bytesLeftInBlock == 0)
1044+
int i;
1045+
for (i = 0; i < toCopyULong; i += 4)
10471046
{
1048-
var nextBlock = pool.Lease();
1049-
block.End = blockIndex;
1050-
Volatile.Write(ref block.Next, nextBlock);
1051-
block = nextBlock;
1052-
1053-
blockIndex = block.Data.Offset;
1054-
bytesLeftInBlock = block.Data.Count;
1055-
bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
1047+
var iUlong = *(ulong*)(input + i);
1048+
*(uint*)(output + i) =
1049+
((uint)((iUlong * Shift16Shift24) >> 24) & 0xffff) |
1050+
((uint)((iUlong * Shift8Identity) >> 24) & 0xffff0000);
1051+
}
1052+
for (; i < toCopy; i++)
1053+
{
1054+
*(output + i) = (byte)*(input + i);
10561055
}
10571056

1058-
var output = (block.DataFixedPtr + block.End);
1059-
var copied = 0;
1060-
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
1057+
block.End = blockIndex;
1058+
if (length <= bytesLeftInBlock)
10611059
{
1062-
*(output) = (byte)*(input);
1063-
*(output + 1) = (byte)*(input + 1);
1064-
*(output + 2) = (byte)*(input + 2);
1065-
*(output + 3) = (byte)*(input + 3);
1066-
output += 4;
1067-
input += 4;
1060+
_index = blockIndex;
10681061
}
1069-
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
1062+
else
10701063
{
1071-
*(output++) = (byte)*(input++);
1064+
CopyFromAsciiMultiblock(input + toCopy, length - bytesLeftInBlock);
10721065
}
1073-
1074-
blockIndex += copied;
1075-
bytesLeftInBlockMinusSpan -= copied;
1076-
bytesLeftInBlock -= copied;
10771066
}
10781067
}
1068+
}
1069+
1070+
private unsafe void CopyFromAsciiMultiblock(char* inputStart, int remainingLength)
1071+
{
1072+
var input = inputStart;
1073+
var length = remainingLength;
1074+
var block = _block;
1075+
var pool = block.Pool;
1076+
int blockIndex;
1077+
do
1078+
{
1079+
var nextBlock = pool.Lease();
1080+
Volatile.Write(ref block.Next, nextBlock);
1081+
block = nextBlock;
1082+
1083+
var bytesLeftInBlock = block.Data.Count;
1084+
1085+
var output = (block.DataFixedPtr + block.End);
1086+
var toCopy = Math.Min(length, block.Data.Count);
1087+
var toCopyULong = toCopy & ~0x3;
1088+
1089+
blockIndex = block.Data.Offset + toCopy;
1090+
1091+
int i;
1092+
for (i = 0; i < toCopyULong; i += 4)
1093+
{
1094+
var iUlong = *(ulong*)(input + i);
1095+
*(uint*)(output + i) =
1096+
((uint)((iUlong * Shift16Shift24) >> 24) & 0xffff) |
1097+
((uint)((iUlong * Shift8Identity) >> 24) & 0xffff0000);
1098+
}
1099+
for (; i < toCopy; i++)
1100+
{
1101+
*(output + i) = (byte)*(input + i);
1102+
}
1103+
1104+
block.End = blockIndex;
1105+
1106+
length -= toCopy;
1107+
input += toCopy;
1108+
} while (length > 0);
10791109

1080-
block.End = blockIndex;
10811110
_block = block;
10821111
_index = blockIndex;
10831112
}

‎test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1214,6 +1214,71 @@ public void TestGetAsciiStringEscaped(string input, string expected, int maxChar
12141214
}
12151215
}
12161216

1217+
[Fact]
1218+
public unsafe void TestCopyFromAscii()
1219+
{
1220+
// Arrange
1221+
var blockSingle = _pool.Lease();
1222+
var blockMultiple = _pool.Lease();
1223+
try
1224+
{
1225+
var dataSingle = new string('\0', blockSingle.Data.Count - 1);
1226+
var dataMultiple = new string('\0', blockMultiple.Data.Count * 4 - 1);
1227+
1228+
fixed (char* pData = dataSingle)
1229+
{
1230+
for (var i = 0; i < dataSingle.Length; i++)
1231+
{
1232+
// ascii chars 32 - 126
1233+
pData[i] = (char)((i % (126 - 32)) + 32);
1234+
}
1235+
}
1236+
fixed (char* pData = dataMultiple)
1237+
{
1238+
for (var i = 0; i < dataMultiple.Length; i++)
1239+
{
1240+
// ascii chars 32 - 126
1241+
pData[i] = (char)((i % (126 - 32)) + 32);
1242+
}
1243+
}
1244+
1245+
// Act
1246+
var singleIter = blockSingle.GetIterator();
1247+
var multiIter = blockMultiple.GetIterator();
1248+
1249+
var singleIterEnd = singleIter;
1250+
var multiIterEnd = multiIter;
1251+
1252+
singleIterEnd.CopyFromAscii(dataSingle);
1253+
multiIterEnd.CopyFromAscii(dataMultiple);
1254+
1255+
// Assert
1256+
Assert.True(singleIterEnd.IsEnd);
1257+
foreach (var ch in dataSingle)
1258+
{
1259+
Assert.Equal(ch, singleIter.Take());
1260+
}
1261+
Assert.True(singleIter.IsEnd);
1262+
Assert.Equal(singleIter.Block, singleIterEnd.Block);
1263+
Assert.Equal(singleIter.Index, singleIterEnd.Index);
1264+
1265+
1266+
Assert.True(multiIterEnd.IsEnd);
1267+
foreach (var ch in dataMultiple)
1268+
{
1269+
Assert.Equal(ch, multiIter.Take());
1270+
}
1271+
Assert.True(multiIter.IsEnd);
1272+
Assert.Equal(multiIter.Block, multiIterEnd.Block);
1273+
Assert.Equal(multiIter.Index, multiIterEnd.Index);
1274+
}
1275+
finally
1276+
{
1277+
ReturnBlocks(blockSingle);
1278+
ReturnBlocks(blockMultiple);
1279+
}
1280+
}
1281+
12171282
private delegate bool GetKnownString(MemoryPoolIterator iter, out string result);
12181283

12191284
private void TestKnownStringsInterning(string input, string expected, GetKnownString action)
@@ -1244,6 +1309,17 @@ private void TestKnownStringsInterning(string input, string expected, GetKnownSt
12441309
Assert.Same(knownString1, knownString2);
12451310
}
12461311

1312+
private static void ReturnBlocks(MemoryPoolBlock block)
1313+
{
1314+
while (block != null)
1315+
{
1316+
var returningBlock = block;
1317+
block = returningBlock.Next;
1318+
1319+
returningBlock.Pool.Return(returningBlock);
1320+
}
1321+
}
1322+
12471323
public static IEnumerable<object[]> SeekByteLimitData
12481324
{
12491325
get

0 commit comments

Comments
 (0)