Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Optimize Memorypool block use #787

Merged
merged 3 commits into from
May 3, 2016
Merged

Optimize Memorypool block use #787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a31d847
Tests only to use pooled blocks
benaadams Apr 30, 2016
2aa10a6
Remove noop pin+unpin w/ memory blocks
benaadams Apr 30, 2016
3a424f6
Remove unnecessary fixed blocks
benaadams Apr 30, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Microsoft.AspNetCore.Server.Kestrel/Http/Connection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ private Libuv.uv_buf_t OnAlloc(UvStreamHandle handle, int suggestedSize)
var result = _rawSocketInput.IncomingStart();

return handle.Libuv.buf_init(
result.Pin() + result.End,
result.DataArrayPtr + result.End,
result.Data.Offset + result.Data.Count - result.End);
}

Expand Down
3 changes: 0 additions & 3 deletions src/Microsoft.AspNetCore.Server.Kestrel/Http/SocketOutput.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -684,14 +684,12 @@ private void ScheduleReturnFullyWrittenBlocks()
var end = _lockedEnd.Block;
if (block == end)
{
end.Unpin();
return;
}

while (block.Next != end)
{
block = block.Next;
block.Unpin();
}
block.Next = null;

Expand All @@ -705,7 +703,6 @@ private static void ReturnWrittenBlocks(MemoryPoolBlock block)
var returnBlock = block;
block = block.Next;

returnBlock.Unpin();
returnBlock.Pool.Return(returnBlock);
}
}
Expand Down
17 changes: 1 addition & 16 deletions src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPool.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,24 +63,9 @@ public class MemoryPool : IDisposable
/// <summary>
/// Called to take a block from the pool.
/// </summary>
/// <param name="minimumSize">The block returned must be at least this size. It may be larger than this minimum size, and if so,
/// the caller may write to the block's entire size rather than being limited to the minumumSize requested.</param>
/// <returns>The block that is reserved for the called. It must be passed to Return when it is no longer being used.</returns>
public MemoryPoolBlock Lease(int minimumSize = MaxPooledBlockLength)
public MemoryPoolBlock Lease()
{
if (minimumSize > _blockLength)
{
// The requested minimumSize is actually larger then the usable memory of a single block.
// Because this is the degenerate case, a one-time-use byte[] array and tracking object are allocated.
// When this block tracking object is returned it is not added to the pool - instead it will be
// allowed to be garbage collected normally.
return MemoryPoolBlock.Create(
new ArraySegment<byte>(new byte[minimumSize]),
dataPtr: IntPtr.Zero,
pool: this,
slab: null);
}

MemoryPoolBlock block;
if (_blocks.TryDequeue(out block))
{
Expand Down
64 changes: 10 additions & 54 deletions src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPoolBlock.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure
/// </summary>
public class MemoryPoolBlock
{
/// <summary>
/// If this block represents a one-time-use memory object, this GCHandle will hold that memory object at a fixed address
/// so it can be used in native operations.
/// </summary>
private GCHandle _pinHandle;

/// <summary>
/// Native address of the first byte of this block's Data memory. It is null for one-time-use memory, or copied from
/// the Slab's ArrayPtr for a slab-block segment. The byte it points to corresponds to Data.Array[0], and in practice you will always
/// use the _dataArrayPtr + Start or _dataArrayPtr + End, which point to the start of "active" bytes, or point to just after the "active" bytes.
/// use the DataArrayPtr + Start or DataArrayPtr + End, which point to the start of "active" bytes, or point to just after the "active" bytes.
/// </summary>
private IntPtr _dataArrayPtr;
public readonly IntPtr DataArrayPtr;

internal unsafe readonly byte* DataFixedPtr;

/// <summary>
/// The array segment describing the range of memory this block is tracking. The caller which has leased this block may only read and
Expand All @@ -33,8 +29,10 @@ public class MemoryPoolBlock
/// <summary>
/// This object cannot be instantiated outside of the static Create method
/// </summary>
protected MemoryPoolBlock()
unsafe protected MemoryPoolBlock(IntPtr dataArrayPtr)
{
DataArrayPtr = dataArrayPtr;
DataFixedPtr = (byte*)dataArrayPtr.ToPointer();
}

/// <summary>
Expand Down Expand Up @@ -76,70 +74,28 @@ protected MemoryPoolBlock()

~MemoryPoolBlock()
{
Debug.Assert(!_pinHandle.IsAllocated, "Ad-hoc memory block wasn't unpinned");
Debug.Assert(Slab == null || !Slab.IsActive, "Block being garbage collected instead of returned to pool");

if (_pinHandle.IsAllocated)
{
// if this is a one-time-use block, ensure that the GCHandle does not leak
_pinHandle.Free();
}

if (Slab != null && Slab.IsActive)
{
Pool.Return(new MemoryPoolBlock
Pool.Return(new MemoryPoolBlock(DataArrayPtr)
{
_dataArrayPtr = _dataArrayPtr,
Data = Data,
Pool = Pool,
Slab = Slab,
});
}
}

/// <summary>
/// Called to ensure that a block is pinned, and return the pointer to the native address
/// of the first byte of this block's Data memory. Arriving data is read into Pin() + End.
/// Outgoing data is read from Pin() + Start.
/// </summary>
/// <returns></returns>
public IntPtr Pin()
{
Debug.Assert(!_pinHandle.IsAllocated);

if (_dataArrayPtr != IntPtr.Zero)
{
// this is a slab managed block - use the native address of the slab which is always locked
return _dataArrayPtr;
}
else
{
// this is one-time-use memory - lock the managed memory until Unpin is called
_pinHandle = GCHandle.Alloc(Data.Array, GCHandleType.Pinned);
return _pinHandle.AddrOfPinnedObject();
}
}

public void Unpin()
{
if (_dataArrayPtr == IntPtr.Zero)
{
// this is one-time-use memory - unlock the managed memory
Debug.Assert(_pinHandle.IsAllocated);
_pinHandle.Free();
}
}

public static MemoryPoolBlock Create(
internal static MemoryPoolBlock Create(
ArraySegment<byte> data,
IntPtr dataPtr,
MemoryPool pool,
MemoryPoolSlab slab)
{
return new MemoryPoolBlock
return new MemoryPoolBlock(dataPtr)
{
Data = data,
_dataArrayPtr = dataPtr,
Pool = pool,
Slab = slab,
Start = data.Offset,
Expand Down
168 changes: 72 additions & 96 deletions src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPoolIterator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,10 +175,7 @@ public unsafe long PeekLong()
}
else if (_block.End - _index >= sizeof(long))
{
fixed (byte* ptr = &_block.Array[_index])
{
return *(long*)(ptr);
}
return *(long*)(_block.DataFixedPtr + _index);
}
else if (_block.Next == null)
{
Expand All @@ -194,17 +191,9 @@ public unsafe long PeekLong()
return -1;
}

long blockLong;
fixed (byte* ptr = &_block.Array[_block.End - sizeof(long)])
{
blockLong = *(long*)(ptr);
}
var blockLong = *(long*)(_block.DataFixedPtr + _block.End - sizeof(long));

long nextLong;
fixed (byte* ptr = &_block.Next.Array[_block.Next.Start])
{
nextLong = *(long*)(ptr);
}
var nextLong = *(long*)(_block.Next.DataFixedPtr + _block.Next.Start);

return (blockLong >> (sizeof(long) - blockBytes) * 8) | (nextLong << (sizeof(long) - nextBytes) * 8);
}
Expand Down Expand Up @@ -266,22 +255,20 @@ public unsafe int Seek(ref Vector<byte> byte0Vector)
#if !DEBUG
}
#endif
fixed (byte* ptr = &block.Array[index])

var pCurrent = (block.DataFixedPtr + index);
var pEnd = pCurrent + following;
do
{
var pCurrent = ptr;
var pEnd = pCurrent + following;
do
if (*pCurrent == byte0)
{
if (*pCurrent == byte0)
{
_block = block;
_index = index;
return byte0;
}
pCurrent++;
index++;
} while (pCurrent < pEnd);
}
_block = block;
_index = index;
return byte0;
}
pCurrent++;
index++;
} while (pCurrent < pEnd);

following = 0;
break;
Expand Down Expand Up @@ -367,28 +354,25 @@ public unsafe int Seek(ref Vector<byte> byte0Vector, ref Vector<byte> byte1Vecto
#if !DEBUG
}
#endif
fixed (byte* ptr = &block.Array[index])
var pCurrent = (block.DataFixedPtr + index);
var pEnd = pCurrent + following;
do
{
var pCurrent = ptr;
var pEnd = pCurrent + following;
do
if (*pCurrent == byte0)
{
if (*pCurrent == byte0)
{
_block = block;
_index = index;
return byte0;
}
if (*pCurrent == byte1)
{
_block = block;
_index = index;
return byte1;
}
pCurrent++;
index++;
} while (pCurrent != pEnd);
}
_block = block;
_index = index;
return byte0;
}
if (*pCurrent == byte1)
{
_block = block;
_index = index;
return byte1;
}
pCurrent++;
index++;
} while (pCurrent != pEnd);

following = 0;
break;
Expand Down Expand Up @@ -502,34 +486,31 @@ public unsafe int Seek(ref Vector<byte> byte0Vector, ref Vector<byte> byte1Vecto
#if !DEBUG
}
#endif
fixed (byte* ptr = &block.Array[index])
var pCurrent = (block.DataFixedPtr + index);
var pEnd = pCurrent + following;
do
{
var pCurrent = ptr;
var pEnd = pCurrent + following;
do
if (*pCurrent == byte0)
{
if (*pCurrent == byte0)
{
_block = block;
_index = index;
return byte0;
}
if (*pCurrent == byte1)
{
_block = block;
_index = index;
return byte1;
}
if (*pCurrent == byte2)
{
_block = block;
_index = index;
return byte2;
}
pCurrent++;
index++;
} while (pCurrent != pEnd);
}
_block = block;
_index = index;
return byte0;
}
if (*pCurrent == byte1)
{
_block = block;
_index = index;
return byte1;
}
if (*pCurrent == byte2)
{
_block = block;
_index = index;
return byte2;
}
pCurrent++;
index++;
} while (pCurrent != pEnd);

following = 0;
break;
Expand Down Expand Up @@ -808,30 +789,25 @@ public unsafe void CopyFromAscii(string data)
bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
}

fixed (byte* pOutput = &block.Data.Array[block.End])
var output = (block.DataFixedPtr + block.End);
var copied = 0;
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
{
//this line is needed to allow output be an register var
var output = pOutput;

var copied = 0;
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
{
*(output) = (byte)*(input);
*(output + 1) = (byte)*(input + 1);
*(output + 2) = (byte)*(input + 2);
*(output + 3) = (byte)*(input + 3);
output += 4;
input += 4;
}
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
{
*(output++) = (byte)*(input++);
}

blockIndex += copied;
bytesLeftInBlockMinusSpan -= copied;
bytesLeftInBlock -= copied;
*(output) = (byte)*(input);
*(output + 1) = (byte)*(input + 1);
*(output + 2) = (byte)*(input + 2);
*(output + 3) = (byte)*(input + 3);
output += 4;
input += 4;
}
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
{
*(output++) = (byte)*(input++);
}

blockIndex += copied;
bytesLeftInBlockMinusSpan -= copied;
bytesLeftInBlock -= copied;
}
}

Expand Down
Loading