Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

[Arm64] Optimize JIT_WriteBarriers #12334

Merged
merged 2 commits into from
Sep 7, 2017
Merged

[Arm64] Optimize JIT_WriteBarriers #12334

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7091c75
[Arm64] Optimize JIT_WriteBarriers
sdmaclea Jun 16, 2017
2de13cf
[Arm64/Windows] Disable JIT_WriteBarrier optimization
sdmaclea Aug 25, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 130 additions & 50 deletions src/vm/arm64/asmhelpers.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,106 @@ LEAF_END ThePreStubPatch, _TEXT
LEAF_END_MARKED \name, _TEXT
.endmacro

// ------------------------------------------------------------------
// Start of the writeable code region
LEAF_ENTRY JIT_PatchedCodeStart, _TEXT
ret lr
LEAF_END JIT_PatchedCodeStart, _TEXT

// void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck)
//
// Update shadow copies of the various state info required for barrier
//
// State info is contained in a literal pool at the end of the function
// Placed in text section so that it is close enough to use ldr literal and still
// be relocatable. Eliminates need for PREPARE_EXTERNAL_VAR in hot code.
//
// Align and group state info together so it fits in a single cache line
// and each entry can be written atomically
//
WRITE_BARRIER_ENTRY JIT_UpdateWriteBarrierState
PROLOG_SAVE_REG_PAIR_INDEXED fp, lr, -16

// x0-x7 will contain intended new state
// x8 will preserve skipEphemeralCheck
// x12 will be used for pointers

mov x8, x0

PREPARE_EXTERNAL_VAR g_card_table, x12
ldr x0, [x12]

#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
PREPARE_EXTERNAL_VAR g_card_bundle_table, x12
ldr x1, [x12]
#endif

#ifdef WRITE_BARRIER_CHECK
PREPARE_EXTERNAL_VAR g_GCShadow, x12
ldr x2, [x12]
#endif

#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
PREPARE_EXTERNAL_VAR g_sw_ww_table, x12
ldr x3, [x12]
#endif

PREPARE_EXTERNAL_VAR g_ephemeral_low, x12
ldr x4, [x12]

PREPARE_EXTERNAL_VAR g_ephemeral_high, x12
ldr x5, [x12]

cbz x8, LOCAL_LABEL(EphemeralCheckEnabled)
movz x4, #0
movn x5, #0
LOCAL_LABEL(EphemeralCheckEnabled):

PREPARE_EXTERNAL_VAR g_lowest_address, x12
ldr x6, [x12]

PREPARE_EXTERNAL_VAR g_highest_address, x12
ldr x7, [x12]

// Update wbs state
adr x12, LOCAL_LABEL(wbs_begin)

stp x0, x1, [x12], 16
stp x2, x3, [x12], 16
stp x4, x5, [x12], 16
stp x6, x7, [x12], 16

EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 16
EPILOG_RETURN

// Begin patchable literal pool
.balign 64 // Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line
LOCAL_LABEL(wbs_begin):
LOCAL_LABEL(wbs_card_table):
.quad 0
LOCAL_LABEL(wbs_card_bundle_table):
.quad 0
LOCAL_LABEL(wbs_GCShadow):
.quad 0
LOCAL_LABEL(wbs_sw_ww_table):
.quad 0
LOCAL_LABEL(wbs_ephemeral_low):
.quad 0
LOCAL_LABEL(wbs_ephemeral_high):
.quad 0
LOCAL_LABEL(wbs_lowest_address):
.quad 0
LOCAL_LABEL(wbs_highest_address):
.quad 0
WRITE_BARRIER_END JIT_UpdateWriteBarrierState


// ------------------------------------------------------------------
// End of the writeable code region
LEAF_ENTRY JIT_PatchedCodeLast, _TEXT
ret lr
LEAF_END JIT_PatchedCodeLast, _TEXT

// void JIT_ByRefWriteBarrier
// On entry:
// x13 : the source address (points to object reference to write)
Expand Down Expand Up @@ -235,19 +335,16 @@ WRITE_BARRIER_END JIT_ByRefWriteBarrier
//
// On exit:
// x12 : trashed
// x14 : incremented by 8
// x14 : trashed (incremented by 8 to implement JIT_ByRefWriteBarrier contract)
// x15 : trashed
// x17 : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
//
WRITE_BARRIER_ENTRY JIT_CheckedWriteBarrier
PREPARE_EXTERNAL_VAR g_lowest_address, x12
ldr x12, [x12]
ldr x12, LOCAL_LABEL(wbs_lowest_address)
cmp x14, x12
blt LOCAL_LABEL(NotInHeap)

PREPARE_EXTERNAL_VAR g_highest_address, x12
ldr x12, [x12]
cmp x14, x12
ldr x12, LOCAL_LABEL(wbs_highest_address)
ccmp x14, x12, #0x0, ge
blt C_FUNC(JIT_WriteBarrier)

LOCAL_LABEL(NotInHeap):
Expand All @@ -262,7 +359,7 @@ WRITE_BARRIER_END JIT_CheckedWriteBarrier
//
// On exit:
// x12 : trashed
// x14 : incremented by 8
// x14 : trashed (incremented by 8 to implement JIT_ByRefWriteBarrier contract)
// x15 : trashed
// x17 : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
//
Expand All @@ -272,23 +369,24 @@ WRITE_BARRIER_ENTRY JIT_WriteBarrier
#ifdef WRITE_BARRIER_CHECK
// Update GC Shadow Heap

// need temporary registers. Save them before using.
stp x12, x13, [sp, #-16]!
// Do not perform the work if g_GCShadow is 0
ldr x12, LOCAL_LABEL(wbs_GCShadow)
cbz x12, LOCAL_LABEL(ShadowUpdateDisabled)

// need temporary register. Save before using.
str x13, [sp, #-16]!

// Compute address of shadow heap location:
// pShadow = g_GCShadow + (x14 - g_lowest_address)
PREPARE_EXTERNAL_VAR g_lowest_address, x12
ldr x12, [x12]
sub x12, x14, x12
PREPARE_EXTERNAL_VAR g_GCShadow, x13
ldr x13, [x13]
ldr x13, LOCAL_LABEL(wbs_lowest_address)
sub x13, x14, x13
add x12, x13, x12

// if (pShadow >= g_GCShadowEnd) goto end
PREPARE_EXTERNAL_VAR g_GCShadowEnd, x13
ldr x13, [x13]
cmp x12, x13
bhs LOCAL_LABEL(shadowupdateend)
bhs LOCAL_LABEL(ShadowUpdateEnd)

// *pShadow = x15
str x15, [x12]
Expand All @@ -300,25 +398,22 @@ WRITE_BARRIER_ENTRY JIT_WriteBarrier
// if ([x14] == x15) goto end
ldr x13, [x14]
cmp x13, x15
beq LOCAL_LABEL(shadowupdateend)
beq LOCAL_LABEL(ShadowUpdateEnd)

// *pShadow = INVALIDGCVALUE (0xcccccccd)
mov x13, #0
movk x13, #0xcccd
movz x13, #0xcccd
movk x13, #0xcccc, LSL #16
str x13, [x12]

LOCAL_LABEL(shadowupdateend):
ldp x12, x13, [sp],#16
LOCAL_LABEL(ShadowUpdateEnd):
ldr x13, [sp], #16
LOCAL_LABEL(ShadowUpdateDisabled):
#endif

#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
// Update the write watch table if necessary
PREPARE_EXTERNAL_VAR g_sw_ww_enabled_for_gc_heap, x12
ldrb w12, [x12]
ldr x12, LOCAL_LABEL(wbs_sw_ww_table)
cbz x12, LOCAL_LABEL(CheckCardTable)
PREPARE_EXTERNAL_VAR g_sw_ww_table, x12
ldr x12, [x12]
add x12, x12, x14, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift
ldrb w17, [x12]
cbnz x17, LOCAL_LABEL(CheckCardTable)
Expand All @@ -329,20 +424,18 @@ LOCAL_LABEL(shadowupdateend):
LOCAL_LABEL(CheckCardTable):
// Branch to Exit if the reference is not in the Gen0 heap
//
PREPARE_EXTERNAL_VAR g_ephemeral_low, x12
ldr x12, [x12]
ldr x12, LOCAL_LABEL(wbs_ephemeral_low)
cbz x12, LOCAL_LABEL(SkipEphemeralCheck)
cmp x15, x12
blt LOCAL_LABEL(Exit)

PREPARE_EXTERNAL_VAR g_ephemeral_high, x12
ldr x12, [x12]
cmp x15, x12
ldr x12, LOCAL_LABEL(wbs_ephemeral_high)
ccmp x15, x12, 0x0, ge
bgt LOCAL_LABEL(Exit)

LOCAL_LABEL(SkipEphemeralCheck):
// Check if we need to update the card table
PREPARE_EXTERNAL_VAR g_card_table, x12
ldr x12, [x12]
add x15, x12, x14, lsr #11
ldr x12, LOCAL_LABEL(wbs_card_table)
add x15, x12, x14, lsr #11
ldrb w12, [x15]
cmp x12, 0xFF
beq LOCAL_LABEL(Exit)
Expand All @@ -352,10 +445,9 @@ LOCAL_LABEL(UpdateCardTable):
strb w12, [x15]

#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
// Check if we need to update the card table
PREPARE_EXTERNAL_VAR g_card_bundle_table, x12
ldr x12, [x12]
add x15, x12, x14, lsr #21
// Check if we need to update the card bundle table
ldr x12, LOCAL_LABEL(wbs_card_bundle_table)
add x15, x12, x14, lsr #21
ldrb w12, [x15]
cmp x12, 0xFF
beq LOCAL_LABEL(Exit)
Expand All @@ -370,18 +462,6 @@ LOCAL_LABEL(Exit):
ret lr
WRITE_BARRIER_END JIT_WriteBarrier

// ------------------------------------------------------------------
// Start of the writeable code region
LEAF_ENTRY JIT_PatchedCodeStart, _TEXT
ret lr
LEAF_END JIT_PatchedCodeStart, _TEXT

// ------------------------------------------------------------------
// End of the writeable code region
LEAF_ENTRY JIT_PatchedCodeLast, _TEXT
ret lr
LEAF_END JIT_PatchedCodeLast, _TEXT

//------------------------------------------------
// VirtualMethodFixupStub
//
Expand Down
21 changes: 16 additions & 5 deletions src/vm/arm64/stubs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ EXTERN_C void JIT_GetSharedNonGCStaticBase_SingleAppDomain();
EXTERN_C void JIT_GetSharedNonGCStaticBaseNoCtor_SingleAppDomain();
EXTERN_C void JIT_GetSharedGCStaticBase_SingleAppDomain();
EXTERN_C void JIT_GetSharedGCStaticBaseNoCtor_SingleAppDomain();
EXTERN_C void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck);


#ifndef DACCESS_COMPILE
//-----------------------------------------------------------------------
Expand Down Expand Up @@ -1119,7 +1121,15 @@ void InitJITHelpers1()
SetJitHelperFunction(CORINFO_HELP_GETSHARED_GCSTATIC_BASE_NOCTOR, JIT_GetSharedGCStaticBaseNoCtor_SingleAppDomain);
SetJitHelperFunction(CORINFO_HELP_GETSHARED_NONGCSTATIC_BASE_NOCTOR,JIT_GetSharedNonGCStaticBaseNoCtor_SingleAppDomain);
}

JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
}
#ifndef FEATURE_PAL // TODO-ARM64-WINDOWS #13592
EXTERN_C void JIT_UpdateWriteBarrierState(bool) {}
#endif

#else
EXTERN_C void JIT_UpdateWriteBarrierState(bool) {}
#endif // !defined(DACCESS_COMPILE) && !defined(CROSSGEN_COMPILE)

EXTERN_C void __stdcall ProfileEnterNaked(UINT_PTR clientData)
Expand Down Expand Up @@ -1338,28 +1348,29 @@ LONG CLRNoCatchHandler(EXCEPTION_POINTERS* pExceptionInfo, PVOID pv)
return EXCEPTION_CONTINUE_SEARCH;
}

#ifndef CROSSGEN_COMPILE
void StompWriteBarrierEphemeral(bool isRuntimeSuspended)
{
return;
JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
}

void StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
{
return;
JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
}

#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
void SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
{
return;
JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
}

void SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
{
return;
JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap());
}
#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP

#endif // CROSSGEN_COMPILE

#ifdef DACCESS_COMPILE
BOOL GetAnyThunkTarget (T_CONTEXT *pctx, TADDR *pTarget, TADDR *pTargetMethodDesc)
Expand Down
14 changes: 13 additions & 1 deletion src/vm/gcenv.ee.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,7 +861,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
#endif

#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
if (args->write_watch_table != nullptr)
if (g_sw_ww_enabled_for_gc_heap && (args->write_watch_table != nullptr))
{
assert(args->is_runtime_suspended);
g_sw_ww_table = args->write_watch_table;
Expand All @@ -888,6 +888,17 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)

g_lowest_address = args->lowest_address;
VolatileStore(&g_highest_address, args->highest_address);

#if defined(_ARM64_)
// Need to reupdate for changes to g_highest_address g_lowest_address
::StompWriteBarrierResize(args->is_runtime_suspended, args->requires_upper_bounds_check);

if(!args->is_runtime_suspended)
{
// If runtime is not suspended, force updated state to be visible to all threads
MemoryBarrier();
}
#endif
return;
case WriteBarrierOp::StompEphemeral:
// StompEphemeral requires a new ephemeral low and a new ephemeral high
Expand Down Expand Up @@ -945,6 +956,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
case WriteBarrierOp::SwitchToNonWriteWatch:
#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
assert(args->is_runtime_suspended && "the runtime must be suspended here!");
g_sw_ww_table = 0;
g_sw_ww_enabled_for_gc_heap = false;
::SwitchToNonWriteWatchBarrier(true);
#else
Expand Down