diff --git a/src/coreclr/gc/gcee.cpp b/src/coreclr/gc/gcee.cpp index a941acb74efab9..9e5aaa6b4739b4 100644 --- a/src/coreclr/gc/gcee.cpp +++ b/src/coreclr/gc/gcee.cpp @@ -503,6 +503,15 @@ bool GCHeap::IsInFrozenSegment(Object *object) #endif } +void GCHeap::UpdateFrozenSegment(segment_handle seg, uint8_t* allocated, uint8_t* committed) +{ +#ifdef FEATURE_BASICFREEZE + heap_segment* heap_seg = reinterpret_cast(seg); + heap_segment_committed(heap_seg) = committed; + heap_segment_allocated(heap_seg) = allocated; +#endif // FEATURE_BASICFREEZE +} + bool GCHeap::RuntimeStructuresValid() { return GCScan::GetGcRuntimeStructuresValid(); diff --git a/src/coreclr/gc/gcimpl.h b/src/coreclr/gc/gcimpl.h index 67dc02943ffcd6..23ddcbc67e73a9 100644 --- a/src/coreclr/gc/gcimpl.h +++ b/src/coreclr/gc/gcimpl.h @@ -245,6 +245,7 @@ class GCHeap : public IGCHeapInternal virtual segment_handle RegisterFrozenSegment(segment_info *pseginfo); virtual void UnregisterFrozenSegment(segment_handle seg); virtual bool IsInFrozenSegment(Object *object); + virtual void UpdateFrozenSegment(segment_handle seg, uint8_t* allocated, uint8_t* committed); // Event control functions void ControlEvents(GCEventKeyword keyword, GCEventLevel level); diff --git a/src/coreclr/gc/gcinterface.h b/src/coreclr/gc/gcinterface.h index 023f257d10fa58..ef2173bd310450 100644 --- a/src/coreclr/gc/gcinterface.h +++ b/src/coreclr/gc/gcinterface.h @@ -955,6 +955,9 @@ class IGCHeap { // Gets all the names and values of the GC configurations. virtual void EnumerateConfigurationValues(void* context, ConfigurationValueFunc configurationValueFunc) = 0; + + // Updates given frozen segment + virtual void UpdateFrozenSegment(segment_handle seg, uint8_t* allocated, uint8_t* committed) = 0; }; #ifdef WRITE_BARRIER_CHECK diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index 9d8413f4ec0141..a600138a65dc20 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -405,6 +405,11 @@ RETAIL_CONFIG_DWORD_INFO(INTERNAL_CodeHeapReserveForJumpStubs, W("CodeHeapReserv RETAIL_CONFIG_DWORD_INFO(INTERNAL_NGenReserveForJumpStubs, W("NGenReserveForJumpStubs"), 0, "Percentage of ngen image size to reserve for jump stubs") RETAIL_CONFIG_DWORD_INFO(INTERNAL_BreakOnOutOfMemoryWithinRange, W("BreakOnOutOfMemoryWithinRange"), 0, "Break before out of memory within range exception is thrown") +/// +/// Frozen segments (aka Frozen Object Heap) +/// +RETAIL_CONFIG_DWORD_INFO(INTERNAL_UseFrozenObjectHeap, W("UseFrozenObjectHeap"), 1, "Use frozen object heap for certain types of objects (e.g. string literals) as an optimization.") + /// /// Log /// diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index 41154770a8cb22..2be3d438c9327d 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -1931,6 +1931,7 @@ struct CORINFO_VarArgInfo #define SIZEOF__CORINFO_Object TARGET_POINTER_SIZE /* methTable */ #define CORINFO_Array_MaxLength 0x7FFFFFC7 +#define CORINFO_String_MaxLength 0x3FFFFFDF #define OFFSETOF__CORINFO_Array__length SIZEOF__CORINFO_Object #ifdef TARGET_64BIT diff --git a/src/coreclr/inc/crsttypes.h b/src/coreclr/inc/crsttypes.h index f96df91d88e821..cc3d34ab58e2b9 100644 --- a/src/coreclr/inc/crsttypes.h +++ b/src/coreclr/inc/crsttypes.h @@ -131,7 +131,8 @@ enum CrstType CrstUnwindInfoTableLock = 113, CrstVSDIndirectionCellLock = 114, CrstWrapperTemplate = 115, - kNumberOfCrstTypes = 116 + CrstFrozenObjectHeap = 116, + kNumberOfCrstTypes = 117 }; #endif // __CRST_TYPES_INCLUDED @@ -258,6 +259,7 @@ int g_rgCrstLevelMap[] = 3, // CrstUnwindInfoTableLock 4, // CrstVSDIndirectionCellLock 3, // CrstWrapperTemplate + 0, // CrstFrozenObjectHeap }; // An array mapping CrstType to a stringized name. @@ -379,6 +381,7 @@ LPCSTR g_rgCrstNameMap[] = "CrstUnwindInfoTableLock", "CrstVSDIndirectionCellLock", "CrstWrapperTemplate", + "CrstFrozenObjectHeap" }; // Define a special level constant for unordered locks. diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 6d746e1b3c2f79..b140c8caf8aa8c 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -1155,8 +1155,14 @@ void Compiler::optPrintAssertion(AssertionDsc* curAssertion, AssertionIndex asse if (op1Type == TYP_REF) { - assert(curAssertion->op2.u1.iconVal == 0); - printf("null"); + if (curAssertion->op2.u1.iconVal == 0) + { + printf("null"); + } + else + { + printf("[%08p]", dspPtr(curAssertion->op2.u1.iconVal)); + } } else { @@ -2646,8 +2652,10 @@ AssertionInfo Compiler::optAssertionGenJtrue(GenTree* tree) GenTree* objectNode = call->gtArgs.GetArgByIndex(1)->GetNode(); GenTree* methodTableNode = call->gtArgs.GetArgByIndex(0)->GetNode(); - assert(objectNode->TypeGet() == TYP_REF); - assert(methodTableNode->TypeGet() == TYP_I_IMPL); + // objectNode can be TYP_I_IMPL in case if it's a constant handle + // (e.g. a string literal from frozen segments) + assert(objectNode->TypeIs(TYP_REF, TYP_I_IMPL)); + assert(methodTableNode->TypeIs(TYP_I_IMPL)); // Reverse the assertion assert((assertionKind == OAK_EQUAL) || (assertionKind == OAK_NOT_EQUAL)); @@ -3134,11 +3142,9 @@ GenTree* Compiler::optVNConstantPropOnTree(BasicBlock* block, GenTree* tree) case TYP_REF: { - assert(vnStore->ConstantValue(vnCns) == 0); - // Support onle ref(ref(0)), do not support other forms (e.g byref(ref(0)). if (tree->TypeGet() == TYP_REF) { - conValTree = gtNewIconNode(0, TYP_REF); + conValTree = gtNewIconNode(vnStore->ConstantValue(vnCns), TYP_REF); } } break; @@ -4285,8 +4291,14 @@ GenTree* Compiler::optAssertionPropGlobal_RelOp(ASSERT_VALARG_TP assertions, Gen else if (op1->TypeGet() == TYP_REF) { // The only constant of TYP_REF that ValueNumbering supports is 'null' - assert(vnStore->ConstantValue(vnCns) == 0); - printf("null\n"); + if (vnStore->ConstantValue(vnCns) == 0) + { + printf("null\n"); + } + else + { + printf("%d (gcref)\n", static_cast(vnStore->ConstantValue(vnCns))); + } } else if (op1->TypeGet() == TYP_BYREF) { @@ -4339,9 +4351,7 @@ GenTree* Compiler::optAssertionPropGlobal_RelOp(ASSERT_VALARG_TP assertions, Gen } else if (op1->TypeGet() == TYP_REF) { - op1->BashToConst(0, TYP_REF); - // The only constant of TYP_REF that ValueNumbering supports is 'null' - noway_assert(vnStore->ConstantValue(vnCns) == 0); + op1->BashToConst(static_cast(vnStore->ConstantValue(vnCns)), TYP_REF); } else if (op1->TypeGet() == TYP_BYREF) { diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 7a0e90ef201b16..664d14147c3b51 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -1374,8 +1374,6 @@ bool CodeGen::genCreateAddrMode( if (varTypeIsGC(rv2->TypeGet())) { - noway_assert(rv1 && !varTypeIsGC(rv1->TypeGet())); - tmp = rv1; rv1 = rv2; rv2 = tmp; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 0becde0a8002e5..eff50b272a9865 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -3838,10 +3838,16 @@ bool Compiler::gtIsLikelyRegVar(GenTree* tree) // bool Compiler::gtCanSwapOrder(GenTree* firstNode, GenTree* secondNode) { - // Relative of order of global / side effects can't be swapped. - bool canSwap = true; + // Don't swap "CONST_HDL op CNS" + if (firstNode->IsIconHandle() && secondNode->IsIntegralConst()) + { + canSwap = false; + } + + // Relative of order of global / side effects can't be swapped. + if (optValnumCSE_phase) { canSwap = optCSE_canSwap(firstNode, secondNode); diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index b97a405d724a64..153fe06f44f4a6 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -320,6 +320,7 @@ set(VM_SOURCES_WKS fcall.cpp fieldmarshaler.cpp finalizerthread.cpp + frozenobjectheap.cpp gccover.cpp gcenv.ee.static.cpp gcenv.ee.common.cpp @@ -424,6 +425,7 @@ set(VM_HEADERS_WKS fcall.h fieldmarshaler.h finalizerthread.h + frozenobjectheap.h gcenv.h gcenv.ee.h gcenv.os.h diff --git a/src/coreclr/vm/appdomain.cpp b/src/coreclr/vm/appdomain.cpp index 90ec6df596a3c0..6fa4b73a65960a 100644 --- a/src/coreclr/vm/appdomain.cpp +++ b/src/coreclr/vm/appdomain.cpp @@ -22,6 +22,7 @@ #include "assemblynative.hpp" #include "shimload.h" #include "stringliteralmap.h" +#include "frozenobjectheap.h" #include "codeman.h" #include "comcallablewrapper.h" #include "eventtrace.h" @@ -97,7 +98,8 @@ CrstStatic BaseDomain::m_SpecialStaticsCrst; int BaseDomain::m_iNumberOfProcessors = 0; // System Domain Statics -GlobalStringLiteralMap* SystemDomain::m_pGlobalStringLiteralMap = NULL; +GlobalStringLiteralMap* SystemDomain::m_pGlobalStringLiteralMap = NULL; +FrozenObjectHeapManager* SystemDomain::m_FrozenObjectHeapManager = NULL; DECLSPEC_ALIGN(16) static BYTE g_pSystemDomainMemory[sizeof(SystemDomain)]; @@ -1191,6 +1193,24 @@ void SystemDomain::LazyInitGlobalStringLiteralMap() } } +void SystemDomain::LazyInitFrozenObjectsHeap() +{ + CONTRACTL + { + THROWS; + GC_TRIGGERS; + MODE_ANY; + INJECT_FAULT(COMPlusThrowOM();); + } + CONTRACTL_END; + + NewHolder pFoh(new FrozenObjectHeapManager()); + if (InterlockedCompareExchangeT(&m_FrozenObjectHeapManager, pFoh, nullptr) == nullptr) + { + pFoh.SuppressRelease(); + } +} + /*static*/ void SystemDomain::EnumAllStaticGCRefs(promote_func* fn, ScanContext* sc) { CONTRACT_VOID diff --git a/src/coreclr/vm/appdomain.hpp b/src/coreclr/vm/appdomain.hpp index 2534cddcb5744b..d56157f8e6ee6a 100644 --- a/src/coreclr/vm/appdomain.hpp +++ b/src/coreclr/vm/appdomain.hpp @@ -40,6 +40,7 @@ class SystemDomain; class AppDomain; class GlobalStringLiteralMap; class StringLiteralMap; +class FrozenObjectHeapManager; class MngStdInterfacesInfo; class DomainAssembly; class LoadLevelLimiter; @@ -2388,6 +2389,7 @@ class SystemDomain : public BaseDomain void Init(); void Stop(); static void LazyInitGlobalStringLiteralMap(); + static void LazyInitFrozenObjectsHeap(); //**************************************************************************************** // @@ -2460,6 +2462,15 @@ class SystemDomain : public BaseDomain _ASSERTE(m_pGlobalStringLiteralMap); return m_pGlobalStringLiteralMap; } + static FrozenObjectHeapManager* GetFrozenObjectHeapManager() + { + WRAPPER_NO_CONTRACT; + if (m_FrozenObjectHeapManager == NULL) + { + LazyInitFrozenObjectsHeap(); + } + return m_FrozenObjectHeapManager; + } #endif // DACCESS_COMPILE #if defined(FEATURE_COMINTEROP_APARTMENT_SUPPORT) @@ -2629,6 +2640,7 @@ class SystemDomain : public BaseDomain static CrstStatic m_SystemDomainCrst; static GlobalStringLiteralMap *m_pGlobalStringLiteralMap; + static FrozenObjectHeapManager *m_FrozenObjectHeapManager; #endif // DACCESS_COMPILE public: diff --git a/src/coreclr/vm/ceeload.cpp b/src/coreclr/vm/ceeload.cpp index e6a1f86114c4e8..73eb37933ab811 100644 --- a/src/coreclr/vm/ceeload.cpp +++ b/src/coreclr/vm/ceeload.cpp @@ -2835,7 +2835,7 @@ void ModuleBase::InitializeStringData(DWORD token, EEStringData *pstrData, CQuic } -OBJECTHANDLE ModuleBase::ResolveStringRef(DWORD token) +OBJECTHANDLE ModuleBase::ResolveStringRef(DWORD token, void** ppPinnedString) { CONTRACTL { @@ -2864,7 +2864,7 @@ OBJECTHANDLE ModuleBase::ResolveStringRef(DWORD token) pLoaderAllocator = this->GetLoaderAllocator(); - string = (OBJECTHANDLE)pLoaderAllocator->GetStringObjRefPtrFromUnicodeString(&strData); + string = (OBJECTHANDLE)pLoaderAllocator->GetStringObjRefPtrFromUnicodeString(&strData, ppPinnedString); return string; } diff --git a/src/coreclr/vm/ceeload.h b/src/coreclr/vm/ceeload.h index 16b3adac4b59f5..d5e5395840c326 100644 --- a/src/coreclr/vm/ceeload.h +++ b/src/coreclr/vm/ceeload.h @@ -579,7 +579,7 @@ class ModuleBase } // Resolving - OBJECTHANDLE ResolveStringRef(DWORD Token); + OBJECTHANDLE ResolveStringRef(DWORD Token, void** ppPinnedString = nullptr); private: // string helper void InitializeStringData(DWORD token, EEStringData *pstrData, CQuickBytes *pqb); diff --git a/src/coreclr/vm/dynamicmethod.cpp b/src/coreclr/vm/dynamicmethod.cpp index d837252b904951..a4227c4d000894 100644 --- a/src/coreclr/vm/dynamicmethod.cpp +++ b/src/coreclr/vm/dynamicmethod.cpp @@ -1275,7 +1275,8 @@ STRINGREF* LCGMethodResolver::GetOrInternString(STRINGREF *pProtectedStringRef) // lock the global string literal interning map CrstHolder gch(pStringLiteralMap->GetHashTableCrstGlobal()); - StringLiteralEntryHolder pEntry(pStringLiteralMap->GetInternedString(pProtectedStringRef, dwHash, /* bAddIfNotFound */ TRUE)); + StringLiteralEntryHolder pEntry(pStringLiteralMap->GetInternedString(pProtectedStringRef, dwHash, + /* bAddIfNotFound */ TRUE, /* bPreferFrozenObjectHeap */ FALSE)); DynamicStringLiteral* pStringLiteral = (DynamicStringLiteral*)m_jitTempData.New(sizeof(DynamicStringLiteral)); pStringLiteral->m_pEntry = pEntry.Extract(); diff --git a/src/coreclr/vm/frozenobjectheap.cpp b/src/coreclr/vm/frozenobjectheap.cpp new file mode 100644 index 00000000000000..cedade43fa1251 --- /dev/null +++ b/src/coreclr/vm/frozenobjectheap.cpp @@ -0,0 +1,170 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "common.h" +#include "frozenobjectheap.h" + +// Size to reserve for a frozen segment +#define FOH_SEGMENT_SIZE (4 * 1024 * 1024) +// Size to commit on demand in that reserved space +#define FOH_COMMIT_SIZE (64 * 1024) + +FrozenObjectHeapManager::FrozenObjectHeapManager(): + m_Crst(CrstFrozenObjectHeap, CRST_UNSAFE_COOPGC), + m_CurrentSegment(nullptr), + m_Enabled(CLRConfig::GetConfigValue(CLRConfig::INTERNAL_UseFrozenObjectHeap) != 0) +{ +} + +// Allocates an object of the give size (including header) on a frozen segment. +// May return nullptr in the following cases: +// 1) DOTNET_UseFrozenObjectHeap is 0 (disabled) +// 2) Object is too large (large than FOH_COMMIT_SIZE) +// in such cases caller is responsible to find a more appropriate heap to allocate it +Object* FrozenObjectHeapManager::TryAllocateObject(PTR_MethodTable type, size_t objectSize) +{ + CONTRACTL + { + THROWS; + MODE_COOPERATIVE; + } + CONTRACTL_END + +#ifndef FEATURE_BASICFREEZE + // GC is required to support frozen segments + return nullptr; +#else // FEATURE_BASICFREEZE + + CrstHolder ch(&m_Crst); + + if (!m_Enabled) + { + // Disabled via DOTNET_UseFrozenObjectHeap=0 + return nullptr; + } + + _ASSERT(type != nullptr); + _ASSERT(FOH_COMMIT_SIZE >= MIN_OBJECT_SIZE); + _ASSERT(FOH_SEGMENT_SIZE > FOH_COMMIT_SIZE); + _ASSERT(FOH_SEGMENT_SIZE % FOH_COMMIT_SIZE == 0); + + // NOTE: objectSize is expected be the full size including header + _ASSERT(objectSize >= MIN_OBJECT_SIZE); + + if (objectSize > FOH_COMMIT_SIZE) + { + // The current design doesn't allow objects larger than FOH_COMMIT_SIZE and + // since FrozenObjectHeap is just an optimization, let's not fill it with huge objects. + return nullptr; + } + + if (m_CurrentSegment == nullptr) + { + // Create the first segment on first allocation + m_CurrentSegment = new FrozenObjectSegment(); + m_FrozenSegments.Append(m_CurrentSegment); + _ASSERT(m_CurrentSegment != nullptr); + } + + Object* obj = m_CurrentSegment->TryAllocateObject(type, objectSize); + + // The only case where it can be null is when the current segment is full and we need + // to create a new one + if (obj == nullptr) + { + m_CurrentSegment = new FrozenObjectSegment(); + m_FrozenSegments.Append(m_CurrentSegment); + + // Try again + obj = m_CurrentSegment->TryAllocateObject(type, objectSize); + + // This time it's not expected to be null + _ASSERT(obj != nullptr); + } + return obj; +#endif // !FEATURE_BASICFREEZE +} + + +FrozenObjectSegment::FrozenObjectSegment(): + m_pStart(nullptr), + m_pCurrent(nullptr), + m_SizeCommitted(0), + m_SegmentHandle(nullptr) + COMMA_INDEBUG(m_ObjectsCount(0)) +{ + void* alloc = ClrVirtualAlloc(nullptr, FOH_SEGMENT_SIZE, MEM_RESERVE, PAGE_READWRITE); + if (alloc == nullptr) + { + ThrowOutOfMemory(); + } + + // Commit a chunk in advance + void* committedAlloc = ClrVirtualAlloc(alloc, FOH_COMMIT_SIZE, MEM_COMMIT, PAGE_READWRITE); + if (committedAlloc == nullptr) + { + ClrVirtualFree(alloc, 0, MEM_RELEASE); + ThrowOutOfMemory(); + } + + // ClrVirtualAlloc is expected to be PageSize-aligned so we can expect + // DATA_ALIGNMENT alignment as well + _ASSERT(IS_ALIGNED(committedAlloc, DATA_ALIGNMENT)); + + segment_info si; + si.pvMem = committedAlloc; + si.ibFirstObject = sizeof(ObjHeader); + si.ibAllocated = si.ibFirstObject; + si.ibCommit = FOH_COMMIT_SIZE; + si.ibReserved = FOH_SEGMENT_SIZE; + + m_SegmentHandle = GCHeapUtilities::GetGCHeap()->RegisterFrozenSegment(&si); + if (m_SegmentHandle == nullptr) + { + ClrVirtualFree(alloc, 0, MEM_RELEASE); + ThrowOutOfMemory(); + } + + m_pStart = static_cast(committedAlloc); + m_pCurrent = m_pStart; + m_SizeCommitted = si.ibCommit; + INDEBUG(m_ObjectsCount = 0); + return; +} + +Object* FrozenObjectSegment::TryAllocateObject(PTR_MethodTable type, size_t objectSize) +{ + _ASSERT(m_pStart != nullptr && FOH_SEGMENT_SIZE > 0 && m_SegmentHandle != nullptr); // Expected to be inited + _ASSERT(IS_ALIGNED(m_pCurrent, DATA_ALIGNMENT)); + + uint8_t* obj = m_pCurrent; + if (reinterpret_cast(m_pStart + FOH_SEGMENT_SIZE) < reinterpret_cast(obj + objectSize)) + { + // Segment is full + return nullptr; + } + + // Check if we need to commit a new chunk + if (reinterpret_cast(m_pStart + m_SizeCommitted) < reinterpret_cast(obj + objectSize)) + { + _ASSERT(m_SizeCommitted + FOH_COMMIT_SIZE <= FOH_SEGMENT_SIZE); + if (ClrVirtualAlloc(m_pStart + m_SizeCommitted, FOH_COMMIT_SIZE, MEM_COMMIT, PAGE_READWRITE) == nullptr) + { + ClrVirtualFree(m_pStart, 0, MEM_RELEASE); + ThrowOutOfMemory(); + } + m_SizeCommitted += FOH_COMMIT_SIZE; + } + + INDEBUG(m_ObjectsCount++); + + m_pCurrent = obj + objectSize; + + Object* object = reinterpret_cast(obj + sizeof(ObjHeader)); + object->SetMethodTable(type); + + // Notify GC that we bumped the pointer and, probably, committed more memory in the reserved part + GCHeapUtilities::GetGCHeap()->UpdateFrozenSegment(m_SegmentHandle, m_pCurrent, m_pStart + m_SizeCommitted); + + return object; +} diff --git a/src/coreclr/vm/frozenobjectheap.h b/src/coreclr/vm/frozenobjectheap.h new file mode 100644 index 00000000000000..703c3aaa6fc458 --- /dev/null +++ b/src/coreclr/vm/frozenobjectheap.h @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#ifndef _FROZENOBJECTHEAP_H +#define _FROZENOBJECTHEAP_H + +#include "gcinterface.h" +#include + +// FrozenObjectHeapManager provides a simple API to allocate objects on GC's Frozen Segments, it can be used as +// an optimization to put certain types of objects there and rely on them to be effectively pinned so, for instance, +// jit can bake direct addresses of them in codegen and avoid extra indirect loads. +// +// Example: a string literal allocated on a normal heap looks like this in JIT for x64: +// +// mov rax, 0xD1FFAB1E ; pinned handle +// mov rax, gword ptr [rax] ; actual string object +// +// and here is the same literal but allocated on a frozen segment: +// +// mov rax, 0xD1FFAB1E ; actual string object +// + +class FrozenObjectSegment; + +class FrozenObjectHeapManager +{ +public: + FrozenObjectHeapManager(); + Object* TryAllocateObject(PTR_MethodTable type, size_t objectSize); + +private: + Crst m_Crst; + SArray m_FrozenSegments; + FrozenObjectSegment* m_CurrentSegment; + bool m_Enabled; +}; + +class FrozenObjectSegment +{ +public: + FrozenObjectSegment(); + Object* TryAllocateObject(PTR_MethodTable type, size_t objectSize); + +private: + uint8_t* m_pStart; + uint8_t* m_pCurrent; + size_t m_SizeCommitted; + segment_handle m_SegmentHandle; + INDEBUG(size_t m_ObjectsCount); +}; + +#endif // _FROZENOBJECTHEAP_H + diff --git a/src/coreclr/vm/gchelpers.cpp b/src/coreclr/vm/gchelpers.cpp index 99954015166fbc..e09f4c9345cd69 100644 --- a/src/coreclr/vm/gchelpers.cpp +++ b/src/coreclr/vm/gchelpers.cpp @@ -28,6 +28,7 @@ #include "gchelpers.inl" #include "eeprofinterfaces.inl" +#include "frozenobjectheap.h" #ifdef FEATURE_COMINTEROP #include "runtimecallablewrapper.h" @@ -838,9 +839,7 @@ STRINGREF AllocateString( DWORD cchStringLength ) // Limit the maximum string size to <2GB to mitigate risk of security issues caused by 32-bit integer // overflows in buffer size calculations. - // - // If the value below is changed, also change AllocateUtf8String. - if (cchStringLength > 0x3FFFFFDF) + if (cchStringLength > CORINFO_String_MaxLength) ThrowOutOfMemory(); SIZE_T totalSize = PtrAlign(StringObject::GetSize(cchStringLength)); @@ -860,6 +859,48 @@ STRINGREF AllocateString( DWORD cchStringLength ) PublishObjectAndNotify(orString, flags); return ObjectToSTRINGREF(orString); + +} + +STRINGREF AllocateString(DWORD cchStringLength, bool preferFrozenHeap, bool* pIsFrozen) +{ + CONTRACTL{ + THROWS; + GC_TRIGGERS; + MODE_COOPERATIVE; + } CONTRACTL_END; + + _ASSERT(pIsFrozen != nullptr); + + STRINGREF orStringRef = NULL; + StringObject* orString = nullptr; + + // Limit the maximum string size to <2GB to mitigate risk of security issues caused by 32-bit integer + // overflows in buffer size calculations. + if (cchStringLength > CORINFO_String_MaxLength) + ThrowOutOfMemory(); + + const SIZE_T totalSize = PtrAlign(StringObject::GetSize(cchStringLength)); + _ASSERTE(totalSize > cchStringLength); + + if (preferFrozenHeap) + { + FrozenObjectHeapManager* foh = SystemDomain::GetFrozenObjectHeapManager(); + orString = static_cast(foh->TryAllocateObject(g_pStringClass, totalSize)); + if (orString != nullptr) + { + orString->SetStringLength(cchStringLength); + _ASSERTE(orString->GetBuffer()[cchStringLength] == W('\0')); + orStringRef = ObjectToSTRINGREF(orString); + *pIsFrozen = true; + } + } + if (orString == nullptr) + { + orStringRef = AllocateString(cchStringLength); + *pIsFrozen = false; + } + return orStringRef; } #ifdef FEATURE_COMINTEROP_UNMANAGED_ACTIVATION diff --git a/src/coreclr/vm/gchelpers.h b/src/coreclr/vm/gchelpers.h index bfebf554b0cbf8..d861f33a34f75b 100644 --- a/src/coreclr/vm/gchelpers.h +++ b/src/coreclr/vm/gchelpers.h @@ -34,7 +34,8 @@ OBJECTREF AllocatePrimitiveArray(CorElementType type, DWORD cElements); OBJECTREF AllocateObjectArray(DWORD cElements, TypeHandle ElementType, BOOL bAllocateInPinnedHeap = FALSE); // Allocate a string -STRINGREF AllocateString( DWORD cchStringLength ); +STRINGREF AllocateString(DWORD cchStringLength); +STRINGREF AllocateString(DWORD cchStringLength, bool preferFrozenHeap, bool* pIsFrozen); OBJECTREF DupArrayForCloning(BASEARRAYREF pRef); diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 86503f3d0efa53..d541266109c532 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -2420,7 +2420,7 @@ HCIMPL1(StringObject*, FramedAllocateString, DWORD stringLength) HCIMPLEND /*********************************************************************/ -OBJECTHANDLE ConstructStringLiteral(CORINFO_MODULE_HANDLE scopeHnd, mdToken metaTok) +OBJECTHANDLE ConstructStringLiteral(CORINFO_MODULE_HANDLE scopeHnd, mdToken metaTok, void** ppPinnedString) { CONTRACTL { THROWS; @@ -2431,7 +2431,7 @@ OBJECTHANDLE ConstructStringLiteral(CORINFO_MODULE_HANDLE scopeHnd, mdToken meta _ASSERTE(TypeFromToken(metaTok) == mdtString); Module* module = GetModule(scopeHnd); - return module->ResolveStringRef(metaTok); + return module->ResolveStringRef(metaTok, ppPinnedString); } /*********************************************************************/ diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index fb46cde8964eb4..bc146b76ef9119 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -11620,7 +11620,19 @@ InfoAccessType CEEJitInfo::constructStringLiteral(CORINFO_MODULE_HANDLE scopeHnd } else { - *ppValue = (LPVOID)ConstructStringLiteral(scopeHnd, metaTok); // throws + // If ConstructStringLiteral returns a pinned reference we can return it by value (IAT_VALUE) + void* ppPinnedString = nullptr; + void** ptr = (void**)ConstructStringLiteral(scopeHnd, metaTok, &ppPinnedString); + + if (ppPinnedString != nullptr) + { + *ppValue = ppPinnedString; + result = IAT_VALUE; + } + else + { + *ppValue = (void*)ptr; + } } EE_TO_JIT_TRANSITION(); @@ -11640,7 +11652,19 @@ InfoAccessType CEEJitInfo::emptyStringLiteral(void ** ppValue) InfoAccessType result = IAT_PVALUE; JIT_TO_EE_TRANSITION(); - *ppValue = StringObject::GetEmptyStringRefPtr(); + void* pinnedStr = nullptr; + void* pinnedStrHandlePtr = StringObject::GetEmptyStringRefPtr(&pinnedStr); + + if (pinnedStr != nullptr) + { + *ppValue = pinnedStr; + result = IAT_VALUE; + } + else + { + *ppValue = pinnedStr; + } + EE_TO_JIT_TRANSITION(); return result; @@ -13326,7 +13350,7 @@ BOOL LoadDynamicInfoEntry(Module *currentModule, if (rid == 0) { // Empty string - result = (size_t)StringObject::GetEmptyStringRefPtr(); + result = (size_t)StringObject::GetEmptyStringRefPtr(nullptr); } else { diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index efb57186571eb3..9b517406ce4f14 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -1095,7 +1095,10 @@ void DoGcStress (PT_CONTEXT regs, NativeCodeVersion nativeCodeVersion); EXTERN_C FCDECL2(LPVOID, ArrayStoreCheck, Object** pElement, PtrArray** pArray); -OBJECTHANDLE ConstructStringLiteral(CORINFO_MODULE_HANDLE scopeHnd, mdToken metaTok); +// ppPinnedString: If the string is pinned (e.g. allocated in frozen heap), +// the pointer to the pinned string is returned in *ppPinnedPointer. ppPinnedPointer == nullptr +// means that the caller does not care whether the string is pinned or not. +OBJECTHANDLE ConstructStringLiteral(CORINFO_MODULE_HANDLE scopeHnd, mdToken metaTok, void** ppPinnedString = nullptr); FCDECL2(Object*, JIT_Box, CORINFO_CLASS_HANDLE type, void* data); FCDECL0(VOID, JIT_PollGC); diff --git a/src/coreclr/vm/loaderallocator.cpp b/src/coreclr/vm/loaderallocator.cpp index a67b9097464f4b..f1ba447b34c952 100644 --- a/src/coreclr/vm/loaderallocator.cpp +++ b/src/coreclr/vm/loaderallocator.cpp @@ -1740,7 +1740,7 @@ void AssemblyLoaderAllocator::RegisterBinder(CustomAssemblyBinder* binderToRelea m_binderToRelease = binderToRelease; } -STRINGREF *LoaderAllocator::GetStringObjRefPtrFromUnicodeString(EEStringData *pStringData) +STRINGREF *LoaderAllocator::GetStringObjRefPtrFromUnicodeString(EEStringData *pStringData, void** ppPinnedString) { CONTRACTL { @@ -1756,7 +1756,7 @@ STRINGREF *LoaderAllocator::GetStringObjRefPtrFromUnicodeString(EEStringData *pS LazyInitStringLiteralMap(); } _ASSERTE(m_pStringLiteralMap); - return m_pStringLiteralMap->GetStringLiteral(pStringData, TRUE, !CanUnload()); + return m_pStringLiteralMap->GetStringLiteral(pStringData, TRUE, CanUnload(), ppPinnedString); } //***************************************************************************** @@ -1814,7 +1814,7 @@ STRINGREF *LoaderAllocator::IsStringInterned(STRINGREF *pString) LazyInitStringLiteralMap(); } _ASSERTE(m_pStringLiteralMap); - return m_pStringLiteralMap->GetInternedString(pString, FALSE, !CanUnload()); + return m_pStringLiteralMap->GetInternedString(pString, FALSE, CanUnload()); } STRINGREF *LoaderAllocator::GetOrInternString(STRINGREF *pString) @@ -1833,7 +1833,7 @@ STRINGREF *LoaderAllocator::GetOrInternString(STRINGREF *pString) LazyInitStringLiteralMap(); } _ASSERTE(m_pStringLiteralMap); - return m_pStringLiteralMap->GetInternedString(pString, TRUE, !CanUnload()); + return m_pStringLiteralMap->GetInternedString(pString, TRUE, CanUnload()); } void AssemblyLoaderAllocator::RegisterHandleForCleanup(OBJECTHANDLE objHandle) diff --git a/src/coreclr/vm/loaderallocator.hpp b/src/coreclr/vm/loaderallocator.hpp index cbe6ce460f3fe8..b943ea37ad4ddf 100644 --- a/src/coreclr/vm/loaderallocator.hpp +++ b/src/coreclr/vm/loaderallocator.hpp @@ -590,7 +590,7 @@ class LoaderAllocator // Methods to retrieve a pointer to the COM+ string STRINGREF for a string constant. // If the string is not currently in the hash table it will be added and if the // copy string flag is set then the string will be copied before it is inserted. - STRINGREF *GetStringObjRefPtrFromUnicodeString(EEStringData *pStringData); + STRINGREF *GetStringObjRefPtrFromUnicodeString(EEStringData *pStringData, void** ppPinnedString = nullptr); void LazyInitStringLiteralMap(); STRINGREF *IsStringInterned(STRINGREF *pString); STRINGREF *GetOrInternString(STRINGREF *pString); diff --git a/src/coreclr/vm/object.cpp b/src/coreclr/vm/object.cpp index a36b4fe9c86cf9..a6a7b20850c492 100644 --- a/src/coreclr/vm/object.cpp +++ b/src/coreclr/vm/object.cpp @@ -817,7 +817,8 @@ STRINGREF StringObject::NewString(LPCUTF8 psz, int cBytes) // STATIC MEMBER VARIABLES // // -STRINGREF* StringObject::EmptyStringRefPtr=NULL; +STRINGREF* StringObject::EmptyStringRefPtr = NULL; +bool StringObject::EmptyStringIsFrozen = false; //The special string helpers are used as flag bits for weird strings that have bytes //after the terminating 0. The only case where we use this right now is the VB BSTR as @@ -854,7 +855,9 @@ STRINGREF* StringObject::InitEmptyStringRefPtr() { GCX_COOP(); EEStringData data(0, W(""), TRUE); - EmptyStringRefPtr = SystemDomain::System()->DefaultDomain()->GetLoaderAllocator()->GetStringObjRefPtrFromUnicodeString(&data); + void* pinnedStr = nullptr; + EmptyStringRefPtr = SystemDomain::System()->DefaultDomain()->GetLoaderAllocator()->GetStringObjRefPtrFromUnicodeString(&data, &pinnedStr); + EmptyStringIsFrozen = pinnedStr != nullptr; return EmptyStringRefPtr; } diff --git a/src/coreclr/vm/object.h b/src/coreclr/vm/object.h index 4c731c4a240f4b..1b7f140aae89f1 100644 --- a/src/coreclr/vm/object.h +++ b/src/coreclr/vm/object.h @@ -928,7 +928,7 @@ class StringObject : public Object static STRINGREF NewString(LPCUTF8 psz, int cBytes); static STRINGREF GetEmptyString(); - static STRINGREF* GetEmptyStringRefPtr(); + static STRINGREF* GetEmptyStringRefPtr(void** pinnedString); static STRINGREF* InitEmptyStringRefPtr(); @@ -962,6 +962,7 @@ class StringObject : public Object private: static STRINGREF* EmptyStringRefPtr; + static bool EmptyStringIsFrozen; }; /*================================GetEmptyString================================ @@ -990,19 +991,27 @@ inline STRINGREF StringObject::GetEmptyString() { return *refptr; } -inline STRINGREF* StringObject::GetEmptyStringRefPtr() { +inline STRINGREF* StringObject::GetEmptyStringRefPtr(void** pinnedString) { CONTRACTL { THROWS; MODE_ANY; GC_TRIGGERS; } CONTRACTL_END; + STRINGREF* refptr = EmptyStringRefPtr; //If we've never gotten a reference to the EmptyString, we need to go get one. - if (refptr==NULL) { + if (refptr == nullptr) + { refptr = InitEmptyStringRefPtr(); } + + if (EmptyStringIsFrozen && pinnedString != nullptr) + { + *pinnedString = *(void**)refptr; + } + //We've already have a reference to the EmptyString, so we can just return it. return refptr; } diff --git a/src/coreclr/vm/stringliteralmap.cpp b/src/coreclr/vm/stringliteralmap.cpp index f607e3abf65d62..4cca97a22b5233 100644 --- a/src/coreclr/vm/stringliteralmap.cpp +++ b/src/coreclr/vm/stringliteralmap.cpp @@ -143,7 +143,7 @@ StringLiteralMap::~StringLiteralMap() -STRINGREF *StringLiteralMap::GetStringLiteral(EEStringData *pStringData, BOOL bAddIfNotFound, BOOL bAppDomainWontUnload) +STRINGREF *StringLiteralMap::GetStringLiteral(EEStringData *pStringData, BOOL bAddIfNotFound, BOOL bIsCollectible, void** ppPinnedString) { CONTRACTL { @@ -165,7 +165,9 @@ STRINGREF *StringLiteralMap::GetStringLiteral(EEStringData *pStringData, BOOL bA // someone beat us to inserting it. (m_StringToEntryHashTable->GetValue(pStringData, &Data)) // (Rather than waiting until after we look the string up in the global map) - StringLiteralEntryHolder pEntry(SystemDomain::GetGlobalStringLiteralMap()->GetStringLiteral(pStringData, dwHash, bAddIfNotFound)); + // Don't use FOH for collectible modules to avoid potential memory leaks + const bool preferFrozenObjectHeap = !bIsCollectible; + StringLiteralEntryHolder pEntry(SystemDomain::GetGlobalStringLiteralMap()->GetStringLiteral(pStringData, dwHash, bAddIfNotFound, preferFrozenObjectHeap)); _ASSERTE(pEntry || !bAddIfNotFound); @@ -177,7 +179,7 @@ STRINGREF *StringLiteralMap::GetStringLiteral(EEStringData *pStringData, BOOL bA // TODO: except that by not inserting into our local table we always take the global map lock // and come into this path, when we could succeed at a lock free lookup above. - if (!bAppDomainWontUnload) + if (bIsCollectible) { // Make sure some other thread has not already added it. if (!m_StringToEntryHashTable->GetValue(pStringData, &Data)) @@ -201,6 +203,12 @@ STRINGREF *StringLiteralMap::GetStringLiteral(EEStringData *pStringData, BOOL bA // Retrieve the string objectref from the string literal entry. pStrObj = pEntry->GetStringObject(); _ASSERTE(!bAddIfNotFound || pStrObj); + + + if (pStrObj != nullptr && ppPinnedString != nullptr && preferFrozenObjectHeap && pEntry->IsStringFrozen()) + { + *ppPinnedString = *reinterpret_cast(pStrObj); + } return pStrObj; } // If the bAddIfNotFound flag is set then we better have a string @@ -209,7 +217,7 @@ STRINGREF *StringLiteralMap::GetStringLiteral(EEStringData *pStringData, BOOL bA return NULL; } -STRINGREF *StringLiteralMap::GetInternedString(STRINGREF *pString, BOOL bAddIfNotFound, BOOL bAppDomainWontUnload) +STRINGREF *StringLiteralMap::GetInternedString(STRINGREF *pString, BOOL bAddIfNotFound, BOOL bIsCollectible) { CONTRACTL { @@ -242,7 +250,10 @@ STRINGREF *StringLiteralMap::GetInternedString(STRINGREF *pString, BOOL bAddIfNo // (Rather than waiting until after we look the string up in the global map) // Retrieve the string literal from the global string literal map. - StringLiteralEntryHolder pEntry(SystemDomain::GetGlobalStringLiteralMap()->GetInternedString(pString, dwHash, bAddIfNotFound)); + + // Don't use FOH for collectible modules to avoid potential memory leaks + const bool preferFrozenObjectHeap = !bIsCollectible; + StringLiteralEntryHolder pEntry(SystemDomain::GetGlobalStringLiteralMap()->GetInternedString(pString, dwHash, bAddIfNotFound, preferFrozenObjectHeap)); _ASSERTE(pEntry || !bAddIfNotFound); @@ -254,7 +265,7 @@ STRINGREF *StringLiteralMap::GetInternedString(STRINGREF *pString, BOOL bAddIfNo // TODO: except that by not inserting into our local table we always take the global map lock // and come into this path, when we could succeed at a lock free lookup above. - if (!bAppDomainWontUnload) + if (bIsCollectible) { // Since GlobalStringLiteralMap::GetInternedString() could have caused a GC, // we need to recreate the string data. @@ -357,7 +368,7 @@ void GlobalStringLiteralMap::Init() ThrowOutOfMemory(); } -StringLiteralEntry *GlobalStringLiteralMap::GetStringLiteral(EEStringData *pStringData, DWORD dwHash, BOOL bAddIfNotFound) +StringLiteralEntry *GlobalStringLiteralMap::GetStringLiteral(EEStringData *pStringData, DWORD dwHash, BOOL bAddIfNotFound, BOOL bPreferFrozenObjectHeap) { CONTRACTL { @@ -383,13 +394,15 @@ StringLiteralEntry *GlobalStringLiteralMap::GetStringLiteral(EEStringData *pStri else { if (bAddIfNotFound) - pEntry = AddStringLiteral(pStringData); + { + pEntry = AddStringLiteral(pStringData, bPreferFrozenObjectHeap); + } } return pEntry; } -StringLiteralEntry *GlobalStringLiteralMap::GetInternedString(STRINGREF *pString, DWORD dwHash, BOOL bAddIfNotFound) +StringLiteralEntry *GlobalStringLiteralMap::GetInternedString(STRINGREF *pString, DWORD dwHash, BOOL bAddIfNotFound, BOOL bPreferFrozenObjectHeap) { CONTRACTL { @@ -417,7 +430,7 @@ StringLiteralEntry *GlobalStringLiteralMap::GetInternedString(STRINGREF *pString else { if (bAddIfNotFound) - pEntry = AddInternedString(pString); + pEntry = AddInternedString(pString, bPreferFrozenObjectHeap); } return pEntry; @@ -439,7 +452,7 @@ static void LogStringLiteral(_In_z_ const char* action, EEStringData *pStringDat } #endif -STRINGREF AllocateStringObject(EEStringData *pStringData) +STRINGREF AllocateStringObject(EEStringData *pStringData, bool preferFrozenObjHeap, bool* pIsFrozen) { CONTRACTL { @@ -452,7 +465,7 @@ STRINGREF AllocateStringObject(EEStringData *pStringData) // Create the COM+ string object. DWORD cCount = pStringData->GetCharCount(); - STRINGREF strObj = AllocateString(cCount); + STRINGREF strObj = AllocateString(cCount, preferFrozenObjHeap, pIsFrozen); GCPROTECT_BEGIN(strObj) { @@ -468,7 +481,7 @@ STRINGREF AllocateStringObject(EEStringData *pStringData) return strObj; } -StringLiteralEntry *GlobalStringLiteralMap::AddStringLiteral(EEStringData *pStringData) +StringLiteralEntry *GlobalStringLiteralMap::AddStringLiteral(EEStringData *pStringData, bool preferFrozenObjHeap) { CONTRACTL { @@ -482,32 +495,46 @@ StringLiteralEntry *GlobalStringLiteralMap::AddStringLiteral(EEStringData *pStri StringLiteralEntry *pRet; + bool isFrozen = false; + STRINGREF strObj = AllocateStringObject(pStringData, preferFrozenObjHeap, &isFrozen); + if (isFrozen) { - PinnedHeapHandleBlockHolder pStrObj(&m_PinnedHeapHandleTable,1); - // Create the COM+ string object. - STRINGREF strObj = AllocateStringObject(pStringData); - - // Allocate a handle for the string. - SetObjectReference(pStrObj[0], (OBJECTREF) strObj); + _ASSERT(preferFrozenObjHeap); + StringLiteralEntryHolder pEntry(StringLiteralEntry::AllocateFrozenEntry(pStringData, strObj)); + m_StringToEntryHashTable->InsertValue(pStringData, pEntry, FALSE); + pEntry.SuppressRelease(); + pRet = pEntry; + _ASSERT(pRet->IsStringFrozen()); + } + else + { + GCPROTECT_BEGIN(strObj) + { + // If it's not frozen we need to gc allocate a pinned handle to the non-pinned string object + PinnedHeapHandleBlockHolder pStrObj(&m_PinnedHeapHandleTable,1); + // Allocate a handle for the string. + SetObjectReference(pStrObj[0], strObj); - // Allocate the StringLiteralEntry. - StringLiteralEntryHolder pEntry(StringLiteralEntry::AllocateEntry(pStringData, (STRINGREF*)pStrObj[0])); - pStrObj.SuppressRelease(); - // Insert the handle to the string into the hash table. - m_StringToEntryHashTable->InsertValue(pStringData, (LPVOID)pEntry, FALSE); - pEntry.SuppressRelease(); - pRet = pEntry; + // Allocate the StringLiteralEntry. + StringLiteralEntryHolder pEntry(StringLiteralEntry::AllocateEntry(pStringData, (STRINGREF*)pStrObj[0])); + pStrObj.SuppressRelease(); + // Insert the handle to the string into the hash table. + m_StringToEntryHashTable->InsertValue(pStringData, pEntry, FALSE); + pEntry.SuppressRelease(); + pRet = pEntry; + } + GCPROTECT_END(); + } #ifdef LOGGING LogStringLiteral("added", pStringData); #endif - } return pRet; } -StringLiteralEntry *GlobalStringLiteralMap::AddInternedString(STRINGREF *pString) +StringLiteralEntry *GlobalStringLiteralMap::AddInternedString(STRINGREF *pString, bool preferFrozenObjHeap) { CONTRACTL @@ -521,26 +548,7 @@ StringLiteralEntry *GlobalStringLiteralMap::AddInternedString(STRINGREF *pString CONTRACTL_END; EEStringData StringData = EEStringData((*pString)->GetStringLength(), (*pString)->GetBuffer()); - StringLiteralEntry *pRet; - - { - PinnedHeapHandleBlockHolder pStrObj(&m_PinnedHeapHandleTable,1); - SetObjectReference(pStrObj[0], (OBJECTREF) *pString); - - // Since the allocation might have caused a GC we need to re-get the - // string data. - StringData = EEStringData((*pString)->GetStringLength(), (*pString)->GetBuffer()); - - StringLiteralEntryHolder pEntry(StringLiteralEntry::AllocateEntry(&StringData, (STRINGREF*)pStrObj[0])); - pStrObj.SuppressRelease(); - - // Insert the handle to the string into the hash table. - m_StringToEntryHashTable->InsertValue(&StringData, (LPVOID)pEntry, FALSE); - pEntry.SuppressRelease(); - pRet = pEntry; - } - - return pRet; + return AddStringLiteral(&StringData, preferFrozenObjHeap); } void GlobalStringLiteralMap::RemoveStringLiteralEntry(StringLiteralEntry *pEntry) @@ -577,16 +585,19 @@ void GlobalStringLiteralMap::RemoveStringLiteralEntry(StringLiteralEntry *pEntry } #endif - // Release the object handle that the entry was using. - STRINGREF *pObjRef = pEntry->GetStringObject(); - m_PinnedHeapHandleTable.ReleaseHandles((OBJECTREF*)pObjRef, 1); + if (!pEntry->IsStringFrozen()) + { + // Release the object handle that the entry was using. + STRINGREF *pObjRef = pEntry->GetStringObject(); + m_PinnedHeapHandleTable.ReleaseHandles((OBJECTREF*)pObjRef, 1); + } } // We do not delete the StringLiteralEntry itself that will be done in the // release method of the StringLiteralEntry. } -StringLiteralEntry *StringLiteralEntry::AllocateEntry(EEStringData *pStringData, STRINGREF *pStringObj) +void* StringLiteralEntry::AllocateEntryInternal() { CONTRACTL { @@ -618,7 +629,17 @@ StringLiteralEntry *StringLiteralEntry::AllocateEntry(EEStringData *pStringData, } _ASSERTE (pMem && "Unable to allocate String literal Entry"); - return new (pMem) StringLiteralEntry (pStringData, pStringObj); + return pMem; +} + +StringLiteralEntry* StringLiteralEntry::AllocateEntry(EEStringData* pStringData, STRINGREF* pStringObj) +{ + return new (AllocateEntryInternal()) StringLiteralEntry(pStringData, pStringObj); +} + +StringLiteralEntry* StringLiteralEntry::AllocateFrozenEntry(EEStringData* pStringData, STRINGREF pFrozenStringObj) +{ + return new (AllocateEntryInternal()) StringLiteralEntry(pStringData, pFrozenStringObj); } void StringLiteralEntry::DeleteEntry (StringLiteralEntry *pEntry) @@ -631,8 +652,7 @@ void StringLiteralEntry::DeleteEntry (StringLiteralEntry *pEntry) } CONTRACTL_END; - _ASSERTE (VolatileLoad(&pEntry->m_dwRefCount) == 0); - + _ASSERTE (pEntry->IsStringFrozen() || pEntry->GetRefCount() == 0); #ifdef _DEBUG memset (&pEntry->m_pStringObj, 0xc, sizeof(pEntry->m_pStringObj)); pEntry->m_bDeleted = TRUE; diff --git a/src/coreclr/vm/stringliteralmap.h b/src/coreclr/vm/stringliteralmap.h index 22622733c779ba..510b66b32bd687 100644 --- a/src/coreclr/vm/stringliteralmap.h +++ b/src/coreclr/vm/stringliteralmap.h @@ -22,7 +22,7 @@ class StringLiteralEntry; // Allocate 16 entries (approx size sizeof(StringLiteralEntry)*16) #define MAX_ENTRIES_PER_CHUNK 16 -STRINGREF AllocateStringObject(EEStringData *pStringData); +STRINGREF AllocateStringObject(EEStringData *pStringData, bool preferFrozenObjHeap, bool* pIsFrozen); // Loader allocator specific string literal map. class StringLiteralMap @@ -42,10 +42,10 @@ class StringLiteralMap } // Method to retrieve a string from the map. - STRINGREF *GetStringLiteral(EEStringData *pStringData, BOOL bAddIfNotFound, BOOL bAppDomainWontUnload); + STRINGREF *GetStringLiteral(EEStringData *pStringData, BOOL bAddIfNotFound, BOOL bIsCollectible, void** ppPinnedString = nullptr); // Method to explicitly intern a string object. - STRINGREF *GetInternedString(STRINGREF *pString, BOOL bAddIfNotFound, BOOL bAppDomainWontUnload); + STRINGREF *GetInternedString(STRINGREF *pString, BOOL bAddIfNotFound, BOOL bIsCollectible); private: // Hash tables that maps a Unicode string to a COM+ string handle. @@ -71,10 +71,10 @@ class GlobalStringLiteralMap void Init(); // Method to retrieve a string from the map. Takes a precomputed hash (for perf). - StringLiteralEntry *GetStringLiteral(EEStringData *pStringData, DWORD dwHash, BOOL bAddIfNotFound); + StringLiteralEntry *GetStringLiteral(EEStringData *pStringData, DWORD dwHash, BOOL bAddIfNotFound, BOOL bPreferFrozenObjectHeap); // Method to explicitly intern a string object. Takes a precomputed hash (for perf). - StringLiteralEntry *GetInternedString(STRINGREF *pString, DWORD dwHash, BOOL bAddIfNotFound); + StringLiteralEntry *GetInternedString(STRINGREF *pString, DWORD dwHash, BOOL bAddIfNotFound, BOOL bPreferFrozenObjectHeap); // Method to calculate the hash DWORD GetHash(EEStringData* pData) @@ -92,10 +92,10 @@ class GlobalStringLiteralMap private: // Helper method to add a string to the global string literal map. - StringLiteralEntry *AddStringLiteral(EEStringData *pStringData); + StringLiteralEntry *AddStringLiteral(EEStringData *pStringData, bool preferFrozenObjHeap); // Helper method to add an interned string. - StringLiteralEntry *AddInternedString(STRINGREF *pString); + StringLiteralEntry *AddInternedString(STRINGREF *pString, bool preferFrozenObjHeap); // Called by StringLiteralEntry when its RefCount falls to 0. void RemoveStringLiteralEntry(StringLiteralEntry *pEntry); @@ -123,6 +123,10 @@ class StringLiteralEntryArray; // Ref counted entry representing a string literal. class StringLiteralEntry { + #define SLE_IS_FROZEN (1u << 31) + #define SLE_IS_OVERFLOWED (1u << 30) + #define SLE_REFCOUNT_MASK (SLE_IS_FROZEN | SLE_IS_OVERFLOWED) + private: StringLiteralEntry(EEStringData *pStringData, STRINGREF *pStringObj) : m_pStringObj(pStringObj), m_dwRefCount(1) @@ -132,6 +136,16 @@ class StringLiteralEntry { LIMITED_METHOD_CONTRACT; } + + StringLiteralEntry(EEStringData *pStringData, STRINGREF frozenStringObj) + : m_FrozenStringObj(frozenStringObj), m_dwRefCount(1 | SLE_IS_FROZEN) +#ifdef _DEBUG + , m_bDeleted(FALSE) +#endif + { + LIMITED_METHOD_CONTRACT; + } + protected: ~StringLiteralEntry() { @@ -152,18 +166,24 @@ class StringLiteralEntry NOTHROW; GC_NOTRIGGER; PRECONDITION(CheckPointer(this)); - PRECONDITION((LONG)VolatileLoad(&m_dwRefCount) > 0); + PRECONDITION(GetRefCount() != 0); PRECONDITION(SystemDomain::GetGlobalStringLiteralMapNoCreate()->m_HashTableCrstGlobal.OwnedByCurrentThread()); } CONTRACTL_END; _ASSERTE (!m_bDeleted); - // We will keep the item alive forever if the refcount overflowed - if ((LONG)VolatileLoad(&m_dwRefCount) < 0) + if (IsAlwaysAlive()) return; - VolatileStore(&m_dwRefCount, VolatileLoad(&m_dwRefCount) + 1); + if ((GetRefCount() + 1) & SLE_IS_OVERFLOWED) + { + VolatileStore(&m_dwRefCount, VolatileLoad(&m_dwRefCount) | SLE_IS_OVERFLOWED); + } + else + { + VolatileStore(&m_dwRefCount, VolatileLoad(&m_dwRefCount) + 1); + } } #ifndef DACCESS_COMPILE FORCEINLINE static void StaticRelease(StringLiteralEntry* pEntry) @@ -192,17 +212,16 @@ class StringLiteralEntry NOTHROW; GC_NOTRIGGER; PRECONDITION(CheckPointer(this)); - PRECONDITION(VolatileLoad(&m_dwRefCount) > 0); + PRECONDITION(GetRefCount() > 0); PRECONDITION(SystemDomain::GetGlobalStringLiteralMapNoCreate()->m_HashTableCrstGlobal.OwnedByCurrentThread()); } CONTRACTL_END; - // We will keep the item alive forever if the refcount overflowed - if ((LONG)VolatileLoad(&m_dwRefCount) < 0) + if (IsAlwaysAlive()) return; VolatileStore(&m_dwRefCount, VolatileLoad(&m_dwRefCount) - 1); - if (VolatileLoad(&m_dwRefCount) == 0) + if (GetRefCount() == 0) { _ASSERTE(SystemDomain::GetGlobalStringLiteralMapNoCreate()); SystemDomain::GetGlobalStringLiteralMapNoCreate()->RemoveStringLiteralEntry(this); @@ -212,7 +231,7 @@ class StringLiteralEntry } #endif // DACCESS_COMPILE - LONG GetRefCount() + DWORD GetRefCount() { CONTRACTL { @@ -224,7 +243,7 @@ class StringLiteralEntry _ASSERTE (!m_bDeleted); - return (VolatileLoad(&m_dwRefCount)); + return VolatileLoad(&m_dwRefCount) & ~SLE_REFCOUNT_MASK; } STRINGREF* GetStringObject() @@ -236,7 +255,7 @@ class StringLiteralEntry PRECONDITION(CheckPointer(this)); } CONTRACTL_END; - return m_pStringObj; + return IsStringFrozen() ? &m_FrozenStringObj : m_pStringObj; } void GetStringData(EEStringData *pStringData) @@ -254,20 +273,41 @@ class StringLiteralEntry WCHAR *thisChars; int thisLength; - ObjectToSTRINGREF(*(StringObject**)m_pStringObj)->RefInterpretGetStringValuesDangerousForGC(&thisChars, &thisLength); + ObjectToSTRINGREF(*GetStringObject())->RefInterpretGetStringValuesDangerousForGC(&thisChars, &thisLength); pStringData->SetCharCount (thisLength); // thisLength is in WCHARs and that's what EEStringData's char count wants pStringData->SetStringBuffer (thisChars); } - static StringLiteralEntry *AllocateEntry(EEStringData *pStringData, STRINGREF *pStringObj); +private: + static void* AllocateEntryInternal(); + +public: + static StringLiteralEntry *AllocateEntry(EEStringData *pStringData, STRINGREF* pStringObj); + static StringLiteralEntry* AllocateFrozenEntry(EEStringData* pStringData, STRINGREF pFrozenStringObj); static void DeleteEntry (StringLiteralEntry *pEntry); + bool IsStringFrozen() + { + return VolatileLoad(&m_dwRefCount) & SLE_IS_FROZEN; + } + + bool IsAlwaysAlive() + { + // If string literal is either frozen or its counter overflowed + // we'll keep it always alive + return VolatileLoad(&m_dwRefCount) & (SLE_IS_OVERFLOWED | SLE_IS_FROZEN); + } + private: - STRINGREF* m_pStringObj; union { - DWORD m_dwRefCount; - StringLiteralEntry *m_pNext; + STRINGREF* m_pStringObj; + STRINGREF m_FrozenStringObj; + }; + union + { + DWORD m_dwRefCount; + StringLiteralEntry *m_pNext; }; #ifdef _DEBUG diff --git a/src/coreclr/vm/syncblk.h b/src/coreclr/vm/syncblk.h index dbad515e8bb309..17d8074c810bf1 100644 --- a/src/coreclr/vm/syncblk.h +++ b/src/coreclr/vm/syncblk.h @@ -74,13 +74,9 @@ class AppDomain; #ifdef EnC_SUPPORTED class EnCSyncBlockInfo; typedef DPTR(EnCSyncBlockInfo) PTR_EnCSyncBlockInfo; - #endif // EnC_SUPPORTED #include "eventstore.hpp" - -#include "eventstore.hpp" - #include "synch.h" // At a negative offset from each Object is an ObjHeader. The 'size' of the diff --git a/src/coreclr/vm/threads.h b/src/coreclr/vm/threads.h index b51804a471705d..54da8e94405fcc 100644 --- a/src/coreclr/vm/threads.h +++ b/src/coreclr/vm/threads.h @@ -3738,6 +3738,9 @@ class Thread // If the pointer lives in the GC heap, than it is protected, and thus valid. if (dac_cast(g_lowest_address) <= val && val < dac_cast(g_highest_address)) return(true); + // Same for frozen segments + if (GCHeapUtilities::GetGCHeap()->IsInFrozenSegment(*(Object**)ref)) + return(true); return(false); }