diff --git a/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h b/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h index dacd100184d6f..0f52225c57794 100644 --- a/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h @@ -105,11 +105,24 @@ class LLVM_ABI SectionMemoryManager : public RTDyldMemoryManager { /// Creates a SectionMemoryManager instance with \p MM as the associated /// memory mapper. If \p MM is nullptr then a default memory mapper is used /// that directly calls into the operating system. - SectionMemoryManager(MemoryMapper *MM = nullptr); + /// + /// If \p ReserveAlloc is true all memory will be pre-allocated, and any + /// attempts to allocate beyond pre-allocated memory will fail. + SectionMemoryManager(MemoryMapper *MM = nullptr, bool ReserveAlloc = false); SectionMemoryManager(const SectionMemoryManager &) = delete; void operator=(const SectionMemoryManager &) = delete; ~SectionMemoryManager() override; + /// Enable reserveAllocationSpace when requested. + bool needsToReserveAllocationSpace() override { return ReserveAllocation; } + + /// Implements allocating all memory in a single block. This is required to + /// limit memory offsets to fit the ARM ABI; large memory systems may + /// otherwise allocate separate sections too far apart. + void reserveAllocationSpace(uintptr_t CodeSize, Align CodeAlign, + uintptr_t RODataSize, Align RODataAlign, + uintptr_t RWDataSize, Align RWDataAlign) override; + /// Allocates a memory block of (at least) the given size suitable for /// executable code. /// @@ -181,6 +194,8 @@ class LLVM_ABI SectionMemoryManager : public RTDyldMemoryManager { std::error_code applyMemoryGroupPermissions(MemoryGroup &MemGroup, unsigned Permissions); + bool hasSpace(const MemoryGroup &MemGroup, uintptr_t Size) const; + void anchor() override; MemoryGroup CodeMem; @@ -188,6 +203,7 @@ class LLVM_ABI SectionMemoryManager : public RTDyldMemoryManager { MemoryGroup RODataMem; MemoryMapper *MMapper; std::unique_ptr OwnedMMapper; + bool ReserveAllocation; }; } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp b/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp index e3391992b8204..8607de338916a 100644 --- a/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp @@ -17,6 +17,97 @@ namespace llvm { +bool SectionMemoryManager::hasSpace(const MemoryGroup &MemGroup, + uintptr_t Size) const { + for (const FreeMemBlock &FreeMB : MemGroup.FreeMem) { + if (FreeMB.Free.allocatedSize() >= Size) + return true; + } + return false; +} + +void SectionMemoryManager::reserveAllocationSpace( + uintptr_t CodeSize, Align CodeAlign, uintptr_t RODataSize, + Align RODataAlign, uintptr_t RWDataSize, Align RWDataAlign) { + if (CodeSize == 0 && RODataSize == 0 && RWDataSize == 0) + return; + + static const size_t PageSize = sys::Process::getPageSizeEstimate(); + + // Code alignment needs to be at least the stub alignment - however, we + // don't have an easy way to get that here so as a workaround, we assume + // it's 8, which is the largest value I observed across all platforms. + constexpr uint64_t StubAlign = 8; + CodeAlign = Align(std::max(CodeAlign.value(), StubAlign)); + RODataAlign = Align(std::max(RODataAlign.value(), StubAlign)); + RWDataAlign = Align(std::max(RWDataAlign.value(), StubAlign)); + + // Get space required for each section. Use the same calculation as + // allocateSection because we need to be able to satisfy it. + uint64_t RequiredCodeSize = alignTo(CodeSize, CodeAlign) + CodeAlign.value(); + uint64_t RequiredRODataSize = + alignTo(RODataSize, RODataAlign) + RODataAlign.value(); + uint64_t RequiredRWDataSize = + alignTo(RWDataSize, RWDataAlign) + RWDataAlign.value(); + + if (hasSpace(CodeMem, RequiredCodeSize) && + hasSpace(RODataMem, RequiredRODataSize) && + hasSpace(RWDataMem, RequiredRWDataSize)) { + // Sufficient space in contiguous block already available. + return; + } + + // MemoryManager does not have functions for releasing memory after it's + // allocated. Normally it tries to use any excess blocks that were allocated + // due to page alignment, but if we have insufficient free memory for the + // request this can lead to allocating disparate memory that can violate the + // ARM ABI. Clear free memory so only the new allocations are used, but do + // not release allocated memory as it may still be in-use. + CodeMem.FreeMem.clear(); + RODataMem.FreeMem.clear(); + RWDataMem.FreeMem.clear(); + + // Round up to the nearest page size. Blocks must be page-aligned. + RequiredCodeSize = alignTo(RequiredCodeSize, PageSize); + RequiredRODataSize = alignTo(RequiredRODataSize, PageSize); + RequiredRWDataSize = alignTo(RequiredRWDataSize, PageSize); + uint64_t RequiredSize = + RequiredCodeSize + RequiredRODataSize + RequiredRWDataSize; + + std::error_code ec; + sys::MemoryBlock MB = MMapper->allocateMappedMemory( + AllocationPurpose::RWData, RequiredSize, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec); + if (ec) { + return; + } + // CodeMem will arbitrarily own this MemoryBlock to handle cleanup. + CodeMem.AllocatedMem.push_back(MB); + uintptr_t Addr = (uintptr_t)MB.base(); + FreeMemBlock FreeMB; + FreeMB.PendingPrefixIndex = (unsigned)-1; + + if (CodeSize > 0) { + assert(isAddrAligned(CodeAlign, (void *)Addr)); + FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredCodeSize); + CodeMem.FreeMem.push_back(FreeMB); + Addr += RequiredCodeSize; + } + + if (RODataSize > 0) { + assert(isAddrAligned(RODataAlign, (void *)Addr)); + FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredRODataSize); + RODataMem.FreeMem.push_back(FreeMB); + Addr += RequiredRODataSize; + } + + if (RWDataSize > 0) { + assert(isAddrAligned(RWDataAlign, (void *)Addr)); + FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredRWDataSize); + RWDataMem.FreeMem.push_back(FreeMB); + } +} + uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size, unsigned Alignment, unsigned SectionID, @@ -264,8 +355,10 @@ class DefaultMMapper final : public SectionMemoryManager::MemoryMapper { }; } // namespace -SectionMemoryManager::SectionMemoryManager(MemoryMapper *UnownedMM) - : MMapper(UnownedMM), OwnedMMapper(nullptr) { +SectionMemoryManager::SectionMemoryManager(MemoryMapper *UnownedMM, + bool ReserveAlloc) + : MMapper(UnownedMM), OwnedMMapper(nullptr), + ReserveAllocation(ReserveAlloc) { if (!MMapper) { OwnedMMapper = std::make_unique(); MMapper = OwnedMMapper.get(); diff --git a/llvm/unittests/ExecutionEngine/MCJIT/MCJITMemoryManagerTest.cpp b/llvm/unittests/ExecutionEngine/MCJIT/MCJITMemoryManagerTest.cpp index 7a756a707160a..109e38be0a8ef 100644 --- a/llvm/unittests/ExecutionEngine/MCJIT/MCJITMemoryManagerTest.cpp +++ b/llvm/unittests/ExecutionEngine/MCJIT/MCJITMemoryManagerTest.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/Support/Process.h" #include "gtest/gtest.h" using namespace llvm; @@ -16,15 +17,17 @@ namespace { TEST(MCJITMemoryManagerTest, BasicAllocations) { std::unique_ptr MemMgr(new SectionMemoryManager()); + EXPECT_FALSE(MemMgr->needsToReserveAllocationSpace()); + uint8_t *code1 = MemMgr->allocateCodeSection(256, 0, 1, ""); uint8_t *data1 = MemMgr->allocateDataSection(256, 0, 2, "", true); uint8_t *code2 = MemMgr->allocateCodeSection(256, 0, 3, ""); uint8_t *data2 = MemMgr->allocateDataSection(256, 0, 4, "", false); - EXPECT_NE((uint8_t*)nullptr, code1); - EXPECT_NE((uint8_t*)nullptr, code2); - EXPECT_NE((uint8_t*)nullptr, data1); - EXPECT_NE((uint8_t*)nullptr, data2); + EXPECT_NE((uint8_t *)nullptr, code1); + EXPECT_NE((uint8_t *)nullptr, code2); + EXPECT_NE((uint8_t *)nullptr, data1); + EXPECT_NE((uint8_t *)nullptr, data2); // Initialize the data for (unsigned i = 0; i < 256; ++i) { @@ -54,10 +57,10 @@ TEST(MCJITMemoryManagerTest, LargeAllocations) { uint8_t *code2 = MemMgr->allocateCodeSection(0x100000, 0, 3, ""); uint8_t *data2 = MemMgr->allocateDataSection(0x100000, 0, 4, "", false); - EXPECT_NE((uint8_t*)nullptr, code1); - EXPECT_NE((uint8_t*)nullptr, code2); - EXPECT_NE((uint8_t*)nullptr, data1); - EXPECT_NE((uint8_t*)nullptr, data2); + EXPECT_NE((uint8_t *)nullptr, code1); + EXPECT_NE((uint8_t *)nullptr, code2); + EXPECT_NE((uint8_t *)nullptr, data1); + EXPECT_NE((uint8_t *)nullptr, data2); // Initialize the data for (unsigned i = 0; i < 0x100000; ++i) { @@ -82,8 +85,8 @@ TEST(MCJITMemoryManagerTest, LargeAllocations) { TEST(MCJITMemoryManagerTest, ManyAllocations) { std::unique_ptr MemMgr(new SectionMemoryManager()); - uint8_t* code[10000]; - uint8_t* data[10000]; + uint8_t *code[10000]; + uint8_t *data[10000]; for (unsigned i = 0; i < 10000; ++i) { const bool isReadOnly = i % 2 == 0; @@ -117,8 +120,8 @@ TEST(MCJITMemoryManagerTest, ManyAllocations) { TEST(MCJITMemoryManagerTest, ManyVariedAllocations) { std::unique_ptr MemMgr(new SectionMemoryManager()); - uint8_t* code[10000]; - uint8_t* data[10000]; + uint8_t *code[10000]; + uint8_t *data[10000]; for (unsigned i = 0; i < 10000; ++i) { uintptr_t CodeSize = i % 16 + 1; @@ -165,5 +168,241 @@ TEST(MCJITMemoryManagerTest, ManyVariedAllocations) { } } +TEST(MCJITMemoryManagerTest, PreAllocation) { + std::unique_ptr MemMgr( + new SectionMemoryManager(nullptr, true)); + + EXPECT_TRUE(MemMgr->needsToReserveAllocationSpace()); + + llvm::Align Align{16}; + MemMgr->reserveAllocationSpace(512, Align, 256, Align, 256, Align); + + uint8_t *code1 = MemMgr->allocateCodeSection(256, 0, 1, ""); + uint8_t *data1 = MemMgr->allocateDataSection(256, 0, 2, "", true); + uint8_t *code2 = MemMgr->allocateCodeSection(256, 0, 3, ""); + uint8_t *data2 = MemMgr->allocateDataSection(256, 0, 4, "", false); + + uint8_t *minAddr = std::min({code1, data1, code2, data2}); + uint8_t *maxAddr = std::max({code1, data1, code2, data2}); + + EXPECT_NE((uint8_t *)nullptr, code1); + EXPECT_NE((uint8_t *)nullptr, code2); + EXPECT_NE((uint8_t *)nullptr, data1); + EXPECT_NE((uint8_t *)nullptr, data2); + + // Initialize the data + for (unsigned i = 0; i < 256; ++i) { + code1[i] = 1; + code2[i] = 2; + data1[i] = 3; + data2[i] = 4; + } + + // Verify the data (this is checking for overlaps in the addresses) + for (unsigned i = 0; i < 256; ++i) { + EXPECT_EQ(1, code1[i]); + EXPECT_EQ(2, code2[i]); + EXPECT_EQ(3, data1[i]); + EXPECT_EQ(4, data2[i]); + } + + std::string Error; + EXPECT_FALSE(MemMgr->finalizeMemory(&Error)); + + MemMgr->reserveAllocationSpace(512, Align, 256, Align, 256, Align); + + code1 = MemMgr->allocateCodeSection(256, 0, 1, ""); + data1 = MemMgr->allocateDataSection(256, 0, 2, "", true); + code2 = MemMgr->allocateCodeSection(256, 0, 3, ""); + data2 = MemMgr->allocateDataSection(256, 0, 4, "", false); + + EXPECT_NE((uint8_t *)nullptr, code1); + EXPECT_NE((uint8_t *)nullptr, code2); + EXPECT_NE((uint8_t *)nullptr, data1); + EXPECT_NE((uint8_t *)nullptr, data2); + + // Validate difference is more than 3x PageSize (the original reservation). + minAddr = std::min({minAddr, code1, data1, code2, data2}); + maxAddr = std::max({maxAddr, code1, data1, code2, data2}); + EXPECT_GT(maxAddr - minAddr, 3 * sys::Process::getPageSizeEstimate()); + + // Initialize the data + for (unsigned i = 0; i < 256; ++i) { + code1[i] = 1; + code2[i] = 2; + data1[i] = 3; + data2[i] = 4; + } + + // Verify the data (this is checking for overlaps in the addresses) + for (unsigned i = 0; i < 256; ++i) { + EXPECT_EQ(1, code1[i]); + EXPECT_EQ(2, code2[i]); + EXPECT_EQ(3, data1[i]); + EXPECT_EQ(4, data2[i]); + } + + EXPECT_FALSE(MemMgr->finalizeMemory(&Error)); +} + +TEST(MCJITMemoryManagerTest, PreAllocationReuse) { + std::unique_ptr MemMgr( + new SectionMemoryManager(nullptr, true)); + + EXPECT_TRUE(MemMgr->needsToReserveAllocationSpace()); + + // Reserve PageSize, because finalizeMemory eliminates blocks that aren't a + // full page size. Alignment adjustment will ensure that 2 pages are + // allocated for each section. + const unsigned PageSize = sys::Process::getPageSizeEstimate(); + EXPECT_GE(PageSize, 512u); + llvm::Align Align{16}; + MemMgr->reserveAllocationSpace(PageSize, Align, PageSize, Align, PageSize, + Align); + + uint8_t *code1 = MemMgr->allocateCodeSection(256, 0, 1, ""); + uint8_t *data1 = MemMgr->allocateDataSection(256, 0, 2, "", true); + uint8_t *code2 = MemMgr->allocateCodeSection(256, 0, 3, ""); + uint8_t *data2 = MemMgr->allocateDataSection(256, 0, 4, "", false); + + uint8_t *minAddr = std::min({code1, data1, code2, data2}); + uint8_t *maxAddr = std::max({code1, data1, code2, data2}); + + EXPECT_NE((uint8_t *)nullptr, code1); + EXPECT_NE((uint8_t *)nullptr, code2); + EXPECT_NE((uint8_t *)nullptr, data1); + EXPECT_NE((uint8_t *)nullptr, data2); + + // Initialize the data + for (unsigned i = 0; i < 256; ++i) { + code1[i] = 1; + code2[i] = 2; + data1[i] = 3; + data2[i] = 4; + } + + // Verify the data (this is checking for overlaps in the addresses) + for (unsigned i = 0; i < 256; ++i) { + EXPECT_EQ(1, code1[i]); + EXPECT_EQ(2, code2[i]); + EXPECT_EQ(3, data1[i]); + EXPECT_EQ(4, data2[i]); + } + + std::string Error; + EXPECT_FALSE(MemMgr->finalizeMemory(&Error)); + + // Each type of data is allocated on PageSize (usually 4KB). Allocate again + // and guarantee we get requests in the same block. + MemMgr->reserveAllocationSpace(512, Align, 256, Align, 256, Align); + + code1 = MemMgr->allocateCodeSection(256, 0, 5, ""); + data1 = MemMgr->allocateDataSection(256, 0, 6, "", true); + code2 = MemMgr->allocateCodeSection(256, 0, 7, ""); + data2 = MemMgr->allocateDataSection(256, 0, 8, "", false); + + EXPECT_NE((uint8_t *)nullptr, code1); + EXPECT_NE((uint8_t *)nullptr, code2); + EXPECT_NE((uint8_t *)nullptr, data1); + EXPECT_NE((uint8_t *)nullptr, data2); + + // Validate difference is less than 6x PageSize + minAddr = std::min({minAddr, code1, data1, code2, data2}); + maxAddr = std::max({maxAddr, code1, data1, code2, data2}); + EXPECT_LT(maxAddr - minAddr, 6 * PageSize); + + // Initialize the data + for (unsigned i = 0; i < 256; ++i) { + code1[i] = 1; + code2[i] = 2; + data1[i] = 3; + data2[i] = 4; + } + + // Verify the data (this is checking for overlaps in the addresses) + for (unsigned i = 0; i < 256; ++i) { + EXPECT_EQ(1, code1[i]); + EXPECT_EQ(2, code2[i]); + EXPECT_EQ(3, data1[i]); + EXPECT_EQ(4, data2[i]); + } + + EXPECT_FALSE(MemMgr->finalizeMemory(&Error)); +} + +TEST(MCJITMemoryManagerTest, ManyPreAllocation) { + std::unique_ptr MemMgr( + new SectionMemoryManager(nullptr, true)); + + uint8_t *code[10000]; + uint8_t *data[10000]; + + // Total size computation needs to take into account how much memory will be + // used including alignment. + uintptr_t CodeSize = 0, RODataSize = 0, RWDataSize = 0; + for (unsigned i = 0; i < 10000; ++i) { + unsigned Align = 8 << (i % 4); + CodeSize += alignTo(i % 16 + 1, Align); + if (i % 3 == 0) { + RODataSize += alignTo(i % 8 + 1, Align); + } else { + RWDataSize += alignTo(i % 8 + 1, Align); + } + } + llvm::Align Align = llvm::Align(8); + MemMgr->reserveAllocationSpace(CodeSize, Align, RODataSize, Align, RWDataSize, + Align); + uint8_t *minAddr = (uint8_t *)std::numeric_limits::max(); + uint8_t *maxAddr = (uint8_t *)std::numeric_limits::min(); + + for (unsigned i = 0; i < 10000; ++i) { + uintptr_t CodeSize = i % 16 + 1; + uintptr_t DataSize = i % 8 + 1; + + bool isReadOnly = i % 3 == 0; + unsigned Align = 8 << (i % 4); + + code[i] = MemMgr->allocateCodeSection(CodeSize, Align, i, ""); + data[i] = + MemMgr->allocateDataSection(DataSize, Align, i + 10000, "", isReadOnly); + minAddr = std::min({minAddr, code[i], data[i]}); + maxAddr = std::max({maxAddr, code[i], data[i]}); + + EXPECT_NE((uint8_t *)nullptr, code[i]); + EXPECT_NE((uint8_t *)nullptr, data[i]); + + for (unsigned j = 0; j < CodeSize; j++) { + code[i][j] = 1 + (i % 254); + } + + for (unsigned j = 0; j < DataSize; j++) { + data[i][j] = 2 + (i % 254); + } + + uintptr_t CodeAlign = Align ? (uintptr_t)code[i] % Align : 0; + uintptr_t DataAlign = Align ? (uintptr_t)data[i] % Align : 0; + + EXPECT_EQ((uintptr_t)0, CodeAlign); + EXPECT_EQ((uintptr_t)0, DataAlign); + } + + EXPECT_LT(maxAddr - minAddr, 1024 * 1024 * 1024); + + for (unsigned i = 0; i < 10000; ++i) { + uintptr_t CodeSize = i % 16 + 1; + uintptr_t DataSize = i % 8 + 1; + + for (unsigned j = 0; j < CodeSize; j++) { + uint8_t ExpectedCode = 1 + (i % 254); + EXPECT_EQ(ExpectedCode, code[i][j]); + } + + for (unsigned j = 0; j < DataSize; j++) { + uint8_t ExpectedData = 2 + (i % 254); + EXPECT_EQ(ExpectedData, data[i][j]); + } + } +} + } // Namespace