diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index c09398f984477..e8c3e3414dce0 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -233,6 +233,21 @@ enum EdgeKind_aarch64 : Edge::Kind { /// out-of-range error will be returned. PageOffset12, + /// The 15-bit offset of the GOT entry from the GOT table. + /// + /// Used for load/store instructions addressing a GOT entry. + /// + /// Fixup expression: + /// + /// Fixup <- ((Target + Addend - Page(GOT))) & 0x7fff) >> 3 : uint12 + /// + /// Errors: + /// - The result of the unshifted part of the fixup expression must be + /// aligned otherwise an alignment error will be returned. + /// - The result of the fixup expression must fit into a uint12 otherwise an + /// out-of-range error will be returned. + GotPageOffset15, + /// A GOT entry getter/constructor, transformed to Page21 pointing at the GOT /// entry for the original target. /// @@ -273,6 +288,23 @@ enum EdgeKind_aarch64 : Edge::Kind { /// RequestGOTAndTransformToPageOffset12, + /// A GOT entry getter/constructor, transformed to Pageoffset15 pointing at + /// the GOT entry for the original target. + /// + /// Indicates that this edge should be transformed into a GotPageOffset15 + /// targeting the GOT entry for the edge's current target, maintaining the + /// same addend. A GOT entry for the target should be created if one does not + /// already exist. + /// + /// Fixup expression: + /// NONE + /// + /// Errors: + /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup + /// phase will result in an assert/unreachable during the fixup phase. + /// + RequestGOTAndTransformToPageOffset15, + /// A GOT entry getter/constructor, transformed to Delta32 pointing at the GOT /// entry for the original target. /// @@ -430,7 +462,8 @@ inline unsigned getMoveWide16Shift(uint32_t Instr) { } /// Apply fixup expression for edge to block content. -inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { +inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E, + const Symbol *GOTSymbol) { using namespace support; char *BlockWorkingMem = B.getAlreadyMutableContent().data(); @@ -603,6 +636,24 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { *(ulittle32_t *)FixupPtr = FixedInstr; break; } + case GotPageOffset15: { + assert(GOTSymbol && "No GOT section symbol"); + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()).getValue() - + (GOTSymbol->getAddress().getValue() & ~static_cast(4096 - 1)); + if (TargetOffset > 0x7fff) + return make_error("PAGEOFF15 target is out of range"); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + const unsigned ImmShift = 3; + if (TargetOffset & ((1 << ImmShift) - 1)) + return make_error("PAGEOFF15 target is not aligned"); + + uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } default: return make_error( "In graph " + G.getName() + ", section " + B.getSection().getName() + @@ -701,6 +752,15 @@ class GOTTableManager : public TableManager { "RawInstr isn't a 64-bit LDR immediate"); break; } + case aarch64::RequestGOTAndTransformToPageOffset15: { + KindToSet = aarch64::GotPageOffset15; + uint32_t RawInstr = *(const support::ulittle32_t *)FixupPtr; + (void)RawInstr; + assert(E.getAddend() == 0 && "GOTPageOffset15 with non-zero addend"); + assert((RawInstr & 0xfffffc00) == 0xf9400000 && + "RawInstr isn't a 64-bit LDR immediate"); + break; + } case aarch64::RequestGOTAndTransformToDelta32: { KindToSet = aarch64::Delta32; break; diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 9ce8aecb717ca..86e5054890799 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -29,6 +29,8 @@ using namespace llvm::jitlink; namespace { +constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_"; + class ELFJITLinker_aarch64 : public JITLinker { friend class JITLinker; @@ -36,11 +38,83 @@ class ELFJITLinker_aarch64 : public JITLinker { ELFJITLinker_aarch64(std::unique_ptr Ctx, std::unique_ptr G, PassConfiguration PassConfig) - : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { + if (shouldAddDefaultTargetPasses(getGraph().getTargetTriple())) + getPassConfig().PostAllocationPasses.push_back( + [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); + } private: + Symbol *GOTSymbol = nullptr; + Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { - return aarch64::applyFixup(G, B, E); + return aarch64::applyFixup(G, B, E, GOTSymbol); + } + + Error getOrCreateGOTSymbol(LinkGraph &G) { + auto DefineExternalGOTSymbolIfPresent = + createDefineExternalSectionStartAndEndSymbolsPass( + [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc { + if (Sym.getName() == ELFGOTSymbolName) + if (auto *GOTSection = G.findSectionByName( + aarch64::GOTTableManager::getSectionName())) { + GOTSymbol = &Sym; + return {*GOTSection, true}; + } + return {}; + }); + + // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an + // external. + if (auto Err = DefineExternalGOTSymbolIfPresent(G)) + return Err; + + // If we succeeded then we're done. + if (GOTSymbol) + return Error::success(); + + // Otherwise look for a GOT section: If it already has a start symbol we'll + // record it, otherwise we'll create our own. + // If there's a GOT section but we didn't find an external GOT symbol... + if (auto *GOTSection = + G.findSectionByName(aarch64::GOTTableManager::getSectionName())) { + + // Check for an existing defined symbol. + for (auto *Sym : GOTSection->symbols()) + if (Sym->getName() == ELFGOTSymbolName) { + GOTSymbol = Sym; + return Error::success(); + } + + // If there's no defined symbol then create one. + SectionRange SR(*GOTSection); + if (SR.empty()) + GOTSymbol = + &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0, + Linkage::Strong, Scope::Local, true); + else + GOTSymbol = + &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0, + Linkage::Strong, Scope::Local, false, true); + } + + // If we still haven't found a GOT symbol then double check the externals. + // We may have a GOT-relative reference but no GOT section, in which case + // we just need to point the GOT symbol at some address in this graph. + if (!GOTSymbol) { + for (auto *Sym : G.external_symbols()) { + if (Sym->getName() == ELFGOTSymbolName) { + auto Blocks = G.blocks(); + if (!Blocks.empty()) { + G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress()); + GOTSymbol = Sym; + break; + } + } + } + } + + return Error::success(); } }; @@ -70,6 +144,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { ELFPrel64, ELFAdrGOTPage21, ELFLd64GOTLo12, + ELFLd64GOTPAGELo15, ELFTLSDescAdrPage21, ELFTLSDescAddLo12, ELFTLSDescLd64Lo12, @@ -125,6 +200,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return ELFAdrGOTPage21; case ELF::R_AARCH64_LD64_GOT_LO12_NC: return ELFLd64GOTLo12; + case ELF::R_AARCH64_LD64_GOTPAGE_LO15: + return ELFLd64GOTPAGELo15; case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: return ELFTLSDescAdrPage21; case ELF::R_AARCH64_TLSDESC_ADD_LO12: @@ -362,6 +439,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { Kind = aarch64::RequestGOTAndTransformToPageOffset12; break; } + case ELFLd64GOTPAGELo15: { + Kind = aarch64::RequestGOTAndTransformToPageOffset15; + break; + } case ELFTLSDescAdrPage21: { Kind = aarch64::RequestTLSDescEntryAndTransformToPage21; break; @@ -427,6 +508,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return "ELFAdrGOTPage21"; case ELFLd64GOTLo12: return "ELFLd64GOTLo12"; + case ELFLd64GOTPAGELo15: + return "ELFLd64GOTPAGELo15"; case ELFTLSDescAdrPage21: return "ELFTLSDescAdrPage21"; case ELFTLSDescAddLo12: diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 8733306bab6b5..125c6373f82d9 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -552,7 +552,7 @@ class MachOJITLinker_arm64 : public JITLinker { private: Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { - return aarch64::applyFixup(G, B, E); + return aarch64::applyFixup(G, B, E, nullptr); } uint64_t NullValue = 0; diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index cc58255a338df..4d3c19574a23c 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -57,10 +57,14 @@ const char *getEdgeKindName(Edge::Kind R) { return "Page21"; case PageOffset12: return "PageOffset12"; + case GotPageOffset15: + return "GotPageOffset15"; case RequestGOTAndTransformToPage21: return "RequestGOTAndTransformToPage21"; case RequestGOTAndTransformToPageOffset12: return "RequestGOTAndTransformToPageOffset12"; + case RequestGOTAndTransformToPageOffset15: + return "RequestGOTAndTransformToPageOffset15"; case RequestGOTAndTransformToDelta32: return "RequestGOTAndTransformToDelta32"; case RequestTLVPAndTransformToPage21: diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s index 75e367bee80a9..aef96f0e31ddb 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s @@ -264,6 +264,20 @@ test_ld64_gotlo12_external: ldr x0, [x0, :got_lo12:external_data] .size test_ld64_gotlo12_external, .-test_ld64_gotlo12_external +# Check R_AARCH64_LD64_GOTPAGE_LO15 handling with a reference to an external +# symbol. Validate the reference to the GOT entry. +# For the LDR :gotpage_lo15: instruction we have the 15-bit offset of the GOT +# entry from the page containing the GOT. +# jitlink-check: decode_operand(test_ld64_gotpagelo15_external, 2) = \ +# jitlink-check: (got_addr(elf_reloc.o, external_data) - \ +# jitlink-check: (section_addr(elf_reloc.o, $__GOT) & 0xfffffffffffff000)) \ +# jitlink-check: [15:3] + .globl test_ld64_gotpagelo15_external + .p2align 2 +test_ld64_gotpagelo15_external: + ldr x0, [x0, :gotpage_lo15:external_data] + .size test_ld64_gotpagelo15_external, .-test_ld64_gotpagelo15_external + # Check R_AARCH64_TSTBR14 for tbz # # jitlink-check: decode_operand(test_tstbr14_tbz, 2) = \