[JITLink][AArch64] Add LD64_GOTPAGE_LO15 rel support

llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h

@@ -233,6 +233,21 @@ enum EdgeKind_aarch64 : Edge::Kind {
   ///     out-of-range error will be returned.
   PageOffset12,
 
+  /// The 15-bit offset of the GOT entry from the GOT table.
+  ///
+  /// Used for load/store instructions addressing a GOT entry.
+  ///
+  /// Fixup expression:
+  ///
+  ///   Fixup <- ((Target + Addend - Page(GOT))) & 0x7fff) >> 3 : uint12
+  ///
+  /// Errors:
+  ///   - The result of the unshifted part of the fixup expression must be
+  ///     aligned otherwise an alignment error will be returned.
+  ///   - The result of the fixup expression must fit into a uint12 otherwise an
+  ///     out-of-range error will be returned.
+  GotPageOffset15,
+
   /// A GOT entry getter/constructor, transformed to Page21 pointing at the GOT
   /// entry for the original target.
   ///
@@ -273,6 +288,23 @@ enum EdgeKind_aarch64 : Edge::Kind {
   ///
   RequestGOTAndTransformToPageOffset12,
 
+  /// A GOT entry getter/constructor, transformed to Pageoffset15 pointing at
+  /// the GOT entry for the original target.
+  ///
+  /// Indicates that this edge should be transformed into a GotPageOffset15
+  /// targeting the GOT entry for the edge's current target, maintaining the
+  /// same addend. A GOT entry for the target should be created if one does not
+  /// already exist.
+  ///
+  /// Fixup expression:
+  ///   NONE
+  ///
+  /// Errors:
+  ///   - *ASSERTION* Failure to handle edges of this kind prior to the fixup
+  ///     phase will result in an assert/unreachable during the fixup phase.
+  ///
+  RequestGOTAndTransformToPageOffset15,
+
   /// A GOT entry getter/constructor, transformed to Delta32 pointing at the GOT
   /// entry for the original target.
   ///
@@ -430,7 +462,8 @@ inline unsigned getMoveWide16Shift(uint32_t Instr) {
 }
 
 /// Apply fixup expression for edge to block content.
-inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
+inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
+                        const Symbol *GOTSymbol) {
   using namespace support;
 
   char *BlockWorkingMem = B.getAlreadyMutableContent().data();
@@ -603,6 +636,24 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
     *(ulittle32_t *)FixupPtr = FixedInstr;
     break;
   }
+  case GotPageOffset15: {
+    assert(GOTSymbol && "No GOT section symbol");
+    uint64_t TargetOffset =
+        (E.getTarget().getAddress() + E.getAddend()).getValue() -
+        (GOTSymbol->getAddress().getValue() & ~static_cast<uint64_t>(4096 - 1));
+    if (TargetOffset > 0x7fff)
+      return make_error<JITLinkError>("PAGEOFF15 target is out of range");
+
+    uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
+    const unsigned ImmShift = 3;
+    if (TargetOffset & ((1 << ImmShift) - 1))
+      return make_error<JITLinkError>("PAGEOFF15 target is not aligned");
+
+    uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10;
+    uint32_t FixedInstr = RawInstr | EncodedImm;
+    *(ulittle32_t *)FixupPtr = FixedInstr;
+    break;
+  }
   default:
     return make_error<JITLinkError>(
         "In graph " + G.getName() + ", section " + B.getSection().getName() +
@@ -701,6 +752,15 @@ class GOTTableManager : public TableManager<GOTTableManager> {
              "RawInstr isn't a 64-bit LDR immediate");
       break;
     }
+    case aarch64::RequestGOTAndTransformToPageOffset15: {
+      KindToSet = aarch64::GotPageOffset15;
+      uint32_t RawInstr = *(const support::ulittle32_t *)FixupPtr;
+      (void)RawInstr;
+      assert(E.getAddend() == 0 && "GOTPageOffset15 with non-zero addend");
+      assert((RawInstr & 0xfffffc00) == 0xf9400000 &&
+             "RawInstr isn't a 64-bit LDR immediate");
+      break;
+    }
     case aarch64::RequestGOTAndTransformToDelta32: {
       KindToSet = aarch64::Delta32;
       break;

llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp

@@ -29,18 +29,92 @@ using namespace llvm::jitlink;
 
 namespace {
 
+constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
+
 class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> {
   friend class JITLinker<ELFJITLinker_aarch64>;
 
 public:
   ELFJITLinker_aarch64(std::unique_ptr<JITLinkContext> Ctx,
                        std::unique_ptr<LinkGraph> G,
                        PassConfiguration PassConfig)
-      : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {}
+      : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
+    if (shouldAddDefaultTargetPasses(getGraph().getTargetTriple()))
+      getPassConfig().PostAllocationPasses.push_back(
+          [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
+  }
 
 private:
+  Symbol *GOTSymbol = nullptr;
+
   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
-    return aarch64::applyFixup(G, B, E);
+    return aarch64::applyFixup(G, B, E, GOTSymbol);
+  }
+
+  Error getOrCreateGOTSymbol(LinkGraph &G) {
+    auto DefineExternalGOTSymbolIfPresent =
+        createDefineExternalSectionStartAndEndSymbolsPass(
+            [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
+              if (Sym.getName() == ELFGOTSymbolName)
+                if (auto *GOTSection = G.findSectionByName(
+                        aarch64::GOTTableManager::getSectionName())) {
+                  GOTSymbol = &Sym;
+                  return {*GOTSection, true};
+                }
+              return {};
+            });
+
+    // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
+    // external.
+    if (auto Err = DefineExternalGOTSymbolIfPresent(G))
+      return Err;
+
+    // If we succeeded then we're done.
+    if (GOTSymbol)
+      return Error::success();
+
+    // Otherwise look for a GOT section: If it already has a start symbol we'll
+    // record it, otherwise we'll create our own.
+    // If there's a GOT section but we didn't find an external GOT symbol...
+    if (auto *GOTSection =
+            G.findSectionByName(aarch64::GOTTableManager::getSectionName())) {
+
+      // Check for an existing defined symbol.
+      for (auto *Sym : GOTSection->symbols())
+        if (Sym->getName() == ELFGOTSymbolName) {
+          GOTSymbol = Sym;
+          return Error::success();
+        }
+
+      // If there's no defined symbol then create one.
+      SectionRange SR(*GOTSection);
+      if (SR.empty())
+        GOTSymbol =
+            &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0,
+                                 Linkage::Strong, Scope::Local, true);
+      else
+        GOTSymbol =
+            &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
+                                Linkage::Strong, Scope::Local, false, true);
+    }
+
+    // If we still haven't found a GOT symbol then double check the externals.
+    // We may have a GOT-relative reference but no GOT section, in which case
+    // we just need to point the GOT symbol at some address in this graph.
+    if (!GOTSymbol) {
+      for (auto *Sym : G.external_symbols()) {
+        if (Sym->getName() == ELFGOTSymbolName) {
+          auto Blocks = G.blocks();
+          if (!Blocks.empty()) {
+            G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress());
+            GOTSymbol = Sym;
+            break;
+          }
+        }
+      }
+    }
+
+    return Error::success();
   }
 };
 
@@ -70,6 +144,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
     ELFPrel64,
     ELFAdrGOTPage21,
     ELFLd64GOTLo12,
+    ELFLd64GOTPAGELo15,
     ELFTLSDescAdrPage21,
     ELFTLSDescAddLo12,
     ELFTLSDescLd64Lo12,
@@ -125,6 +200,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
       return ELFAdrGOTPage21;
     case ELF::R_AARCH64_LD64_GOT_LO12_NC:
       return ELFLd64GOTLo12;
+    case ELF::R_AARCH64_LD64_GOTPAGE_LO15:
+      return ELFLd64GOTPAGELo15;
     case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
       return ELFTLSDescAdrPage21;
     case ELF::R_AARCH64_TLSDESC_ADD_LO12:
@@ -362,6 +439,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
       Kind = aarch64::RequestGOTAndTransformToPageOffset12;
       break;
     }
+    case ELFLd64GOTPAGELo15: {
+      Kind = aarch64::RequestGOTAndTransformToPageOffset15;
+      break;
+    }
     case ELFTLSDescAdrPage21: {
       Kind = aarch64::RequestTLSDescEntryAndTransformToPage21;
       break;
@@ -427,6 +508,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
       return "ELFAdrGOTPage21";
     case ELFLd64GOTLo12:
       return "ELFLd64GOTLo12";
+    case ELFLd64GOTPAGELo15:
+      return "ELFLd64GOTPAGELo15";
     case ELFTLSDescAdrPage21:
       return "ELFTLSDescAdrPage21";
     case ELFTLSDescAddLo12:

llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp

@@ -552,7 +552,7 @@ class MachOJITLinker_arm64 : public JITLinker<MachOJITLinker_arm64> {
 
 private:
   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
-    return aarch64::applyFixup(G, B, E);
+    return aarch64::applyFixup(G, B, E, nullptr);
   }
 
   uint64_t NullValue = 0;

llvm/lib/ExecutionEngine/JITLink/aarch64.cpp

@@ -57,10 +57,14 @@ const char *getEdgeKindName(Edge::Kind R) {
     return "Page21";
   case PageOffset12:
     return "PageOffset12";
+  case GotPageOffset15:
+    return "GotPageOffset15";
   case RequestGOTAndTransformToPage21:
     return "RequestGOTAndTransformToPage21";
   case RequestGOTAndTransformToPageOffset12:
     return "RequestGOTAndTransformToPageOffset12";
+  case RequestGOTAndTransformToPageOffset15:
+    return "RequestGOTAndTransformToPageOffset15";
   case RequestGOTAndTransformToDelta32:
     return "RequestGOTAndTransformToDelta32";
   case RequestTLVPAndTransformToPage21:

llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s

@@ -264,6 +264,20 @@ test_ld64_gotlo12_external:
         ldr   x0, [x0, :got_lo12:external_data]
         .size test_ld64_gotlo12_external, .-test_ld64_gotlo12_external
 
+# Check R_AARCH64_LD64_GOTPAGE_LO15 handling with a reference to an external
+# symbol. Validate the reference to the GOT entry.
+# For the LDR :gotpage_lo15: instruction we have the 15-bit offset of the GOT
+# entry from the page containing the GOT.
+# jitlink-check: decode_operand(test_ld64_gotpagelo15_external, 2) = \
+# jitlink-check:     (got_addr(elf_reloc.o, external_data) - \
+# jitlink-check:     (section_addr(elf_reloc.o, $__GOT) & 0xfffffffffffff000)) \
+# jitlink-check:     [15:3]
+        .globl  test_ld64_gotpagelo15_external
+        .p2align  2
+test_ld64_gotpagelo15_external:
+        ldr   x0, [x0, :gotpage_lo15:external_data]
+        .size test_ld64_gotpagelo15_external, .-test_ld64_gotpagelo15_external
+
 # Check R_AARCH64_TSTBR14 for tbz
 #
 # jitlink-check: decode_operand(test_tstbr14_tbz, 2) = \