diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst index 42f56348885b4..b05394aeee003 100644 --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -60,6 +60,17 @@ The address of a global value. %0(p0) = G_GLOBAL_VALUE @var_local +G_PTRAUTH_GLOBAL_VALUE +^^^^^^^^^^^^^^^^^^^^^^ + +The signed address of a global value. Operands: address to be signed (pointer), +key (32-bit imm), address for address discrimination (zero if not needed) and +an extra discriminator (64-bit imm). + +.. code-block:: none + + %0:_(p0) = G_PTRAUTH_GLOBAL_VALUE %1:_(p0), s32, %2:_(p0), s64 + G_BLOCK_ADDR ^^^^^^^^^^^^ diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index a95ed3109feb8..954511eb731fa 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -886,6 +886,13 @@ class MachineIRBuilder { MachineInstrBuilder buildFConstant(const DstOp &Res, double Val); MachineInstrBuilder buildFConstant(const DstOp &Res, const APFloat &Val); + /// Build and insert G_PTRAUTH_GLOBAL_VALUE + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildConstantPtrAuth(const DstOp &Res, + const ConstantPtrAuth *CPA, + Register Addr, Register AddrDisc); + /// Build and insert \p Res = COPY Op /// /// Register-to-register COPY sets \p Res to \p Op. diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index 6bb89fb58a296..88e3339e2453f 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -83,6 +83,12 @@ enum NodeType { ExternalSymbol, BlockAddress, + /// A ptrauth constant. + /// ptr, key, addr-disc, disc + /// Note that the addr-disc can be a non-constant value, to allow representing + /// a constant global address signed using address-diversification, in code. + PtrAuthGlobalAddress, + /// The address of the GOT GLOBAL_OFFSET_TABLE, diff --git a/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h b/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h index f8a328f13eded..64d841d86c7c4 100644 --- a/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h +++ b/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h @@ -61,10 +61,20 @@ class MachineModuleInfoMachO : public MachineModuleInfoImpl { /// MachineModuleInfoELF - This is a MachineModuleInfoImpl implementation /// for ELF targets. class MachineModuleInfoELF : public MachineModuleInfoImpl { +public: + struct AuthStubInfo { + const MCExpr *AuthPtrRef; + }; + +private: /// GVStubs - These stubs are used to materialize global addresses in PIC /// mode. DenseMap GVStubs; + /// AuthPtrStubs - These stubs are used to materialize signed addresses for + /// extern_weak symbols. + DenseMap AuthPtrStubs; + virtual void anchor(); // Out of line virtual method. public: @@ -75,9 +85,19 @@ class MachineModuleInfoELF : public MachineModuleInfoImpl { return GVStubs[Sym]; } + AuthStubInfo &getAuthPtrStubEntry(MCSymbol *Sym) { + assert(Sym && "Key cannot be null"); + return AuthPtrStubs[Sym]; + } + /// Accessor methods to return the set of stubs in sorted order. SymbolListTy GetGVStubList() { return getSortedStubs(GVStubs); } + + using AuthStubPairTy = std::pair; + typedef std::vector AuthStubListTy; + + AuthStubListTy getAuthGVStubList(); }; /// MachineModuleInfoCOFF - This is a MachineModuleInfoImpl implementation diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index df4b264af72a8..e7f40e87ed24a 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -294,6 +294,9 @@ HANDLE_TARGET_OPCODE(G_FRAME_INDEX) /// Generic reference to global value. HANDLE_TARGET_OPCODE(G_GLOBAL_VALUE) +/// Generic ptrauth-signed reference to global value. +HANDLE_TARGET_OPCODE(G_PTRAUTH_GLOBAL_VALUE) + /// Generic instruction to materialize the address of an object in the constant /// pool. HANDLE_TARGET_OPCODE(G_CONSTANT_POOL) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 4abffe6476c85..e1710ff2d8abf 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -110,6 +110,12 @@ def G_GLOBAL_VALUE : GenericInstruction { let hasSideEffects = false; } +def G_PTRAUTH_GLOBAL_VALUE : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins unknown:$addr, i32imm:$key, type1:$addrdisc, i64imm:$disc); + let hasSideEffects = 0; +} + def G_CONSTANT_POOL : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins unknown:$src); diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index ddf6d3c20f022..c9986dfede1a1 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -3494,7 +3494,11 @@ bool IRTranslator::translate(const Constant &C, Register Reg) { EntryBuilder->buildConstant(Reg, 0); else if (auto GV = dyn_cast(&C)) EntryBuilder->buildGlobalValue(Reg, GV); - else if (auto CAZ = dyn_cast(&C)) { + else if (auto CPA = dyn_cast(&C)) { + Register Addr = getOrCreateVReg(*CPA->getPointer()); + Register AddrDisc = getOrCreateVReg(*CPA->getAddrDiscriminator()); + EntryBuilder->buildConstantPtrAuth(Reg, CPA, Addr, AddrDisc); + } else if (auto CAZ = dyn_cast(&C)) { if (!isa(CAZ->getType())) return false; // Return the scalar if it is a <1 x Ty> vector. diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 37aa4e0aed7e8..06a6c1f93ef1f 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -397,6 +397,19 @@ MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, return buildFConstant(Res, *CFP); } +MachineInstrBuilder +MachineIRBuilder::buildConstantPtrAuth(const DstOp &Res, + const ConstantPtrAuth *CPA, + Register Addr, Register AddrDisc) { + auto MIB = buildInstr(TargetOpcode::G_PTRAUTH_GLOBAL_VALUE); + Res.addDefToMIB(*getMRI(), MIB); + MIB.addUse(Addr); + MIB.addImm(CPA->getKey()->getZExtValue()); + MIB.addUse(AddrDisc); + MIB.addImm(CPA->getDiscriminator()->getZExtValue()); + return MIB; +} + MachineInstrBuilder MachineIRBuilder::buildBrCond(const SrcOp &Tst, MachineBasicBlock &Dest) { assert(Tst.getLLTTy(*getMRI()).isScalar() && "invalid operand type"); diff --git a/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp b/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp index 9c3b31935f6d6..f114f1ecc0bae 100644 --- a/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp +++ b/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp @@ -13,6 +13,7 @@ #include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/MC/MCSymbol.h" using namespace llvm; @@ -41,3 +42,25 @@ MachineModuleInfoImpl::SymbolListTy MachineModuleInfoImpl::getSortedStubs( Map.clear(); return List; } + +template +static typename MachineModuleInfoTarget::AuthStubListTy getAuthGVStubListHelper( + DenseMap + &AuthPtrStubs) { + typename MachineModuleInfoTarget::AuthStubListTy List(AuthPtrStubs.begin(), + AuthPtrStubs.end()); + + if (!List.empty()) + llvm::sort(List.begin(), List.end(), + [](const typename MachineModuleInfoTarget::AuthStubPairTy &LHS, + const typename MachineModuleInfoTarget::AuthStubPairTy &RHS) { + return LHS.first->getName() < RHS.first->getName(); + }); + + AuthPtrStubs.clear(); + return List; +} + +MachineModuleInfoELF::AuthStubListTy MachineModuleInfoELF::getAuthGVStubList() { + return getAuthGVStubListHelper(AuthPtrStubs); +} diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 9ea238c61ed91..0c8a0f2b24a1e 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -2066,6 +2066,12 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { report("Dst operand 0 must be a pointer", MI); break; } + case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE: { + const MachineOperand &AddrOp = MI->getOperand(1); + if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer()) + report("addr operand must be a pointer", &AddrOp, 1); + break; + } default: break; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 296b06187ec0f..b2f9a9e1d58ef 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1802,6 +1802,13 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { if (const GlobalValue *GV = dyn_cast(C)) return DAG.getGlobalAddress(GV, getCurSDLoc(), VT); + if (const ConstantPtrAuth *CPA = dyn_cast(C)) { + return DAG.getNode(ISD::PtrAuthGlobalAddress, getCurSDLoc(), VT, + getValue(CPA->getPointer()), getValue(CPA->getKey()), + getValue(CPA->getAddrDiscriminator()), + getValue(CPA->getDiscriminator())); + } + if (isa(C)) { unsigned AS = V->getType()->getPointerAddressSpace(); return DAG.getConstant(0, getCurSDLoc(), diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index a7555d6d31f26..c1d2c095b103c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -75,6 +75,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { } return "<>"; + // clang-format off #ifndef NDEBUG case ISD::DELETED_NODE: return "<>"; #endif @@ -126,6 +127,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::ConstantFP: return "ConstantFP"; case ISD::GlobalAddress: return "GlobalAddress"; case ISD::GlobalTLSAddress: return "GlobalTLSAddress"; + case ISD::PtrAuthGlobalAddress: return "PtrAuthGlobalAddress"; case ISD::FrameIndex: return "FrameIndex"; case ISD::JumpTable: return "JumpTable"; case ISD::JUMP_TABLE_DEBUG_INFO: @@ -168,8 +170,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { return "OpaqueTargetConstant"; return "TargetConstant"; - // clang-format off - case ISD::TargetConstantFP: return "TargetConstantFP"; case ISD::TargetGlobalAddress: return "TargetGlobalAddress"; case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress"; diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index da11539eab348..611c3b579ff15 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -133,6 +133,13 @@ class AArch64AsmPrinter : public AsmPrinter { unsigned emitPtrauthDiscriminator(uint16_t Disc, unsigned AddrDisc, unsigned &InstsEmitted); + // Emit the sequence for LOADauthptrstatic + void LowerLOADauthptrstatic(const MachineInstr &MI); + + // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or + // adrp-add followed by PAC sign) + void LowerMOVaddrPAC(const MachineInstr &MI); + /// tblgen'erated driver function for lowering simple MI->MC /// pseudo instructions. bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, @@ -840,6 +847,15 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) { } } +template +static void emitAuthenticatedPointer( + MCStreamer &OutStreamer, MCSymbol *StubLabel, + const typename MachineModuleInfoTarget::AuthStubInfo &StubInfo) { + // sym$auth_ptr$key$disc: + OutStreamer.emitLabel(StubLabel); + OutStreamer.emitValue(StubInfo.AuthPtrRef, /*size=*/8); +} + void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) { emitHwasanMemaccessSymbols(M); @@ -853,6 +869,25 @@ void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) { OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols); } + if (TT.isOSBinFormatELF()) { + // Output authenticated pointers as indirect symbols, if we have any. + MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo(); + + auto Stubs = MMIELF.getAuthGVStubList(); + + if (!Stubs.empty()) { + const TargetLoweringObjectFile &TLOF = getObjFileLowering(); + OutStreamer->switchSection(TLOF.getDataSection()); + emitAlignment(Align(8)); + + for (const auto &Stub : Stubs) + emitAuthenticatedPointer(*OutStreamer, Stub.first, + Stub.second); + + OutStreamer->addBlankLine(); + } + } + // Emit stack and fault map information. FM.serializeToFaultMapSection(); @@ -1623,6 +1658,214 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { CPA.hasAddressDiscriminator(), Ctx); } +void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) { + unsigned DstReg = MI.getOperand(0).getReg(); + const MachineOperand &GAOp = MI.getOperand(1); + const uint64_t KeyC = MI.getOperand(2).getImm(); + assert(KeyC <= AArch64PACKey::LAST && + "key is out of range [0, AArch64PACKey::LAST]"); + const auto Key = (AArch64PACKey::ID)KeyC; + const uint64_t Disc = MI.getOperand(3).getImm(); + assert(isUInt<16>(Disc) && + "constant discriminator is out of range [0, 0xffff]"); + + // Emit instruction sequence like the following: + // ADRP x16, symbol$auth_ptr$key$disc + // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc] + // + // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer + // to symbol. + assert(TM.getTargetTriple().isOSBinFormatELF() && + "LOADauthptrstatic is implemented only for ELF"); + const auto &TLOF = + static_cast(getObjFileLowering()); + + assert(GAOp.getOffset() == 0 && + "non-zero offset for $auth_ptr$ stub slots is not supported"); + const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal()); + MCSymbol *AuthPtrStubSym = + TLOF.getAuthPtrSlotSymbol(TM, &MF->getMMI(), GASym, Key, Disc); + + MachineOperand StubMOHi = + MachineOperand::CreateMCSymbol(AuthPtrStubSym, AArch64II::MO_PAGE); + MachineOperand StubMOLo = MachineOperand::CreateMCSymbol( + AuthPtrStubSym, AArch64II::MO_PAGEOFF | AArch64II::MO_NC); + MCOperand StubMCHi, StubMCLo; + + MCInstLowering.lowerOperand(StubMOHi, StubMCHi); + MCInstLowering.lowerOperand(StubMOLo, StubMCLo); + + EmitToStreamer( + *OutStreamer, + MCInstBuilder(AArch64::ADRP).addReg(DstReg).addOperand(StubMCHi)); + + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRXui) + .addReg(DstReg) + .addReg(DstReg) + .addOperand(StubMCLo)); +} + +void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) { + unsigned InstsEmitted = 0; + auto EmitAndIncrement = [this, &InstsEmitted](const MCInst &Inst) { + EmitToStreamer(*OutStreamer, Inst); + ++InstsEmitted; + }; + + const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC; + MachineOperand GAOp = MI.getOperand(0); + const uint64_t KeyC = MI.getOperand(1).getImm(); + assert(KeyC <= AArch64PACKey::LAST && + "key is out of range [0, AArch64PACKey::LAST]"); + const auto Key = (AArch64PACKey::ID)KeyC; + const unsigned AddrDisc = MI.getOperand(2).getReg(); + const uint64_t Disc = MI.getOperand(3).getImm(); + assert(isUInt<16>(Disc) && + "constant discriminator is out of range [0, 0xffff]"); + + const int64_t Offset = GAOp.getOffset(); + GAOp.setOffset(0); + + // Emit: + // target materialization: + // - via GOT: + // adrp x16, :got:target + // ldr x16, [x16, :got_lo12:target] + // add offset to x16 if offset != 0 + // + // - direct: + // adrp x16, target + // add x16, x16, :lo12:target + // add offset to x16 if offset != 0 + // + // add offset to x16: + // - abs(offset) fits 24 bits: + // add/sub x16, x16, #[, #lsl 12] (up to 2 instructions) + // - abs(offset) does not fit 24 bits: + // - offset < 0: + // movn+movk sequence filling x17 register with the offset (up to 4 + // instructions) + // add x16, x16, x17 + // - offset > 0: + // movz+movk sequence filling x17 register with the offset (up to 4 + // instructions) + // add x16, x16, x17 + // + // signing: + // - 0 discriminator: + // paciza x16 + // - Non-0 discriminator, no address discriminator: + // mov x17, #Disc + // pacia x16, x17 + // - address discriminator (with potentially folded immediate discriminator): + // pacia x16, xAddrDisc + + MachineOperand GAMOHi(GAOp), GAMOLo(GAOp); + MCOperand GAMCHi, GAMCLo; + + GAMOHi.setTargetFlags(AArch64II::MO_PAGE); + GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC); + if (IsGOTLoad) { + GAMOHi.addTargetFlag(AArch64II::MO_GOT); + GAMOLo.addTargetFlag(AArch64II::MO_GOT); + } + + MCInstLowering.lowerOperand(GAMOHi, GAMCHi); + MCInstLowering.lowerOperand(GAMOLo, GAMCLo); + + EmitAndIncrement( + MCInstBuilder(AArch64::ADRP).addReg(AArch64::X16).addOperand(GAMCHi)); + + if (IsGOTLoad) { + EmitAndIncrement(MCInstBuilder(AArch64::LDRXui) + .addReg(AArch64::X16) + .addReg(AArch64::X16) + .addOperand(GAMCLo)); + } else { + EmitAndIncrement(MCInstBuilder(AArch64::ADDXri) + .addReg(AArch64::X16) + .addReg(AArch64::X16) + .addOperand(GAMCLo) + .addImm(0)); + } + + if (Offset != 0) { + const uint64_t AbsOffset = (Offset > 0 ? Offset : -((uint64_t)Offset)); + const bool IsNeg = Offset < 0; + if (isUInt<24>(AbsOffset)) { + for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos); + BitPos += 12) { + EmitAndIncrement( + MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri) + .addReg(AArch64::X16) + .addReg(AArch64::X16) + .addImm((AbsOffset >> BitPos) & 0xfff) + .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, BitPos))); + } + } else { + constexpr uint64_t Mask16 = 0xffff; + const uint64_t UOffset = Offset; + EmitAndIncrement(MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi) + .addReg(AArch64::X17) + .addImm((IsNeg ? ~UOffset : UOffset) & Mask16) + .addImm(/*shift=*/0)); + auto NeedMovk = [Mask16, IsNeg, UOffset](int BitPos) -> bool { + assert(BitPos == 16 || BitPos == 32 || BitPos == 48); + uint64_t Shifted = UOffset >> BitPos; + if (!IsNeg) + return Shifted != 0; + for (int I = 0; I != 64 - BitPos; I += 16) + if (((Shifted >> I) & Mask16) != Mask16) + return true; + return false; + }; + for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16) { + EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi) + .addReg(AArch64::X17) + .addReg(AArch64::X17) + .addImm((UOffset >> BitPos) & Mask16) + .addImm(/*shift=*/BitPos)); + } + EmitAndIncrement(MCInstBuilder(AArch64::ADDXrs) + .addReg(AArch64::X16) + .addReg(AArch64::X16) + .addReg(AArch64::X17) + .addImm(/*shift=*/0)); + } + } + + unsigned DiscReg = AddrDisc; + if (Disc != 0) { + if (AddrDisc != AArch64::XZR) { + EmitAndIncrement(MCInstBuilder(AArch64::ORRXrs) + .addReg(AArch64::X17) + .addReg(AArch64::XZR) + .addReg(AddrDisc) + .addImm(0)); + EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi) + .addReg(AArch64::X17) + .addReg(AArch64::X17) + .addImm(Disc) + .addImm(/*shift=*/48)); + } else { + EmitAndIncrement(MCInstBuilder(AArch64::MOVZXi) + .addReg(AArch64::X17) + .addImm(Disc) + .addImm(/*shift=*/0)); + } + DiscReg = AArch64::X17; + } + + auto MIB = MCInstBuilder(getPACOpcodeForKey(Key, DiscReg == AArch64::XZR)) + .addReg(AArch64::X16) + .addReg(AArch64::X16); + if (DiscReg != AArch64::XZR) + MIB.addReg(DiscReg); + EmitAndIncrement(MIB); + + assert(STI->getInstrInfo()->getInstSizeInBytes(MI) >= InstsEmitted * 4); +} + // Simple pseudo-instructions have their lowering (with expansion to real // instructions) auto-generated. #include "AArch64GenMCPseudoLowering.inc" @@ -1758,6 +2001,15 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { return; } + case AArch64::LOADauthptrstatic: + LowerLOADauthptrstatic(*MI); + return; + + case AArch64::LOADgotPAC: + case AArch64::MOVaddrPAC: + LowerMOVaddrPAC(*MI); + return; + case AArch64::BLRA: emitPtrauthBranch(MI); return; diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 8b1107f9e08af..8562a627e5fbe 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -511,6 +511,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::JumpTable, MVT::i64, Custom); setOperationAction(ISD::SETCCCARRY, MVT::i64, Custom); + setOperationAction(ISD::PtrAuthGlobalAddress, MVT::i64, Custom); + setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); @@ -6660,6 +6662,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op, return LowerGlobalAddress(Op, DAG); case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); + case ISD::PtrAuthGlobalAddress: + return LowerPtrAuthGlobalAddress(Op, DAG); case ISD::SETCC: case ISD::STRICT_FSETCC: case ISD::STRICT_FSETCCS: @@ -9510,6 +9514,134 @@ SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op, llvm_unreachable("Unexpected platform trying to use TLS"); } +//===----------------------------------------------------------------------===// +// PtrAuthGlobalAddress lowering +// +// We have 3 lowering alternatives to choose from: +// - MOVaddrPAC: similar to MOVaddr, with added PAC. +// If the GV doesn't need a GOT load (i.e., is locally defined) +// materialize the pointer using adrp+add+pac. See LowerMOVaddrPAC. +// +// - LOADgotPAC: similar to LOADgot, with added PAC. +// If the GV needs a GOT load, materialize the pointer using the usual +// GOT adrp+ldr, +pac. Pointers in GOT are assumed to be not signed, the GOT +// section is assumed to be read-only (for example, via relro mechanism). See +// LowerMOVaddrPAC. +// +// - LOADauthptrstatic: similar to LOADgot, but use a +// special stub slot instead of a GOT slot. +// Load a signed pointer for symbol 'sym' from a stub slot named +// 'sym$auth_ptr$key$disc' filled by dynamic linker during relocation +// resolving. This usually lowers to adrp+ldr, but also emits an entry into +// .data with an +// @AUTH relocation. See LowerLOADauthptrstatic. +// +// All 3 are pseudos that are expand late to longer sequences: this lets us +// provide integrity guarantees on the to-be-signed intermediate values. +// +// LOADauthptrstatic is undesirable because it requires a large section filled +// with often similarly-signed pointers, making it a good harvesting target. +// Thus, it's only used for ptrauth references to extern_weak to avoid null +// checks. + +SDValue AArch64TargetLowering::LowerPtrAuthGlobalAddressStatically( + SDValue TGA, SDLoc DL, EVT VT, AArch64PACKey::ID KeyC, + SDValue Discriminator, SDValue AddrDiscriminator, SelectionDAG &DAG) const { + const auto *TGN = cast(TGA.getNode()); + const GlobalValue *GV = TGN->getGlobal(); + assert(GV->hasExternalWeakLinkage()); + + // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the + // offset alone as a pointer if the symbol wasn't available, which would + // probably break null checks in users. Ptrauth complicates things further: + // error out. + if (TGN->getOffset() != 0) + report_fatal_error( + "unsupported non-zero offset in weak ptrauth global reference"); + + if (!isNullConstant(AddrDiscriminator)) + report_fatal_error("unsupported weak addr-div ptrauth global"); + + SDValue Key = DAG.getTargetConstant(KeyC, DL, MVT::i32); + return SDValue(DAG.getMachineNode(AArch64::LOADauthptrstatic, DL, MVT::i64, + {TGA, Key, Discriminator}), + 0); +} + +SDValue +AArch64TargetLowering::LowerPtrAuthGlobalAddress(SDValue Op, + SelectionDAG &DAG) const { + SDValue Ptr = Op.getOperand(0); + uint64_t KeyC = Op.getConstantOperandVal(1); + SDValue AddrDiscriminator = Op.getOperand(2); + uint64_t DiscriminatorC = Op.getConstantOperandVal(3); + EVT VT = Op.getValueType(); + SDLoc DL(Op); + + if (KeyC > AArch64PACKey::LAST) + report_fatal_error("key in ptrauth global out of range [0, " + + Twine((int)AArch64PACKey::LAST) + "]"); + + // Blend only works if the integer discriminator is 16-bit wide. + if (!isUInt<16>(DiscriminatorC)) + report_fatal_error( + "constant discriminator in ptrauth global out of range [0, 0xffff]"); + + // Choosing between 3 lowering alternatives is target-specific. + if (!Subtarget->isTargetELF()) + report_fatal_error("ptrauth global lowering is only implemented for ELF"); + + int64_t PtrOffsetC = 0; + if (Ptr.getOpcode() == ISD::ADD) { + PtrOffsetC = Ptr.getConstantOperandVal(1); + Ptr = Ptr.getOperand(0); + } + const auto *PtrN = cast(Ptr.getNode()); + const GlobalValue *PtrGV = PtrN->getGlobal(); + + // Classify the reference to determine whether it needs a GOT load. + const unsigned OpFlags = + Subtarget->ClassifyGlobalReference(PtrGV, getTargetMachine()); + const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0); + assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) && + "unsupported non-GOT op flags on ptrauth global reference"); + + // Fold any offset into the GV; our pseudos expect it there. + PtrOffsetC += PtrN->getOffset(); + SDValue TPtr = DAG.getTargetGlobalAddress(PtrGV, DL, VT, PtrOffsetC, + /*TargetFlags=*/0); + assert(PtrN->getTargetFlags() == 0 && + "unsupported target flags on ptrauth global"); + + SDValue Key = DAG.getTargetConstant(KeyC, DL, MVT::i32); + SDValue Discriminator = DAG.getTargetConstant(DiscriminatorC, DL, MVT::i64); + SDValue TAddrDiscriminator = !isNullConstant(AddrDiscriminator) + ? AddrDiscriminator + : DAG.getRegister(AArch64::XZR, MVT::i64); + + // No GOT load needed -> MOVaddrPAC + if (!NeedsGOTLoad) { + assert(!PtrGV->hasExternalWeakLinkage() && "extern_weak should use GOT"); + return SDValue( + DAG.getMachineNode(AArch64::MOVaddrPAC, DL, MVT::i64, + {TPtr, Key, TAddrDiscriminator, Discriminator}), + 0); + } + + // GOT load -> LOADgotPAC + // Note that we disallow extern_weak refs to avoid null checks later. + if (!PtrGV->hasExternalWeakLinkage()) + return SDValue( + DAG.getMachineNode(AArch64::LOADgotPAC, DL, MVT::i64, + {TPtr, Key, TAddrDiscriminator, Discriminator}), + 0); + + // extern_weak ref -> LOADauthptrstatic + return LowerPtrAuthGlobalAddressStatically( + TPtr, DL, VT, (AArch64PACKey::ID)KeyC, Discriminator, AddrDiscriminator, + DAG); +} + // Looks through \param Val to determine the bit that can be used to // check the sign of the value. It returns the unextended value and // the sign bit position. diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index 986f1b67ee513..cb00d0b713d6b 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -1128,6 +1128,12 @@ class AArch64TargetLowering : public TargetLowering { SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL, SelectionDAG &DAG) const; SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerPtrAuthGlobalAddress(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerPtrAuthGlobalAddressStatically(SDValue TGA, SDLoc DL, EVT VT, + AArch64PACKey::ID Key, + SDValue Discriminator, + SDValue AddrDiscriminator, + SelectionDAG &DAG) const; SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const; SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const; diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index f3aac3b46d173..1e06d5fdc7562 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -1765,6 +1765,38 @@ let Predicates = [HasPAuth] in { defm LDRAA : AuthLoad<0, "ldraa", simm10Scaled>; defm LDRAB : AuthLoad<1, "ldrab", simm10Scaled>; + // Materialize a signed global address, with adrp+add and PAC. + def MOVaddrPAC : Pseudo<(outs), + (ins i64imm:$Addr, i32imm:$Key, + GPR64noip:$AddrDisc, i64imm:$Disc), []>, + Sched<[WriteI, ReadI]> { + let isReMaterializable = 1; + let isCodeGenOnly = 1; + let Size = 40; // 12 fixed + 28 variable, for pointer offset, and discriminator + let Defs = [X16,X17]; + } + + // Materialize a signed global address, using a GOT load and PAC. + def LOADgotPAC : Pseudo<(outs), + (ins i64imm:$Addr, i32imm:$Key, + GPR64noip:$AddrDisc, i64imm:$Disc), []>, + Sched<[WriteI, ReadI]> { + let isReMaterializable = 1; + let isCodeGenOnly = 1; + let Size = 40; // 12 fixed + 28 variable, for pointer offset, and discriminator + let Defs = [X16,X17]; + } + + // Load a signed global address from a special $auth_ptr$ stub slot. + def LOADauthptrstatic : Pseudo<(outs GPR64:$dst), + (ins i64imm:$Addr, i32imm:$Key, + i64imm:$Disc), []>, + Sched<[WriteI, ReadI]> { + let isReMaterializable = 1; + let isCodeGenOnly = 1; + let Size = 8; + } + // Size 16: 4 fixed + 8 variable, to compute discriminator. let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Size = 16, Uses = [SP] in { diff --git a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp index e5c6b6d60d068..1c1e87d723ac7 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp @@ -8,7 +8,10 @@ #include "AArch64TargetObjectFile.h" #include "AArch64TargetMachine.h" +#include "MCTargetDesc/AArch64MCExpr.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/IR/Mangler.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" @@ -88,3 +91,37 @@ void AArch64_MachoTargetObjectFile::getNameWithPrefix( // be accessed via at least a linker-private symbol. getMangler().getNameWithPrefix(OutName, GV, /* CannotUsePrivateLabel */ true); } + +template +static MCSymbol *getAuthPtrSlotSymbolHelper( + MCContext &Ctx, const TargetMachine &TM, MachineModuleInfo *MMI, + MachineModuleInfoTarget &TargetMMI, const MCSymbol *RawSym, + AArch64PACKey::ID Key, uint16_t Discriminator) { + const DataLayout &DL = MMI->getModule()->getDataLayout(); + + MCSymbol *StubSym = Ctx.getOrCreateSymbol( + DL.getLinkerPrivateGlobalPrefix() + RawSym->getName() + + Twine("$auth_ptr$") + AArch64PACKeyIDToString(Key) + Twine('$') + + Twine(Discriminator)); + + typename MachineModuleInfoTarget::AuthStubInfo &StubInfo = + TargetMMI.getAuthPtrStubEntry(StubSym); + + if (StubInfo.AuthPtrRef) + return StubSym; + + const MCExpr *Sym = MCSymbolRefExpr::create(RawSym, Ctx); + + StubInfo.AuthPtrRef = + AArch64AuthMCExpr::create(Sym, Discriminator, Key, + /*HasAddressDiversity=*/false, Ctx); + return StubSym; +} + +MCSymbol *AArch64_ELFTargetObjectFile::getAuthPtrSlotSymbol( + const TargetMachine &TM, MachineModuleInfo *MMI, const MCSymbol *RawSym, + AArch64PACKey::ID Key, uint16_t Discriminator) const { + auto &ELFMMI = MMI->getObjFileInfo(); + return getAuthPtrSlotSymbolHelper(getContext(), TM, MMI, ELFMMI, RawSym, Key, + Discriminator); +} diff --git a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h index 7cd56fc9a189d..c5ebf03c39c77 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h +++ b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64TARGETOBJECTFILE_H #define LLVM_LIB_TARGET_AARCH64_AARCH64TARGETOBJECTFILE_H +#include "Utils/AArch64BaseInfo.h" #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" #include "llvm/Target/TargetLoweringObjectFile.h" @@ -29,6 +30,11 @@ class AArch64_ELFTargetObjectFile : public TargetLoweringObjectFileELF { const MCValue &MV, int64_t Offset, MachineModuleInfo *MMI, MCStreamer &Streamer) const override; + + MCSymbol *getAuthPtrSlotSymbol(const TargetMachine &TM, + MachineModuleInfo *MMI, const MCSymbol *RawSym, + AArch64PACKey::ID Key, + uint16_t Discriminator) const; }; /// AArch64_MachoTargetObjectFile - This TLOF implementation is used for Darwin. diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 0357a7206c478..9e0860934f777 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -224,6 +224,8 @@ class AArch64InstructionSelector : public InstructionSelector { bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI); bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI); bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI); + bool selectPtrAuthGlobalValue(MachineInstr &I, + MachineRegisterInfo &MRI) const; bool selectReduction(MachineInstr &I, MachineRegisterInfo &MRI); bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI); bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI); @@ -2848,6 +2850,9 @@ bool AArch64InstructionSelector::select(MachineInstr &I) { return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } + case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE: + return selectPtrAuthGlobalValue(I, MRI); + case TargetOpcode::G_ZEXTLOAD: case TargetOpcode::G_LOAD: case TargetOpcode::G_STORE: { @@ -6583,6 +6588,145 @@ bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I, return false; } +// G_PTRAUTH_GLOBAL_VALUE lowering +// +// We have 3 lowering alternatives to choose from: +// - MOVaddrPAC: similar to MOVaddr, with added PAC. +// If the GV doesn't need a GOT load (i.e., is locally defined) +// materialize the pointer using adrp+add+pac. See LowerMOVaddrPAC. +// +// - LOADgotPAC: similar to LOADgot, with added PAC. +// If the GV needs a GOT load, materialize the pointer using the usual +// GOT adrp+ldr, +pac. Pointers in GOT are assumed to be not signed, the GOT +// section is assumed to be read-only (for example, via relro mechanism). See +// LowerMOVaddrPAC. +// +// - LOADauthptrstatic: similar to LOADgot, but use a +// special stub slot instead of a GOT slot. +// Load a signed pointer for symbol 'sym' from a stub slot named +// 'sym$auth_ptr$key$disc' filled by dynamic linker during relocation +// resolving. This usually lowers to adrp+ldr, but also emits an entry into +// .data with an +// @AUTH relocation. See LowerLOADauthptrstatic. +// +// All 3 are pseudos that are expand late to longer sequences: this lets us +// provide integrity guarantees on the to-be-signed intermediate values. +// +// LOADauthptrstatic is undesirable because it requires a large section filled +// with often similarly-signed pointers, making it a good harvesting target. +// Thus, it's only used for ptrauth references to extern_weak to avoid null +// checks. + +bool AArch64InstructionSelector::selectPtrAuthGlobalValue( + MachineInstr &I, MachineRegisterInfo &MRI) const { + Register DefReg = I.getOperand(0).getReg(); + Register Addr = I.getOperand(1).getReg(); + uint64_t Key = I.getOperand(2).getImm(); + Register AddrDisc = I.getOperand(3).getReg(); + uint64_t Disc = I.getOperand(4).getImm(); + int64_t Offset = 0; + + if (Key > AArch64PACKey::LAST) + report_fatal_error("key in ptrauth global out of range [0, " + + Twine((int)AArch64PACKey::LAST) + "]"); + + // Blend only works if the integer discriminator is 16-bit wide. + if (!isUInt<16>(Disc)) + report_fatal_error( + "constant discriminator in ptrauth global out of range [0, 0xffff]"); + + // Choosing between 3 lowering alternatives is target-specific. + if (!STI.isTargetELF()) + report_fatal_error("ptrauth global lowering is only implemented for ELF"); + + if (!MRI.hasOneDef(Addr)) + return false; + + // First match any offset we take from the real global. + const MachineInstr *DefMI = &*MRI.def_instr_begin(Addr); + if (DefMI->getOpcode() == TargetOpcode::G_PTR_ADD) { + Register OffsetReg = DefMI->getOperand(2).getReg(); + if (!MRI.hasOneDef(OffsetReg)) + return false; + const MachineInstr &OffsetMI = *MRI.def_instr_begin(OffsetReg); + if (OffsetMI.getOpcode() != TargetOpcode::G_CONSTANT) + return false; + + Addr = DefMI->getOperand(1).getReg(); + if (!MRI.hasOneDef(Addr)) + return false; + + DefMI = &*MRI.def_instr_begin(Addr); + Offset = OffsetMI.getOperand(1).getCImm()->getSExtValue(); + } + + // We should be left with a genuine unauthenticated GlobalValue. + const GlobalValue *GV; + if (DefMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { + GV = DefMI->getOperand(1).getGlobal(); + Offset += DefMI->getOperand(1).getOffset(); + } else if (DefMI->getOpcode() == AArch64::G_ADD_LOW) { + GV = DefMI->getOperand(2).getGlobal(); + Offset += DefMI->getOperand(2).getOffset(); + } else { + return false; + } + + MachineIRBuilder MIB(I); + + // Classify the reference to determine whether it needs a GOT load. + unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM); + const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0); + assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) && + "unsupported non-GOT op flags on ptrauth global reference"); + assert((!GV->hasExternalWeakLinkage() || NeedsGOTLoad) && + "unsupported non-GOT reference to weak ptrauth global"); + + std::optional AddrDiscVal = getIConstantVRegVal(AddrDisc, MRI); + bool HasAddrDisc = !AddrDiscVal || *AddrDiscVal != 0; + + // Non-extern_weak: + // - No GOT load needed -> MOVaddrPAC + // - GOT load for non-extern_weak -> LOADgotPAC + // Note that we disallow extern_weak refs to avoid null checks later. + if (!GV->hasExternalWeakLinkage()) { + MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {AArch64::X16}, {}); + MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {AArch64::X17}, {}); + MIB.buildInstr(NeedsGOTLoad ? AArch64::LOADgotPAC : AArch64::MOVaddrPAC) + .addGlobalAddress(GV, Offset) + .addImm(Key) + .addReg(HasAddrDisc ? AddrDisc : AArch64::XZR) + .addImm(Disc) + .constrainAllUses(TII, TRI, RBI); + MIB.buildCopy(DefReg, Register(AArch64::X16)); + RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI); + I.eraseFromParent(); + return true; + } + + // extern_weak -> LOADauthptrstatic + + // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the + // offset alone as a pointer if the symbol wasn't available, which would + // probably break null checks in users. Ptrauth complicates things further: + // error out. + if (Offset != 0) + report_fatal_error( + "unsupported non-zero offset in weak ptrauth global reference"); + + if (HasAddrDisc) + report_fatal_error("unsupported weak addr-div ptrauth global"); + + MIB.buildInstr(AArch64::LOADauthptrstatic, {DefReg}, {}) + .addGlobalAddress(GV, Offset) + .addImm(Key) + .addImm(Disc); + RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI); + + I.eraseFromParent(); + return true; +} + void AArch64InstructionSelector::SelectTable(MachineInstr &I, MachineRegisterInfo &MRI, unsigned NumVec, unsigned Opc1, diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index fef0b722efe45..50ba3057928ee 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -760,6 +760,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) else getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0}); + getActionDefinitionsBuilder(G_PTRAUTH_GLOBAL_VALUE) + .legalIf(all(typeIs(0, p0), typeIs(1, p0))); + getActionDefinitionsBuilder(G_PTRTOINT) .legalFor({{s64, p0}, {v2s64, v2p0}}) .widenScalarToNextPow2(0, 64) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index f03491924f7f4..1f048528ea153 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -86,6 +86,10 @@ # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected # +# DEBUG-NEXT: G_PTRAUTH_GLOBAL_VALUE (opcode {{[0-9]+}}): 2 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected +# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected +# # DEBUG-NEXT: G_CONSTANT_POOL (opcode {{[0-9]+}}): 1 type index, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll b/llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll new file mode 100644 index 0000000000000..7b85b12bb8952 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll @@ -0,0 +1,235 @@ +; RUN: rm -rf %t && split-file %s %t && cd %t + +;--- err1.ll + +; RUN: not --crash llc < err1.ll -mtriple aarch64-elf -mattr=+pauth \ +; RUN: -global-isel=1 -verify-machineinstrs -global-isel-abort=1 2>&1 | \ +; RUN: FileCheck --check-prefix=ERR1 %s + +@g = external global i32 + +define ptr @foo() { +; ERR1: LLVM ERROR: key in ptrauth global out of range [0, 3] + ret ptr ptrauth (ptr @g, i32 4) +} + +;--- err2.ll + +; RUN: not --crash llc < err2.ll -mtriple aarch64-elf -mattr=+pauth \ +; RUN: -global-isel=1 -verify-machineinstrs -global-isel-abort=1 2>&1 | \ +; RUN: FileCheck --check-prefix=ERR2 %s + +@g = external global i32 + +define ptr @foo() { +; ERR2: LLVM ERROR: constant discriminator in ptrauth global out of range [0, 0xffff] + ret ptr ptrauth (ptr @g, i32 2, i64 65536) +} + +;--- err3.ll + +; RUN: not --crash llc < err3.ll -mtriple aarch64-elf -mattr=+pauth \ +; RUN: -global-isel=1 -verify-machineinstrs -global-isel-abort=1 2>&1 | \ +; RUN: FileCheck --check-prefix=ERR3 %s + +@g_weak = extern_weak global i32 + +define ptr @foo() { +; ERR3: LLVM ERROR: unsupported non-zero offset in weak ptrauth global reference + ret ptr ptrauth (ptr getelementptr (i8, ptr @g_weak, i64 16), i32 2, i64 42) +} + +;--- err4.ll + +; RUN: not --crash llc < err4.ll -mtriple aarch64-elf -mattr=+pauth \ +; RUN: -global-isel=1 -verify-machineinstrs -global-isel-abort=1 2>&1 | \ +; RUN: FileCheck --check-prefix=ERR4 %s + +@g_weak = extern_weak global i32 +@g_weak.ref.da.42.addr = dso_local constant ptr ptrauth (ptr @g_weak, i32 2, i64 42, ptr @g_weak.ref.da.42.addr) + +define ptr @foo() { +; ERR4: LLVM ERROR: unsupported weak addr-div ptrauth global + ret ptr ptrauth (ptr @g_weak, i32 0, i64 42, ptr @g_weak.ref.da.42.addr) +} + +;--- err5.ll + +; RUN: not --crash llc < err5.ll -mtriple arm64-apple-darwin -mattr=+pauth \ +; RUN: -global-isel=1 -verify-machineinstrs -global-isel-abort=1 2>&1 | \ +; RUN: FileCheck --check-prefix=ERR5 %s + +@g = external global i32 + +define ptr @foo() { +; ERR5: LLVM ERROR: ptrauth global lowering is only implemented for ELF + ret ptr ptrauth (ptr @g, i32 0) +} + +;--- ok.ll + +; RUN: llc < ok.ll -mtriple aarch64-elf -mattr=+pauth -global-isel=1 \ +; RUN: -verify-machineinstrs -global-isel-abort=1 | FileCheck %s +; RUN: llc < ok.ll -mtriple aarch64-elf -mattr=+pauth -global-isel=1 \ +; RUN: -verify-machineinstrs -global-isel-abort=1 -filetype=obj + +@g = external global i32 +@g_weak = extern_weak global i32 +@g_strong_def = dso_local constant i32 42 + +define ptr @test_global_zero_disc() { +; CHECK-LABEL: test_global_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: paciza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr @g, i32 0) +} + +define ptr @test_global_offset_zero_disc() { +; CHECK-LABEL: test_global_offset_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: add x16, x16, #16 +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 16), i32 2) +} + +define ptr @test_global_neg_offset_zero_disc() { +; CHECK-LABEL: test_global_neg_offset_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: sub x16, x16, #576 +; CHECK-NEXT: sub x16, x16, #30, lsl #12 +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -123456), i32 2) +} + +define ptr @test_global_big_offset_zero_disc() { +; CHECK-LABEL: test_global_big_offset_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: mov x17, #1 +; CHECK-NEXT: movk x17, #32769, lsl #16 +; CHECK-NEXT: add x16, x16, x17 +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 add (i64 2147483648, i64 65537)), i32 2) +} + +define ptr @test_global_big_neg_offset_zero_disc() { +; CHECK-LABEL: test_global_big_neg_offset_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: mov x17, #-52501 +; CHECK-NEXT: movk x17, #63652, lsl #16 +; CHECK-NEXT: add x16, x16, x17 +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -123456789), i32 2) +} + +define ptr @test_global_huge_neg_offset_zero_disc() { +; CHECK-LABEL: test_global_huge_neg_offset_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: mov x17, #-65536 +; CHECK-NEXT: movk x17, #0, lsl #16 +; CHECK-NEXT: movk x17, #0, lsl #32 +; CHECK-NEXT: movk x17, #32768, lsl #48 +; CHECK-NEXT: add x16, x16, x17 +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -9223372036854775808), i32 2) +} + +define ptr @test_global_disc() { +; CHECK-LABEL: test_global_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: mov x17, #42 // =0x2a +; CHECK-NEXT: pacia x16, x17 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr @g, i32 0, i64 42) +} + +@g.ref.da.42.addr = dso_local constant ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr) + +define ptr @test_global_addr_disc() { +; CHECK-LABEL: test_global_addr_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, g.ref.da.42.addr +; CHECK-NEXT: add x8, x8, :lo12:g.ref.da.42.addr +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: mov x17, x8 +; CHECK-NEXT: movk x17, #42, lsl #48 +; CHECK-NEXT: pacda x16, x17 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr) +} + +define ptr @test_global_process_specific() { +; CHECK-LABEL: test_global_process_specific: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: pacizb x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + ret ptr ptrauth (ptr @g, i32 1) +} + +; weak symbols can't be assumed to be non-nil. Use $auth_ptr$ stub slot always. +; The alternative is to emit a null-check here, but that'd be redundant with +; whatever null-check follows in user code. + +define ptr @test_global_weak() { +; CHECK-LABEL: test_global_weak: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x0, g_weak$auth_ptr$ia$42 +; CHECK-NEXT: ldr x0, [x0, :lo12:g_weak$auth_ptr$ia$42] +; CHECK-NEXT: ret + ret ptr ptrauth (ptr @g_weak, i32 0, i64 42) +} + +; Non-external symbols don't need to be accessed through the GOT. + +define ptr @test_global_strong_def() { +; CHECK-LABEL: test_global_strong_def: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, g_strong_def +; CHECK-NEXT: add x16, x16, :lo12:g_strong_def +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + ret ptr ptrauth (ptr @g_strong_def, i32 2) +} + +; CHECK-LABEL: g_weak$auth_ptr$ia$42: +; CHECK-NEXT: .xword g_weak@AUTH(ia,42) diff --git a/llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll b/llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll new file mode 100644 index 0000000000000..73993fa87bf27 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll @@ -0,0 +1,230 @@ +; RUN: rm -rf %t && split-file %s %t && cd %t + +;--- err1.ll + +; RUN: not --crash llc < err1.ll -mtriple aarch64-elf -mattr=+pauth \ +; RUN: -global-isel=0 -verify-machineinstrs 2>&1 | FileCheck --check-prefix=ERR1 %s + +@g = external global i32 + +define ptr @foo() { +; ERR1: LLVM ERROR: key in ptrauth global out of range [0, 3] + ret ptr ptrauth (ptr @g, i32 4) +} + +;--- err2.ll + +; RUN: not --crash llc < err2.ll -mtriple aarch64-elf -mattr=+pauth \ +; RUN: -global-isel=0 -verify-machineinstrs 2>&1 | FileCheck --check-prefix=ERR2 %s + +@g = external global i32 + +define ptr @foo() { +; ERR2: LLVM ERROR: constant discriminator in ptrauth global out of range [0, 0xffff] + ret ptr ptrauth (ptr @g, i32 2, i64 65536) +} + +;--- err3.ll + +; RUN: not --crash llc < err3.ll -mtriple aarch64-elf -mattr=+pauth \ +; RUN: -global-isel=0 -verify-machineinstrs 2>&1 | FileCheck --check-prefix=ERR3 %s + +@g_weak = extern_weak global i32 + +define ptr @foo() { +; ERR3: LLVM ERROR: unsupported non-zero offset in weak ptrauth global reference + ret ptr ptrauth (ptr getelementptr (i8, ptr @g_weak, i64 16), i32 2, i64 42) +} + +;--- err4.ll + +; RUN: not --crash llc < err4.ll -mtriple aarch64-elf -mattr=+pauth \ +; RUN: -global-isel=0 -verify-machineinstrs 2>&1 | FileCheck --check-prefix=ERR4 %s + +@g_weak = extern_weak global i32 +@g_weak.ref.da.42.addr = dso_local constant ptr ptrauth (ptr @g_weak, i32 2, i64 42, ptr @g_weak.ref.da.42.addr) + +define ptr @foo() { +; ERR4: LLVM ERROR: unsupported weak addr-div ptrauth global + ret ptr ptrauth (ptr @g_weak, i32 0, i64 42, ptr @g_weak.ref.da.42.addr) +} + +;--- err5.ll + +; RUN: not --crash llc < err5.ll -mtriple arm64-apple-darwin -mattr=+pauth \ +; RUN: -global-isel=0 -verify-machineinstrs 2>&1 | FileCheck --check-prefix=ERR5 %s + +@g = external global i32 + +define ptr @foo() { +; ERR5: LLVM ERROR: ptrauth global lowering is only implemented for ELF + ret ptr ptrauth (ptr @g, i32 0) +} + +;--- ok.ll + +; RUN: llc < ok.ll -mtriple aarch64-elf -mattr=+pauth -global-isel=0 \ +; RUN: -verify-machineinstrs | FileCheck %s +; RUN: llc < ok.ll -mtriple aarch64-elf -mattr=+pauth -global-isel=0 \ +; RUN: -verify-machineinstrs -filetype=obj + +@g = external global i32 +@g_weak = extern_weak global i32 +@g_strong_def = dso_local constant i32 42 + +define ptr @test_global_zero_disc() { +; CHECK-LABEL: test_global_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: paciza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr @g, i32 0) +} + +define ptr @test_global_offset_zero_disc() { +; CHECK-LABEL: test_global_offset_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: add x16, x16, #16 +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 16), i32 2) +} + +define ptr @test_global_neg_offset_zero_disc() { +; CHECK-LABEL: test_global_neg_offset_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: sub x16, x16, #576 +; CHECK-NEXT: sub x16, x16, #30, lsl #12 +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -123456), i32 2) +} + +define ptr @test_global_big_offset_zero_disc() { +; CHECK-LABEL: test_global_big_offset_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: mov x17, #1 +; CHECK-NEXT: movk x17, #32769, lsl #16 +; CHECK-NEXT: add x16, x16, x17 +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 add (i64 2147483648, i64 65537)), i32 2) +} + +define ptr @test_global_big_neg_offset_zero_disc() { +; CHECK-LABEL: test_global_big_neg_offset_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: mov x17, #-52501 +; CHECK-NEXT: movk x17, #63652, lsl #16 +; CHECK-NEXT: add x16, x16, x17 +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -123456789), i32 2) +} + +define ptr @test_global_huge_neg_offset_zero_disc() { +; CHECK-LABEL: test_global_huge_neg_offset_zero_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: mov x17, #-65536 +; CHECK-NEXT: movk x17, #0, lsl #16 +; CHECK-NEXT: movk x17, #0, lsl #32 +; CHECK-NEXT: movk x17, #32768, lsl #48 +; CHECK-NEXT: add x16, x16, x17 +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -9223372036854775808), i32 2) +} + +define ptr @test_global_disc() { +; CHECK-LABEL: test_global_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: mov x17, #42 // =0x2a +; CHECK-NEXT: pacia x16, x17 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr @g, i32 0, i64 42) +} + +@g.ref.da.42.addr = dso_local constant ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr) + +define ptr @test_global_addr_disc() { +; CHECK-LABEL: test_global_addr_disc: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, g.ref.da.42.addr +; CHECK-NEXT: add x8, x8, :lo12:g.ref.da.42.addr +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: mov x17, x8 +; CHECK-NEXT: movk x17, #42, lsl #48 +; CHECK-NEXT: pacda x16, x17 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + + ret ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr) +} + +define ptr @test_global_process_specific() { +; CHECK-LABEL: test_global_process_specific: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, :got:g +; CHECK-NEXT: ldr x16, [x16, :got_lo12:g] +; CHECK-NEXT: pacizb x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + ret ptr ptrauth (ptr @g, i32 1) +} + +; weak symbols can't be assumed to be non-nil. Use $auth_ptr$ stub slot always. +; The alternative is to emit a null-check here, but that'd be redundant with +; whatever null-check follows in user code. + +define ptr @test_global_weak() { +; CHECK-LABEL: test_global_weak: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x0, g_weak$auth_ptr$ia$42 +; CHECK-NEXT: ldr x0, [x0, :lo12:g_weak$auth_ptr$ia$42] +; CHECK-NEXT: ret + ret ptr ptrauth (ptr @g_weak, i32 0, i64 42) +} + +; Non-external symbols don't need to be accessed through the GOT. + +define ptr @test_global_strong_def() { +; CHECK-LABEL: test_global_strong_def: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x16, g_strong_def +; CHECK-NEXT: add x16, x16, :lo12:g_strong_def +; CHECK-NEXT: pacdza x16 +; CHECK-NEXT: mov x0, x16 +; CHECK-NEXT: ret + ret ptr ptrauth (ptr @g_strong_def, i32 2) +} + +; CHECK-LABEL: g_weak$auth_ptr$ia$42: +; CHECK-NEXT: .xword g_weak@AUTH(ia,42) diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td index bb1acde296429..2825ade2d2134 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td @@ -85,7 +85,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(94), GIMT_Encode2(194), /*)*//*default:*//*Label 4*/ GIMT_Encode4(464), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(95), GIMT_Encode2(195), /*)*//*default:*//*Label 4*/ GIMT_Encode4(464), // CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4(410), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4(428), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 2*/ GIMT_Encode4(440), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td index 35bddf912a541..43cdd4f5546bb 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td @@ -37,7 +37,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(73), /*)*//*default:*//*Label 2*/ GIMT_Encode4(84), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(70), GIMT_Encode2(74), /*)*//*default:*//*Label 2*/ GIMT_Encode4(84), // CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(26), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_BUILD_VECTOR*//*Label 1*/ GIMT_Encode4(55), // CHECK-NEXT: // Label 0: @26 diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td index 513a86754d6d9..7ff637fa1e0e3 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td @@ -135,15 +135,15 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // Verify match table. // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(133), /*)*//*default:*//*Label 6*/ GIMT_Encode4(653), -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(466), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(502), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(549), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(583), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(606), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(618), -// CHECK-NEXT: // Label 0: @466 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(490), // Rule ID 4 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(134), /*)*//*default:*//*Label 6*/ GIMT_Encode4(657), +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(470), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(506), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(553), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(587), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(610), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(622), +// CHECK-NEXT: // Label 0: @470 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(494), // Rule ID 4 // // CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasAnswerToEverything), // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled), // CHECK-NEXT: // MIs[0] a @@ -156,8 +156,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIM_CheckIsSafeToFold, /*NumInsns*/1, // CHECK-NEXT: // Combiner Rule #3: InstTest1 // CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2), -// CHECK-NEXT: // Label 7: @490 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(501), // Rule ID 3 // +// CHECK-NEXT: // Label 7: @494 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(505), // Rule ID 3 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -165,10 +165,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // No operand predicates // CHECK-NEXT: // Combiner Rule #2: InstTest0 // CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1), -// CHECK-NEXT: // Label 8: @501 +// CHECK-NEXT: // Label 8: @505 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @502 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(548), // Rule ID 6 // +// CHECK-NEXT: // Label 1: @506 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(552), // Rule ID 6 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled), // CHECK-NEXT: GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] dst @@ -185,10 +185,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z // CHECK-NEXT: GIR_EraseRootFromParent_Done, -// CHECK-NEXT: // Label 9: @548 +// CHECK-NEXT: // Label 9: @552 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @549 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(582), // Rule ID 5 // +// CHECK-NEXT: // Label 2: @553 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(586), // Rule ID 5 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled), // CHECK-NEXT: // MIs[0] tmp // CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1] @@ -204,29 +204,29 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/1, // ptr // CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/2, /*MergeInsnID's*/0, 1, // CHECK-NEXT: GIR_EraseRootFromParent_Done, -// CHECK-NEXT: // Label 10: @582 +// CHECK-NEXT: // Label 10: @586 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @583 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(594), // Rule ID 0 // +// CHECK-NEXT: // Label 3: @587 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(598), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0), -// CHECK-NEXT: // Label 11: @594 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(605), // Rule ID 1 // +// CHECK-NEXT: // Label 11: @598 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(609), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0), -// CHECK-NEXT: // Label 12: @605 +// CHECK-NEXT: // Label 12: @609 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 4: @606 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(617), // Rule ID 2 // +// CHECK-NEXT: // Label 4: @610 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(621), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT' // CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0), -// CHECK-NEXT: // Label 13: @617 +// CHECK-NEXT: // Label 13: @621 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 5: @618 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(652), // Rule ID 7 // +// CHECK-NEXT: // Label 5: @622 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(656), // Rule ID 7 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule6Enabled), // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -240,10 +240,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseRootFromParent_Done, -// CHECK-NEXT: // Label 14: @652 +// CHECK-NEXT: // Label 14: @656 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 6: @653 +// CHECK-NEXT: // Label 6: @657 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 654 bytes +// CHECK-NEXT: }; // Size: 658 bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td index 5d5bf92664a79..796f595930319 100644 --- a/llvm/test/TableGen/GlobalISelEmitter.td +++ b/llvm/test/TableGen/GlobalISelEmitter.td @@ -513,7 +513,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00O-NEXT: GIM_Reject, // R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]] // R00O-NEXT: GIM_Reject, -// R00O-NEXT: }; // Size: 1804 bytes +// R00O-NEXT: }; // Size: 1808 bytes def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4), [(set GPR32:$dst,