@@ -5483,199 +5483,21 @@ AArch64InstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const {
54835483 return 0u ;
54845484}
54855485
5486- static bool
5487- outliningCandidatesSigningScopeConsensus (const outliner::Candidate &a,
5488- const outliner::Candidate &b) {
5489- const Function &Fa = a.getMF ()->getFunction ();
5490- const Function &Fb = b.getMF ()->getFunction ();
5491-
5492- // If none of the functions have the "sign-return-address" attribute their
5493- // signing behaviour is equal
5494- if (!Fa.hasFnAttribute (" sign-return-address" ) &&
5495- !Fb.hasFnAttribute (" sign-return-address" )) {
5496- return true ;
5497- }
5498-
5499- // If both functions have the "sign-return-address" attribute their signing
5500- // behaviour is equal, if the values of the attributes are equal
5501- if (Fa.hasFnAttribute (" sign-return-address" ) &&
5502- Fb.hasFnAttribute (" sign-return-address" )) {
5503- StringRef ScopeA =
5504- Fa.getFnAttribute (" sign-return-address" ).getValueAsString ();
5505- StringRef ScopeB =
5506- Fb.getFnAttribute (" sign-return-address" ).getValueAsString ();
5507- return ScopeA.equals (ScopeB);
5508- }
5509-
5510- // If function B doesn't have the "sign-return-address" attribute but A does,
5511- // the functions' signing behaviour is equal if A's value for
5512- // "sign-return-address" is "none" and vice versa.
5513- if (Fa.hasFnAttribute (" sign-return-address" )) {
5514- StringRef ScopeA =
5515- Fa.getFnAttribute (" sign-return-address" ).getValueAsString ();
5516- return ScopeA.equals (" none" );
5517- }
5518-
5519- if (Fb.hasFnAttribute (" sign-return-address" )) {
5520- StringRef ScopeB =
5521- Fb.getFnAttribute (" sign-return-address" ).getValueAsString ();
5522- return ScopeB.equals (" none" );
5523- }
5524-
5525- llvm_unreachable (" Unkown combination of sign-return-address attributes" );
5526- }
5527-
5528- static bool
5529- outliningCandidatesSigningKeyConsensus (const outliner::Candidate &a,
5530- const outliner::Candidate &b) {
5531- const Function &Fa = a.getMF ()->getFunction ();
5532- const Function &Fb = b.getMF ()->getFunction ();
5533-
5534- // If none of the functions have the "sign-return-address-key" attribute
5535- // their keys are equal
5536- if (!Fa.hasFnAttribute (" sign-return-address-key" ) &&
5537- !Fb.hasFnAttribute (" sign-return-address-key" )) {
5538- return true ;
5539- }
5540-
5541- // If both functions have the "sign-return-address-key" attribute their
5542- // keys are equal if the values of "sign-return-address-key" are equal
5543- if (Fa.hasFnAttribute (" sign-return-address-key" ) &&
5544- Fb.hasFnAttribute (" sign-return-address-key" )) {
5545- StringRef KeyA =
5546- Fa.getFnAttribute (" sign-return-address-key" ).getValueAsString ();
5547- StringRef KeyB =
5548- Fb.getFnAttribute (" sign-return-address-key" ).getValueAsString ();
5549- return KeyA.equals (KeyB);
5550- }
5551-
5552- // If B doesn't have the "sign-return-address-key" attribute, both keys are
5553- // equal, if function a has the default key (a_key)
5554- if (Fa.hasFnAttribute (" sign-return-address-key" )) {
5555- StringRef KeyA =
5556- Fa.getFnAttribute (" sign-return-address-key" ).getValueAsString ();
5557- return KeyA.equals_lower (" a_key" );
5558- }
5559-
5560- if (Fb.hasFnAttribute (" sign-return-address-key" )) {
5561- StringRef KeyB =
5562- Fb.getFnAttribute (" sign-return-address-key" ).getValueAsString ();
5563- return KeyB.equals_lower (" a_key" );
5564- }
5565-
5566- llvm_unreachable (" Unkown combination of sign-return-address-key attributes" );
5567- }
5568-
5569- static bool outliningCandidatesV8_3OpsConsensus (const outliner::Candidate &a,
5570- const outliner::Candidate &b) {
5571- const AArch64Subtarget &SubtargetA =
5572- a.getMF ()->getSubtarget <AArch64Subtarget>();
5573- const AArch64Subtarget &SubtargetB =
5574- b.getMF ()->getSubtarget <AArch64Subtarget>();
5575- return SubtargetA.hasV8_3aOps () == SubtargetB.hasV8_3aOps ();
5576- }
5577-
5578- outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo (
5486+ outliner::OutlinedFunction
5487+ AArch64InstrInfo::getOutliningCandidateInfo (
55795488 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
55805489 outliner::Candidate &FirstCand = RepeatedSequenceLocs[0 ];
55815490 unsigned SequenceSize =
55825491 std::accumulate (FirstCand.front (), std::next (FirstCand.back ()), 0 ,
55835492 [this ](unsigned Sum, const MachineInstr &MI) {
55845493 return Sum + getInstSizeInBytes (MI);
55855494 });
5586- unsigned NumBytesToCreateFrame = 0 ;
5587-
5588- // We only allow outlining for functions having exactly matching return
5589- // address signing attributes, i.e., all share the same value for the
5590- // attribute "sign-return-address" and all share the same type of key they
5591- // are signed with.
5592- // Additionally we require all functions to simultaniously either support
5593- // v8.3a features or not. Otherwise an outlined function could get signed
5594- // using dedicated v8.3 instructions and a call from a function that doesn't
5595- // support v8.3 instructions would therefore be invalid.
5596- if (std::adjacent_find (
5597- RepeatedSequenceLocs.begin (), RepeatedSequenceLocs.end (),
5598- [](const outliner::Candidate &a, const outliner::Candidate &b) {
5599- // Return true if a and b are non-equal w.r.t. return address
5600- // signing or support of v8.3a features
5601- if (outliningCandidatesSigningScopeConsensus (a, b) &&
5602- outliningCandidatesSigningKeyConsensus (a, b) &&
5603- outliningCandidatesV8_3OpsConsensus (a, b)) {
5604- return false ;
5605- }
5606- return true ;
5607- }) != RepeatedSequenceLocs.end ()) {
5608- return outliner::OutlinedFunction ();
5609- }
5610-
5611- // Since at this point all candidates agree on their return address signing
5612- // picking just one is fine. If the candidate functions potentially sign their
5613- // return addresses, the outlined function should do the same. Note that in
5614- // the case of "sign-return-address"="non-leaf" this is an assumption: It is
5615- // not certainly true that the outlined function will have to sign its return
5616- // address but this decision is made later, when the decision to outline
5617- // has already been made.
5618- // The same holds for the number of additional instructions we need: On
5619- // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
5620- // necessary. However, at this point we don't know if the outlined function
5621- // will have a RET instruction so we assume the worst.
5622- const Function &FCF = FirstCand.getMF ()->getFunction ();
5623- const TargetRegisterInfo &TRI = getRegisterInfo ();
5624- if (FCF.hasFnAttribute (" sign-return-address" )) {
5625- // One PAC and one AUT instructions
5626- NumBytesToCreateFrame += 8 ;
5627-
5628- // We have to check if sp modifying instructions would get outlined.
5629- // If so we only allow outlining if sp is unchanged overall, so matching
5630- // sub and add instructions are okay to outline, all other sp modifications
5631- // are not
5632- auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
5633- int SPValue = 0 ;
5634- MachineBasicBlock::iterator MBBI = C.front ();
5635- for (;;) {
5636- if (MBBI->modifiesRegister (AArch64::SP, &TRI)) {
5637- switch (MBBI->getOpcode ()) {
5638- case AArch64::ADDXri:
5639- case AArch64::ADDWri:
5640- assert (MBBI->getNumOperands () == 4 && " Wrong number of operands" );
5641- assert (MBBI->getOperand (2 ).isImm () &&
5642- " Expected operand to be immediate" );
5643- SPValue += MBBI->getOperand (2 ).getImm ();
5644- break ;
5645- case AArch64::SUBXri:
5646- case AArch64::SUBWri:
5647- assert (MBBI->getNumOperands () == 4 && " Wrong number of operands" );
5648- assert (MBBI->getOperand (2 ).isImm () &&
5649- " Expected operand to be immediate" );
5650- SPValue -= MBBI->getOperand (2 ).getImm ();
5651- break ;
5652- default :
5653- return true ;
5654- }
5655- }
5656- if (MBBI == C.back ())
5657- break ;
5658- ++MBBI;
5659- }
5660- if (SPValue)
5661- return true ;
5662- return false ;
5663- };
5664- // Remove candidates with illegal stack modifying instructions
5665- RepeatedSequenceLocs.erase (std::remove_if (RepeatedSequenceLocs.begin (),
5666- RepeatedSequenceLocs.end (),
5667- hasIllegalSPModification),
5668- RepeatedSequenceLocs.end ());
5669-
5670- // If the sequence doesn't have enough candidates left, then we're done.
5671- if (RepeatedSequenceLocs.size () < 2 )
5672- return outliner::OutlinedFunction ();
5673- }
56745495
56755496 // Properties about candidate MBBs that hold for all of them.
56765497 unsigned FlagsSetInAll = 0xF ;
56775498
56785499 // Compute liveness information for each candidate, and set FlagsSetInAll.
5500+ const TargetRegisterInfo &TRI = getRegisterInfo ();
56795501 std::for_each (RepeatedSequenceLocs.begin (), RepeatedSequenceLocs.end (),
56805502 [&FlagsSetInAll](outliner::Candidate &C) {
56815503 FlagsSetInAll &= C.Flags ;
@@ -5731,7 +5553,7 @@ outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo(
57315553 };
57325554
57335555 unsigned FrameID = MachineOutlinerDefault;
5734- NumBytesToCreateFrame + = 4 ;
5556+ unsigned NumBytesToCreateFrame = 4 ;
57355557
57365558 bool HasBTI = any_of (RepeatedSequenceLocs, [](outliner::Candidate &C) {
57375559 return C.getMF ()->getFunction ().hasFnAttribute (" branch-target-enforcement" );
@@ -6000,19 +5822,6 @@ AArch64InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
60005822 MachineFunction *MF = MBB->getParent ();
60015823 AArch64FunctionInfo *FuncInfo = MF->getInfo <AArch64FunctionInfo>();
60025824
6003- // Don't outline anything used for return address signing. The outlined
6004- // function will get signed later if needed
6005- switch (MI.getOpcode ()) {
6006- case AArch64::PACIASP:
6007- case AArch64::PACIBSP:
6008- case AArch64::AUTIASP:
6009- case AArch64::AUTIBSP:
6010- case AArch64::RETAA:
6011- case AArch64::RETAB:
6012- case AArch64::EMITBKEY:
6013- return outliner::InstrType::Illegal;
6014- }
6015-
60165825 // Don't outline LOHs.
60175826 if (FuncInfo->getLOHRelated ().count (&MI))
60185827 return outliner::InstrType::Illegal;
@@ -6165,59 +5974,6 @@ void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
61655974 }
61665975}
61675976
6168- static void signOutlinedFunction (MachineFunction &MF, MachineBasicBlock &MBB,
6169- bool ShouldSignReturnAddr,
6170- bool ShouldSignReturnAddrWithAKey) {
6171- if (ShouldSignReturnAddr) {
6172- MachineBasicBlock::iterator MBBPAC = MBB.begin ();
6173- MachineBasicBlock::iterator MBBAUT = MBB.getFirstTerminator ();
6174- const AArch64Subtarget &Subtarget = MF.getSubtarget <AArch64Subtarget>();
6175- const TargetInstrInfo *TII = Subtarget.getInstrInfo ();
6176- DebugLoc DL;
6177-
6178- if (MBBAUT != MBB.end ())
6179- DL = MBBAUT->getDebugLoc ();
6180-
6181- // At the very beginning of the basic block we insert the following
6182- // depending on the key type
6183- //
6184- // a_key: b_key:
6185- // PACIASP EMITBKEY
6186- // CFI_INSTRUCTION PACIBSP
6187- // CFI_INSTRUCTION
6188- if (ShouldSignReturnAddrWithAKey) {
6189- BuildMI (MBB, MBBPAC, DebugLoc (), TII->get (AArch64::PACIASP))
6190- .setMIFlag (MachineInstr::FrameSetup);
6191- } else {
6192- BuildMI (MBB, MBBPAC, DebugLoc (), TII->get (AArch64::EMITBKEY))
6193- .setMIFlag (MachineInstr::FrameSetup);
6194- BuildMI (MBB, MBBPAC, DebugLoc (), TII->get (AArch64::PACIBSP))
6195- .setMIFlag (MachineInstr::FrameSetup);
6196- }
6197- unsigned CFIIndex =
6198- MF.addFrameInst (MCCFIInstruction::createNegateRAState (nullptr ));
6199- BuildMI (MBB, MBBPAC, DebugLoc (), TII->get (AArch64::CFI_INSTRUCTION))
6200- .addCFIIndex (CFIIndex)
6201- .setMIFlags (MachineInstr::FrameSetup);
6202-
6203- // If v8.3a features are available we can replace a RET instruction by
6204- // RETAA or RETAB and omit the AUT instructions
6205- if (Subtarget.hasV8_3aOps () && MBBAUT != MBB.end () &&
6206- MBBAUT->getOpcode () == AArch64::RET) {
6207- BuildMI (MBB, MBBAUT, DL,
6208- TII->get (ShouldSignReturnAddrWithAKey ? AArch64::RETAA
6209- : AArch64::RETAB))
6210- .copyImplicitOps (*MBBAUT);
6211- MBB.erase (MBBAUT);
6212- } else {
6213- BuildMI (MBB, MBBAUT, DL,
6214- TII->get (ShouldSignReturnAddrWithAKey ? AArch64::AUTIASP
6215- : AArch64::AUTIBSP))
6216- .setMIFlag (MachineInstr::FrameDestroy);
6217- }
6218- }
6219- }
6220-
62215977void AArch64InstrInfo::buildOutlinedFrame (
62225978 MachineBasicBlock &MBB, MachineFunction &MF,
62235979 const outliner::OutlinedFunction &OF) const {
@@ -6233,28 +5989,23 @@ void AArch64InstrInfo::buildOutlinedFrame(
62335989 TailOpcode = AArch64::TCRETURNriALL;
62345990 }
62355991 MachineInstr *TC = BuildMI (MF, DebugLoc (), get (TailOpcode))
6236- .add (Call->getOperand (0 ))
6237- .addImm (0 );
5992+ .add (Call->getOperand (0 ))
5993+ .addImm (0 );
62385994 MBB.insert (MBB.end (), TC);
62395995 Call->eraseFromParent ();
62405996 }
62415997
6242- bool IsLeafFunction = true ;
6243-
62445998 // Is there a call in the outlined range?
6245- auto IsNonTailCall = [](const MachineInstr &MI) {
5999+ auto IsNonTailCall = [](MachineInstr &MI) {
62466000 return MI.isCall () && !MI.isReturn ();
62476001 };
6248-
62496002 if (std::any_of (MBB.instr_begin (), MBB.instr_end (), IsNonTailCall)) {
62506003 // Fix up the instructions in the range, since we're going to modify the
62516004 // stack.
62526005 assert (OF.FrameConstructionID != MachineOutlinerDefault &&
62536006 " Can only fix up stack references once" );
62546007 fixupPostOutline (MBB);
62556008
6256- IsLeafFunction = false ;
6257-
62586009 // LR has to be a live in so that we can save it.
62596010 MBB.addLiveIn (AArch64::LR);
62606011
@@ -6301,47 +6052,16 @@ void AArch64InstrInfo::buildOutlinedFrame(
63016052 Et = MBB.insert (Et, LDRXpost);
63026053 }
63036054
6304- // If a bunch of candidates reach this point they must agree on their return
6305- // address signing. It is therefore enough to just consider the signing
6306- // behaviour of one of them
6307- const Function &CF = OF.Candidates .front ().getMF ()->getFunction ();
6308- bool ShouldSignReturnAddr = false ;
6309- if (CF.hasFnAttribute (" sign-return-address" )) {
6310- StringRef Scope =
6311- CF.getFnAttribute (" sign-return-address" ).getValueAsString ();
6312- if (Scope.equals (" all" ))
6313- ShouldSignReturnAddr = true ;
6314- else if (Scope.equals (" non-leaf" ) && !IsLeafFunction)
6315- ShouldSignReturnAddr = true ;
6316- }
6317-
6318- // a_key is the default
6319- bool ShouldSignReturnAddrWithAKey = true ;
6320- if (CF.hasFnAttribute (" sign-return-address-key" )) {
6321- const StringRef Key =
6322- CF.getFnAttribute (" sign-return-address-key" ).getValueAsString ();
6323- // Key can either be a_key or b_key
6324- assert ((Key.equals_lower (" a_key" ) || Key.equals_lower (" b_key" )) &&
6325- " Return address signing key must be either a_key or b_key" );
6326- ShouldSignReturnAddrWithAKey = Key.equals_lower (" a_key" );
6327- }
6328-
63296055 // If this is a tail call outlined function, then there's already a return.
63306056 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6331- OF.FrameConstructionID == MachineOutlinerThunk) {
6332- signOutlinedFunction (MF, MBB, ShouldSignReturnAddr,
6333- ShouldSignReturnAddrWithAKey);
6057+ OF.FrameConstructionID == MachineOutlinerThunk)
63346058 return ;
6335- }
63366059
63376060 // It's not a tail call, so we have to insert the return ourselves.
63386061 MachineInstr *ret = BuildMI (MF, DebugLoc (), get (AArch64::RET))
63396062 .addReg (AArch64::LR, RegState::Undef);
63406063 MBB.insert (MBB.end (), ret);
63416064
6342- signOutlinedFunction (MF, MBB, ShouldSignReturnAddr,
6343- ShouldSignReturnAddrWithAKey);
6344-
63456065 // Did we have to modify the stack by saving the link register?
63466066 if (OF.FrameConstructionID != MachineOutlinerDefault)
63476067 return ;
0 commit comments