diff --git a/llvm/include/llvm/CodeGen/StackSlotColoring.h b/llvm/include/llvm/CodeGen/StackSlotColoring.h new file mode 100644 index 0000000000000..8db59a339280d --- /dev/null +++ b/llvm/include/llvm/CodeGen/StackSlotColoring.h @@ -0,0 +1,24 @@ +//===- llvm/CodeGen/StackSlotColoring.h -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_STACKSLOTCOLORING_H +#define LLVM_CODEGEN_STACKSLOTCOLORING_H + +#include "llvm/CodeGen/MachinePassManager.h" + +namespace llvm { + +class StackSlotColoringPass : public PassInfoMixin { +public: + PreservedAnalyses run(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM); +}; + +} // namespace llvm + +#endif // LLVM_CODEGEN_STACKSLOTCOLORING_H diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 1acc57bf219a6..b8df4d1ecab1d 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -298,7 +298,7 @@ void initializeStackMapLivenessPass(PassRegistry &); void initializeStackProtectorPass(PassRegistry &); void initializeStackSafetyGlobalInfoWrapperPassPass(PassRegistry &); void initializeStackSafetyInfoWrapperPassPass(PassRegistry &); -void initializeStackSlotColoringPass(PassRegistry &); +void initializeStackSlotColoringLegacyPass(PassRegistry &); void initializeStraightLineStrengthReduceLegacyPassPass(PassRegistry &); void initializeStripDebugMachineModulePass(PassRegistry &); void initializeStructurizeCFGLegacyPassPass(PassRegistry &); diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index 16b7e4c0336d8..1458318ff021a 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -70,6 +70,7 @@ #include "llvm/CodeGen/SjLjEHPrepare.h" #include "llvm/CodeGen/StackColoring.h" #include "llvm/CodeGen/StackProtector.h" +#include "llvm/CodeGen/StackSlotColoring.h" #include "llvm/CodeGen/TailDuplication.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TwoAddressInstructionPass.h" diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index f816c9345d829..e6b4a4b0a56ae 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -173,6 +173,7 @@ MACHINE_FUNCTION_PASS("rename-independent-subregs", RenameIndependentSubregsPass MACHINE_FUNCTION_PASS("require-all-machine-function-properties", RequireAllMachineFunctionPropertiesPass()) MACHINE_FUNCTION_PASS("stack-coloring", StackColoringPass()) +MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass()) MACHINE_FUNCTION_PASS("tailduplication", TailDuplicatePass()) MACHINE_FUNCTION_PASS("trigger-verifier-error", TriggerVerifierErrorPass()) MACHINE_FUNCTION_PASS("two-address-instruction", TwoAddressInstructionPass()) @@ -267,7 +268,6 @@ DUMMY_MACHINE_FUNCTION_PASS("removeredundantdebugvalues", RemoveRedundantDebugVa DUMMY_MACHINE_FUNCTION_PASS("reset-machine-function", ResetMachineFunctionPass) DUMMY_MACHINE_FUNCTION_PASS("shrink-wrap", ShrinkWrapPass) DUMMY_MACHINE_FUNCTION_PASS("stack-frame-layout", StackFrameLayoutAnalysisPass) -DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass) DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass) DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass) DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass) diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp index 5955d0fc32940..35df2a479a545 100644 --- a/llvm/lib/CodeGen/CodeGen.cpp +++ b/llvm/lib/CodeGen/CodeGen.cpp @@ -130,7 +130,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeStackFrameLayoutAnalysisPassPass(Registry); initializeStackMapLivenessPass(Registry); initializeStackProtectorPass(Registry); - initializeStackSlotColoringPass(Registry); + initializeStackSlotColoringLegacyPass(Registry); initializeStaticDataSplitterPass(Registry); initializeStripDebugMachineModulePass(Registry); initializeTailDuplicateLegacyPass(Registry); diff --git a/llvm/lib/CodeGen/StackSlotColoring.cpp b/llvm/lib/CodeGen/StackSlotColoring.cpp index 3e57ee036081c..22c5c2e080b05 100644 --- a/llvm/lib/CodeGen/StackSlotColoring.cpp +++ b/llvm/lib/CodeGen/StackSlotColoring.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/StackSlotColoring.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" @@ -20,6 +21,7 @@ #include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" +#include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -59,136 +61,145 @@ STATISTIC(NumDead, "Number of trivially dead stack accesses eliminated"); namespace { - class StackSlotColoring : public MachineFunctionPass { - LiveStacks *LS = nullptr; - MachineFrameInfo *MFI = nullptr; - const TargetInstrInfo *TII = nullptr; - const MachineBlockFrequencyInfo *MBFI = nullptr; - SlotIndexes *Indexes = nullptr; - - // SSIntervals - Spill slot intervals. - std::vector SSIntervals; - - // SSRefs - Keep a list of MachineMemOperands for each spill slot. - // MachineMemOperands can be shared between instructions, so we need - // to be careful that renames like [FI0, FI1] -> [FI1, FI2] do not - // become FI0 -> FI1 -> FI2. - SmallVector, 16> SSRefs; - - // OrigAlignments - Alignments of stack objects before coloring. - SmallVector OrigAlignments; - - // OrigSizes - Sizes of stack objects before coloring. - SmallVector OrigSizes; - - // AllColors - If index is set, it's a spill slot, i.e. color. - // FIXME: This assumes PEI locate spill slot with smaller indices - // closest to stack pointer / frame pointer. Therefore, smaller - // index == better color. This is per stack ID. - SmallVector AllColors; - - // NextColor - Next "color" that's not yet used. This is per stack ID. - SmallVector NextColors = { -1 }; - - // UsedColors - "Colors" that have been assigned. This is per stack ID - SmallVector UsedColors; - - // Join all intervals sharing one color into a single LiveIntervalUnion to - // speedup range overlap test. - class ColorAssignmentInfo { - // Single liverange (used to avoid creation of LiveIntervalUnion). - LiveInterval *SingleLI = nullptr; - // LiveIntervalUnion to perform overlap test. - LiveIntervalUnion *LIU = nullptr; - // LiveIntervalUnion has a parameter in its constructor so doing this - // dirty magic. - uint8_t LIUPad[sizeof(LiveIntervalUnion)]; - - public: - ~ColorAssignmentInfo() { - if (LIU) - LIU->~LiveIntervalUnion(); // Dirty magic again. - } - - // Return true if LiveInterval overlaps with any - // intervals that have already been assigned to this color. - bool overlaps(LiveInterval *LI) const { - if (LIU) - return LiveIntervalUnion::Query(*LI, *LIU).checkInterference(); - return SingleLI ? SingleLI->overlaps(*LI) : false; - } - - // Add new LiveInterval to this color. - void add(LiveInterval *LI, LiveIntervalUnion::Allocator &Alloc) { - assert(!overlaps(LI)); - if (LIU) { - LIU->unify(*LI, *LI); - } else if (SingleLI) { - LIU = new (LIUPad) LiveIntervalUnion(Alloc); - LIU->unify(*SingleLI, *SingleLI); - LIU->unify(*LI, *LI); - SingleLI = nullptr; - } else - SingleLI = LI; - } - }; - - LiveIntervalUnion::Allocator LIUAlloc; - - // Assignments - Color to intervals mapping. - SmallVector Assignments; +class StackSlotColoring { + MachineFrameInfo *MFI = nullptr; + const TargetInstrInfo *TII = nullptr; + LiveStacks *LS = nullptr; + const MachineBlockFrequencyInfo *MBFI = nullptr; + SlotIndexes *Indexes = nullptr; + + // SSIntervals - Spill slot intervals. + std::vector SSIntervals; + + // SSRefs - Keep a list of MachineMemOperands for each spill slot. + // MachineMemOperands can be shared between instructions, so we need + // to be careful that renames like [FI0, FI1] -> [FI1, FI2] do not + // become FI0 -> FI1 -> FI2. + SmallVector, 16> SSRefs; + + // OrigAlignments - Alignments of stack objects before coloring. + SmallVector OrigAlignments; + + // OrigSizes - Sizes of stack objects before coloring. + SmallVector OrigSizes; + + // AllColors - If index is set, it's a spill slot, i.e. color. + // FIXME: This assumes PEI locate spill slot with smaller indices + // closest to stack pointer / frame pointer. Therefore, smaller + // index == better color. This is per stack ID. + SmallVector AllColors; + + // NextColor - Next "color" that's not yet used. This is per stack ID. + SmallVector NextColors = {-1}; + + // UsedColors - "Colors" that have been assigned. This is per stack ID + SmallVector UsedColors; + + // Join all intervals sharing one color into a single LiveIntervalUnion to + // speedup range overlap test. + class ColorAssignmentInfo { + // Single liverange (used to avoid creation of LiveIntervalUnion). + LiveInterval *SingleLI = nullptr; + // LiveIntervalUnion to perform overlap test. + LiveIntervalUnion *LIU = nullptr; + // LiveIntervalUnion has a parameter in its constructor so doing this + // dirty magic. + uint8_t LIUPad[sizeof(LiveIntervalUnion)]; public: - static char ID; // Pass identification + ~ColorAssignmentInfo() { + if (LIU) + LIU->~LiveIntervalUnion(); // Dirty magic again. + } - StackSlotColoring() : MachineFunctionPass(ID) { - initializeStackSlotColoringPass(*PassRegistry::getPassRegistry()); + // Return true if LiveInterval overlaps with any + // intervals that have already been assigned to this color. + bool overlaps(LiveInterval *LI) const { + if (LIU) + return LiveIntervalUnion::Query(*LI, *LIU).checkInterference(); + return SingleLI ? SingleLI->overlaps(*LI) : false; } - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.setPreservesCFG(); - AU.addRequired(); - AU.addPreserved(); - AU.addRequired(); - AU.addRequired(); - AU.addPreserved(); - AU.addPreservedID(MachineDominatorsID); - - // In some Target's pipeline, register allocation (RA) might be - // split into multiple phases based on register class. So, this pass - // may be invoked multiple times requiring it to save these analyses to be - // used by RA later. - AU.addPreserved(); - AU.addPreserved(); - - MachineFunctionPass::getAnalysisUsage(AU); + // Add new LiveInterval to this color. + void add(LiveInterval *LI, LiveIntervalUnion::Allocator &Alloc) { + assert(!overlaps(LI)); + if (LIU) { + LIU->unify(*LI, *LI); + } else if (SingleLI) { + LIU = new (LIUPad) LiveIntervalUnion(Alloc); + LIU->unify(*SingleLI, *SingleLI); + LIU->unify(*LI, *LI); + SingleLI = nullptr; + } else + SingleLI = LI; } + }; + + LiveIntervalUnion::Allocator LIUAlloc; + + // Assignments - Color to intervals mapping. + SmallVector Assignments; + +public: + StackSlotColoring(MachineFunction &MF, LiveStacks *LS, + MachineBlockFrequencyInfo *MBFI, SlotIndexes *Indexes) + : MFI(&MF.getFrameInfo()), TII(MF.getSubtarget().getInstrInfo()), LS(LS), + MBFI(MBFI), Indexes(Indexes) {} + bool run(MachineFunction &MF); + +private: + void InitializeSlots(); + void ScanForSpillSlotRefs(MachineFunction &MF); + int ColorSlot(LiveInterval *li); + bool ColorSlots(MachineFunction &MF); + void RewriteInstruction(MachineInstr &MI, SmallVectorImpl &SlotMapping, + MachineFunction &MF); + bool RemoveDeadStores(MachineBasicBlock *MBB); +}; - bool runOnMachineFunction(MachineFunction &MF) override; +class StackSlotColoringLegacy : public MachineFunctionPass { +public: + static char ID; // Pass identification - private: - void InitializeSlots(); - void ScanForSpillSlotRefs(MachineFunction &MF); - int ColorSlot(LiveInterval *li); - bool ColorSlots(MachineFunction &MF); - void RewriteInstruction(MachineInstr &MI, SmallVectorImpl &SlotMapping, - MachineFunction &MF); - bool RemoveDeadStores(MachineBasicBlock* MBB); - }; + StackSlotColoringLegacy() : MachineFunctionPass(ID) { + initializeStackSlotColoringLegacyPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addRequired(); + AU.addPreserved(); + AU.addPreservedID(MachineDominatorsID); + + // In some Target's pipeline, register allocation (RA) might be + // split into multiple phases based on register class. So, this pass + // may be invoked multiple times requiring it to save these analyses to be + // used by RA later. + AU.addPreserved(); + AU.addPreserved(); + + MachineFunctionPass::getAnalysisUsage(AU); + } + + bool runOnMachineFunction(MachineFunction &MF) override; +}; } // end anonymous namespace -char StackSlotColoring::ID = 0; +char StackSlotColoringLegacy::ID = 0; -char &llvm::StackSlotColoringID = StackSlotColoring::ID; +char &llvm::StackSlotColoringID = StackSlotColoringLegacy::ID; -INITIALIZE_PASS_BEGIN(StackSlotColoring, DEBUG_TYPE, - "Stack Slot Coloring", false, false) +INITIALIZE_PASS_BEGIN(StackSlotColoringLegacy, DEBUG_TYPE, + "Stack Slot Coloring", false, false) INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass) INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass) -INITIALIZE_PASS_END(StackSlotColoring, DEBUG_TYPE, - "Stack Slot Coloring", false, false) +INITIALIZE_PASS_END(StackSlotColoringLegacy, DEBUG_TYPE, "Stack Slot Coloring", + false, false) namespace { @@ -511,21 +522,12 @@ bool StackSlotColoring::RemoveDeadStores(MachineBasicBlock* MBB) { return changed; } -bool StackSlotColoring::runOnMachineFunction(MachineFunction &MF) { +bool StackSlotColoring::run(MachineFunction &MF) { LLVM_DEBUG({ dbgs() << "********** Stack Slot Coloring **********\n" << "********** Function: " << MF.getName() << '\n'; }); - if (skipFunction(MF.getFunction())) - return false; - - MFI = &MF.getFrameInfo(); - TII = MF.getSubtarget().getInstrInfo(); - LS = &getAnalysis().getLS(); - MBFI = &getAnalysis().getMBFI(); - Indexes = &getAnalysis().getSI(); - bool Changed = false; unsigned NumSlots = LS->getNumIntervals(); @@ -559,3 +561,37 @@ bool StackSlotColoring::runOnMachineFunction(MachineFunction &MF) { return Changed; } + +bool StackSlotColoringLegacy::runOnMachineFunction(MachineFunction &MF) { + if (skipFunction(MF.getFunction())) + return false; + + LiveStacks *LS = &getAnalysis().getLS(); + MachineBlockFrequencyInfo *MBFI = + &getAnalysis().getMBFI(); + SlotIndexes *Indexes = &getAnalysis().getSI(); + StackSlotColoring Impl(MF, LS, MBFI, Indexes); + return Impl.run(MF); +} + +PreservedAnalyses +StackSlotColoringPass::run(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM) { + LiveStacks *LS = &MFAM.getResult(MF); + MachineBlockFrequencyInfo *MBFI = + &MFAM.getResult(MF); + SlotIndexes *Indexes = &MFAM.getResult(MF); + StackSlotColoring Impl(MF, LS, MBFI, Indexes); + bool Changed = Impl.run(MF); + if (!Changed) + return PreservedAnalyses::all(); + + auto PA = getMachineFunctionPassPreservedAnalyses(); + PA.preserveSet(); + PA.preserve(); + PA.preserve(); + PA.preserve(); + PA.preserve(); + PA.preserve(); + return PA; +} diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 7a384b1857164..650d23ac1d5ef 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -141,6 +141,7 @@ #include "llvm/CodeGen/SpillPlacement.h" #include "llvm/CodeGen/StackColoring.h" #include "llvm/CodeGen/StackProtector.h" +#include "llvm/CodeGen/StackSlotColoring.h" #include "llvm/CodeGen/TailDuplication.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TwoAddressInstructionPass.h"