diff --git a/llvm/include/llvm/Transforms/Scalar/SROA.h b/llvm/include/llvm/Transforms/Scalar/SROA.h index b18e3054ef3ae..c03cdf48fb1c6 100644 --- a/llvm/include/llvm/Transforms/Scalar/SROA.h +++ b/llvm/include/llvm/Transforms/Scalar/SROA.h @@ -15,146 +15,16 @@ #ifndef LLVM_TRANSFORMS_SCALAR_SROA_H #define LLVM_TRANSFORMS_SCALAR_SROA_H -#include "llvm/ADT/MapVector.h" -#include "llvm/ADT/PointerIntPair.h" -#include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" -#include "llvm/IR/ValueHandle.h" -#include -#include namespace llvm { -class AllocaInst; -class LoadInst; -class StoreInst; -class AssumptionCache; -class DominatorTree; -class DomTreeUpdater; class Function; -class LLVMContext; -class PHINode; -class SelectInst; -class Use; - -/// A private "module" namespace for types and utilities used by SROA. These -/// are implementation details and should not be used by clients. -namespace LLVM_LIBRARY_VISIBILITY sroa { - -class AllocaSliceRewriter; -class AllocaSlices; -class Partition; -class SROALegacyPass; - -class SelectHandSpeculativity { - unsigned char Storage = 0; // None are speculatable by default. - using TrueVal = Bitfield::Element; // Low 0'th bit. - using FalseVal = Bitfield::Element; // Low 1'th bit. -public: - SelectHandSpeculativity() = default; - SelectHandSpeculativity &setAsSpeculatable(bool isTrueVal); - bool isSpeculatable(bool isTrueVal) const; - bool areAllSpeculatable() const; - bool areAnySpeculatable() const; - bool areNoneSpeculatable() const; - // For interop as int half of PointerIntPair. - explicit operator intptr_t() const { return static_cast(Storage); } - explicit SelectHandSpeculativity(intptr_t Storage_) : Storage(Storage_) {} -}; -static_assert(sizeof(SelectHandSpeculativity) == sizeof(unsigned char)); - -using PossiblySpeculatableLoad = - PointerIntPair; -using UnspeculatableStore = StoreInst *; -using RewriteableMemOp = - std::variant; -using RewriteableMemOps = SmallVector; - -} // end namespace sroa enum class SROAOptions : bool { ModifyCFG, PreserveCFG }; -/// An optimization pass providing Scalar Replacement of Aggregates. -/// -/// This pass takes allocations which can be completely analyzed (that is, they -/// don't escape) and tries to turn them into scalar SSA values. There are -/// a few steps to this process. -/// -/// 1) It takes allocations of aggregates and analyzes the ways in which they -/// are used to try to split them into smaller allocations, ideally of -/// a single scalar data type. It will split up memcpy and memset accesses -/// as necessary and try to isolate individual scalar accesses. -/// 2) It will transform accesses into forms which are suitable for SSA value -/// promotion. This can be replacing a memset with a scalar store of an -/// integer value, or it can involve speculating operations on a PHI or -/// select to be a PHI or select of the results. -/// 3) Finally, this will try to detect a pattern of accesses which map cleanly -/// onto insert and extract operations on a vector value, and convert them to -/// this form. By doing so, it will enable promotion of vector aggregates to -/// SSA vector values. class SROAPass : public PassInfoMixin { - LLVMContext *C = nullptr; - DomTreeUpdater *DTU = nullptr; - AssumptionCache *AC = nullptr; - const bool PreserveCFG; - - /// Worklist of alloca instructions to simplify. - /// - /// Each alloca in the function is added to this. Each new alloca formed gets - /// added to it as well to recursively simplify unless that alloca can be - /// directly promoted. Finally, each time we rewrite a use of an alloca other - /// the one being actively rewritten, we add it back onto the list if not - /// already present to ensure it is re-visited. - SmallSetVector Worklist; - - /// A collection of instructions to delete. - /// We try to batch deletions to simplify code and make things a bit more - /// efficient. We also make sure there is no dangling pointers. - SmallVector DeadInsts; - - /// Post-promotion worklist. - /// - /// Sometimes we discover an alloca which has a high probability of becoming - /// viable for SROA after a round of promotion takes place. In those cases, - /// the alloca is enqueued here for re-processing. - /// - /// Note that we have to be very careful to clear allocas out of this list in - /// the event they are deleted. - SmallSetVector PostPromotionWorklist; - - /// A collection of alloca instructions we can directly promote. - std::vector PromotableAllocas; - - /// A worklist of PHIs to speculate prior to promoting allocas. - /// - /// All of these PHIs have been checked for the safety of speculation and by - /// being speculated will allow promoting allocas currently in the promotable - /// queue. - SmallSetVector SpeculatablePHIs; - - /// A worklist of select instructions to rewrite prior to promoting - /// allocas. - SmallMapVector SelectsToRewrite; - - /// Select instructions that use an alloca and are subsequently loaded can be - /// rewritten to load both input pointers and then select between the result, - /// allowing the load of the alloca to be promoted. - /// From this: - /// %P2 = select i1 %cond, ptr %Alloca, ptr %Other - /// %V = load , ptr %P2 - /// to: - /// %V1 = load , ptr %Alloca -> will be mem2reg'd - /// %V2 = load , ptr %Other - /// %V = select i1 %cond, %V1, %V2 - /// - /// We can do this to a select if its only uses are loads - /// and if either the operand to the select can be loaded unconditionally, - /// or if we are allowed to perform CFG modifications. - /// If found an intervening bitcast with a single use of the load, - /// allow the promotion. - static std::optional - isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG); + const SROAOptions PreserveCFG; public: /// If \p PreserveCFG is set, then the pass is not allowed to modify CFG @@ -166,25 +36,6 @@ class SROAPass : public PassInfoMixin { void printPipeline(raw_ostream &OS, function_ref MapClassName2PassName); - -private: - friend class sroa::AllocaSliceRewriter; - friend class sroa::SROALegacyPass; - - /// Helper used by both the public run method and by the legacy pass. - PreservedAnalyses runImpl(Function &F, DomTreeUpdater &RunDTU, - AssumptionCache &RunAC); - PreservedAnalyses runImpl(Function &F, DominatorTree &RunDT, - AssumptionCache &RunAC); - - bool presplitLoadsAndStores(AllocaInst &AI, sroa::AllocaSlices &AS); - AllocaInst *rewritePartition(AllocaInst &AI, sroa::AllocaSlices &AS, - sroa::Partition &P); - bool splitAlloca(AllocaInst &AI, sroa::AllocaSlices &AS); - std::pair runOnAlloca(AllocaInst &AI); - void clobberUse(Use &U); - bool deleteDeadInstructions(SmallPtrSetImpl &DeletedAllocas); - bool promoteAllocas(Function &F); }; } // end namespace llvm diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index 095f12cd414b8..1344765444d29 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -26,6 +26,7 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" @@ -70,6 +71,7 @@ #include "llvm/IR/Use.h" #include "llvm/IR/User.h" #include "llvm/IR/Value.h" +#include "llvm/IR/ValueHandle.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" @@ -91,10 +93,10 @@ #include #include #include +#include #include using namespace llvm; -using namespace llvm::sroa; #define DEBUG_TYPE "sroa" @@ -123,6 +125,138 @@ static cl::opt SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(false), cl::Hidden); namespace { +class AllocaSliceRewriter; +class AllocaSlices; +class Partition; + +class SelectHandSpeculativity { + unsigned char Storage = 0; // None are speculatable by default. + using TrueVal = Bitfield::Element; // Low 0'th bit. + using FalseVal = Bitfield::Element; // Low 1'th bit. +public: + SelectHandSpeculativity() = default; + SelectHandSpeculativity &setAsSpeculatable(bool isTrueVal); + bool isSpeculatable(bool isTrueVal) const; + bool areAllSpeculatable() const; + bool areAnySpeculatable() const; + bool areNoneSpeculatable() const; + // For interop as int half of PointerIntPair. + explicit operator intptr_t() const { return static_cast(Storage); } + explicit SelectHandSpeculativity(intptr_t Storage_) : Storage(Storage_) {} +}; +static_assert(sizeof(SelectHandSpeculativity) == sizeof(unsigned char)); + +using PossiblySpeculatableLoad = + PointerIntPair; +using UnspeculatableStore = StoreInst *; +using RewriteableMemOp = + std::variant; +using RewriteableMemOps = SmallVector; + +/// An optimization pass providing Scalar Replacement of Aggregates. +/// +/// This pass takes allocations which can be completely analyzed (that is, they +/// don't escape) and tries to turn them into scalar SSA values. There are +/// a few steps to this process. +/// +/// 1) It takes allocations of aggregates and analyzes the ways in which they +/// are used to try to split them into smaller allocations, ideally of +/// a single scalar data type. It will split up memcpy and memset accesses +/// as necessary and try to isolate individual scalar accesses. +/// 2) It will transform accesses into forms which are suitable for SSA value +/// promotion. This can be replacing a memset with a scalar store of an +/// integer value, or it can involve speculating operations on a PHI or +/// select to be a PHI or select of the results. +/// 3) Finally, this will try to detect a pattern of accesses which map cleanly +/// onto insert and extract operations on a vector value, and convert them to +/// this form. By doing so, it will enable promotion of vector aggregates to +/// SSA vector values. +class SROA { + LLVMContext *const C; + DomTreeUpdater *const DTU; + AssumptionCache *const AC; + const bool PreserveCFG; + + /// Worklist of alloca instructions to simplify. + /// + /// Each alloca in the function is added to this. Each new alloca formed gets + /// added to it as well to recursively simplify unless that alloca can be + /// directly promoted. Finally, each time we rewrite a use of an alloca other + /// the one being actively rewritten, we add it back onto the list if not + /// already present to ensure it is re-visited. + SmallSetVector Worklist; + + /// A collection of instructions to delete. + /// We try to batch deletions to simplify code and make things a bit more + /// efficient. We also make sure there is no dangling pointers. + SmallVector DeadInsts; + + /// Post-promotion worklist. + /// + /// Sometimes we discover an alloca which has a high probability of becoming + /// viable for SROA after a round of promotion takes place. In those cases, + /// the alloca is enqueued here for re-processing. + /// + /// Note that we have to be very careful to clear allocas out of this list in + /// the event they are deleted. + SmallSetVector PostPromotionWorklist; + + /// A collection of alloca instructions we can directly promote. + std::vector PromotableAllocas; + + /// A worklist of PHIs to speculate prior to promoting allocas. + /// + /// All of these PHIs have been checked for the safety of speculation and by + /// being speculated will allow promoting allocas currently in the promotable + /// queue. + SmallSetVector SpeculatablePHIs; + + /// A worklist of select instructions to rewrite prior to promoting + /// allocas. + SmallMapVector SelectsToRewrite; + + /// Select instructions that use an alloca and are subsequently loaded can be + /// rewritten to load both input pointers and then select between the result, + /// allowing the load of the alloca to be promoted. + /// From this: + /// %P2 = select i1 %cond, ptr %Alloca, ptr %Other + /// %V = load , ptr %P2 + /// to: + /// %V1 = load , ptr %Alloca -> will be mem2reg'd + /// %V2 = load , ptr %Other + /// %V = select i1 %cond, %V1, %V2 + /// + /// We can do this to a select if its only uses are loads + /// and if either the operand to the select can be loaded unconditionally, + /// or if we are allowed to perform CFG modifications. + /// If found an intervening bitcast with a single use of the load, + /// allow the promotion. + static std::optional + isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG); + +public: + SROA(LLVMContext *C, DomTreeUpdater *DTU, AssumptionCache *AC, + SROAOptions PreserveCFG_) + : C(C), DTU(DTU), AC(AC), + PreserveCFG(PreserveCFG_ == SROAOptions::PreserveCFG) {} + + /// Main run method used by both the SROAPass and by the legacy pass. + std::pair runSROA(Function &F); + +private: + friend class AllocaSliceRewriter; + + bool presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS); + AllocaInst *rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P); + bool splitAlloca(AllocaInst &AI, AllocaSlices &AS); + std::pair runOnAlloca(AllocaInst &AI); + void clobberUse(Use &U); + bool deleteDeadInstructions(SmallPtrSetImpl &DeletedAllocas); + bool promoteAllocas(Function &F); +}; + +} // end anonymous namespace + /// Calculate the fragment of a variable to use when slicing a store /// based on the slice dimensions, existing fragment, and base storage /// fragment. @@ -131,7 +265,9 @@ namespace { /// UseNoFrag - The new slice already covers the whole variable. /// Skip - The new alloca slice doesn't include this variable. /// FIXME: Can we use calculateFragmentIntersect instead? +namespace { enum FragCalcResult { UseFrag, UseNoFrag, Skip }; +} static FragCalcResult calculateFragment(DILocalVariable *Variable, uint64_t NewStorageSliceOffsetInBits, @@ -330,6 +466,8 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, } } +namespace { + /// A custom IRBuilder inserter which prefixes all names, but only in /// Assert builds. class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter { @@ -422,8 +560,6 @@ class Slice { bool operator!=(const Slice &RHS) const { return !operator==(RHS); } }; -} // end anonymous namespace - /// Representation of the alloca slices. /// /// This class represents the slices of an alloca which are formed by its @@ -431,7 +567,7 @@ class Slice { /// for the slices used and we reflect that in this structure. The uses are /// stored, sorted by increasing beginning offset and with unsplittable slices /// starting at a particular offset before splittable slices. -class llvm::sroa::AllocaSlices { +class AllocaSlices { public: /// Construct the slices of a particular alloca. AllocaSlices(const DataLayout &DL, AllocaInst &AI); @@ -563,7 +699,7 @@ class llvm::sroa::AllocaSlices { /// /// Objects of this type are produced by traversing the alloca's slices, but /// are only ephemeral and not persistent. -class llvm::sroa::Partition { +class Partition { private: friend class AllocaSlices; friend class AllocaSlices::partition_iterator; @@ -628,6 +764,8 @@ class llvm::sroa::Partition { ArrayRef splitSliceTails() const { return SplitTails; } }; +} // end anonymous namespace + /// An iterator over partitions of the alloca's slices. /// /// This iterator implements the core algorithm for partitioning the alloca's @@ -1534,38 +1672,37 @@ static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN) { PN.eraseFromParent(); } -sroa::SelectHandSpeculativity & -sroa::SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) { +SelectHandSpeculativity & +SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) { if (isTrueVal) - Bitfield::set(Storage, true); + Bitfield::set(Storage, true); else - Bitfield::set(Storage, true); + Bitfield::set(Storage, true); return *this; } -bool sroa::SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const { - return isTrueVal - ? Bitfield::get(Storage) - : Bitfield::get(Storage); +bool SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const { + return isTrueVal ? Bitfield::get(Storage) + : Bitfield::get(Storage); } -bool sroa::SelectHandSpeculativity::areAllSpeculatable() const { +bool SelectHandSpeculativity::areAllSpeculatable() const { return isSpeculatable(/*isTrueVal=*/true) && isSpeculatable(/*isTrueVal=*/false); } -bool sroa::SelectHandSpeculativity::areAnySpeculatable() const { +bool SelectHandSpeculativity::areAnySpeculatable() const { return isSpeculatable(/*isTrueVal=*/true) || isSpeculatable(/*isTrueVal=*/false); } -bool sroa::SelectHandSpeculativity::areNoneSpeculatable() const { +bool SelectHandSpeculativity::areNoneSpeculatable() const { return !areAnySpeculatable(); } -static sroa::SelectHandSpeculativity +static SelectHandSpeculativity isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) { assert(LI.isSimple() && "Only for simple loads"); - sroa::SelectHandSpeculativity Spec; + SelectHandSpeculativity Spec; const DataLayout &DL = SI.getModule()->getDataLayout(); for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()}) @@ -1578,8 +1715,8 @@ isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) { return Spec; } -std::optional -SROAPass::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) { +std::optional +SROA::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) { RewriteableMemOps Ops; for (User *U : SI.users()) { @@ -1613,7 +1750,7 @@ SROAPass::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) { continue; } - sroa::SelectHandSpeculativity Spec = + SelectHandSpeculativity Spec = isSafeLoadOfSelectToSpeculate(*LI, SI, PreserveCFG); if (PreserveCFG && !Spec.areAllSpeculatable()) return {}; // Give up on this `select`. @@ -1664,7 +1801,7 @@ static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI, template static void rewriteMemOpOfSelect(SelectInst &SI, T &I, - sroa::SelectHandSpeculativity Spec, + SelectHandSpeculativity Spec, DomTreeUpdater &DTU) { assert((isa(I) || isa(I)) && "Only for load and store!"); LLVM_DEBUG(dbgs() << " original mem op: " << I << "\n"); @@ -1720,7 +1857,7 @@ static void rewriteMemOpOfSelect(SelectInst &SI, T &I, } static void rewriteMemOpOfSelect(SelectInst &SelInst, Instruction &I, - sroa::SelectHandSpeculativity Spec, + SelectHandSpeculativity Spec, DomTreeUpdater &DTU) { if (auto *LI = dyn_cast(&I)) rewriteMemOpOfSelect(SelInst, *LI, Spec, DTU); @@ -1731,13 +1868,13 @@ static void rewriteMemOpOfSelect(SelectInst &SelInst, Instruction &I, } static bool rewriteSelectInstMemOps(SelectInst &SI, - const sroa::RewriteableMemOps &Ops, + const RewriteableMemOps &Ops, IRBuilderTy &IRB, DomTreeUpdater *DTU) { bool CFGChanged = false; LLVM_DEBUG(dbgs() << " original select: " << SI << "\n"); for (const RewriteableMemOp &Op : Ops) { - sroa::SelectHandSpeculativity Spec; + SelectHandSpeculativity Spec; Instruction *I; if (auto *const *US = std::get_if(&Op)) { I = *US; @@ -2430,14 +2567,15 @@ static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V, return V; } +namespace { + /// Visitor to rewrite instructions using p particular slice of an alloca /// to use a new alloca. /// /// Also implements the rewriting to vector-based accesses when the partition /// passes the isVectorPromotionViable predicate. Most of the rewriting logic /// lives here. -class llvm::sroa::AllocaSliceRewriter - : public InstVisitor { +class AllocaSliceRewriter : public InstVisitor { // Befriend the base class so it can delegate to private visit methods. friend class InstVisitor; @@ -2445,7 +2583,7 @@ class llvm::sroa::AllocaSliceRewriter const DataLayout &DL; AllocaSlices &AS; - SROAPass &Pass; + SROA &Pass; AllocaInst &OldAI, &NewAI; const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset; Type *NewAllocaTy; @@ -2503,7 +2641,7 @@ class llvm::sroa::AllocaSliceRewriter } public: - AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROAPass &Pass, + AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass, AllocaInst &OldAI, AllocaInst &NewAI, uint64_t NewAllocaBeginOffset, uint64_t NewAllocaEndOffset, bool IsIntegerPromotable, @@ -3469,8 +3607,6 @@ class llvm::sroa::AllocaSliceRewriter } }; -namespace { - /// Visitor to rewrite aggregate loads and stores as scalar. /// /// This pass aggressively rewrites all aggregate loads and stores on @@ -4043,7 +4179,7 @@ static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset, /// there all along. /// /// \returns true if any changes are made. -bool SROAPass::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { +bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n"); // Track the loads and stores which are candidates for pre-splitting here, in @@ -4522,8 +4658,8 @@ bool SROAPass::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { /// appropriate new offsets. It also evaluates how successful the rewrite was /// at enabling promotion and if it was successful queues the alloca to be /// promoted. -AllocaInst *SROAPass::rewritePartition(AllocaInst &AI, AllocaSlices &AS, - Partition &P) { +AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, + Partition &P) { // Try to compute a friendly type for this partition of the alloca. This // won't always succeed, in which case we fall back to a legal integer type // or an i8 array of an appropriate size. @@ -4705,7 +4841,7 @@ AllocaInst *SROAPass::rewritePartition(AllocaInst &AI, AllocaSlices &AS, /// Walks the slices of an alloca and form partitions based on them, /// rewriting each of their uses. -bool SROAPass::splitAlloca(AllocaInst &AI, AllocaSlices &AS) { +bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) { if (AS.begin() == AS.end()) return false; @@ -4896,7 +5032,7 @@ bool SROAPass::splitAlloca(AllocaInst &AI, AllocaSlices &AS) { } /// Clobber a use with poison, deleting the used value if it becomes dead. -void SROAPass::clobberUse(Use &U) { +void SROA::clobberUse(Use &U) { Value *OldV = U; // Replace the use with an poison value. U = PoisonValue::get(OldV->getType()); @@ -4916,7 +5052,7 @@ void SROAPass::clobberUse(Use &U) { /// the slices of the alloca, and then hands it off to be split and /// rewritten as needed. std::pair -SROAPass::runOnAlloca(AllocaInst &AI) { +SROA::runOnAlloca(AllocaInst &AI) { bool Changed = false; bool CFGChanged = false; @@ -4998,7 +5134,7 @@ SROAPass::runOnAlloca(AllocaInst &AI) { /// /// We also record the alloca instructions deleted here so that they aren't /// subsequently handed to mem2reg to promote. -bool SROAPass::deleteDeadInstructions( +bool SROA::deleteDeadInstructions( SmallPtrSetImpl &DeletedAllocas) { bool Changed = false; while (!DeadInsts.empty()) { @@ -5039,7 +5175,7 @@ bool SROAPass::deleteDeadInstructions( /// This attempts to promote whatever allocas have been identified as viable in /// the PromotableAllocas list. If that list is empty, there is nothing to do. /// This function returns whether any promotion occurred. -bool SROAPass::promoteAllocas(Function &F) { +bool SROA::promoteAllocas(Function &F) { if (PromotableAllocas.empty()) return false; @@ -5056,12 +5192,8 @@ bool SROAPass::promoteAllocas(Function &F) { return true; } -PreservedAnalyses SROAPass::runImpl(Function &F, DomTreeUpdater &RunDTU, - AssumptionCache &RunAC) { +std::pair SROA::runSROA(Function &F) { LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n"); - C = &F.getContext(); - DTU = &RunDTU; - AC = &RunAC; const DataLayout &DL = F.getParent()->getDataLayout(); BasicBlock &EntryBB = F.getEntryBlock(); @@ -5112,56 +5244,50 @@ PreservedAnalyses SROAPass::runImpl(Function &F, DomTreeUpdater &RunDTU, assert((!CFGChanged || !PreserveCFG) && "Should not have modified the CFG when told to preserve it."); - if (!Changed) - return PreservedAnalyses::all(); - - if (isAssignmentTrackingEnabled(*F.getParent())) { + if (Changed && isAssignmentTrackingEnabled(*F.getParent())) { for (auto &BB : F) RemoveRedundantDbgInstrs(&BB); } - PreservedAnalyses PA; - if (!CFGChanged) - PA.preserveSet(); - PA.preserve(); - return PA; -} - -PreservedAnalyses SROAPass::runImpl(Function &F, DominatorTree &RunDT, - AssumptionCache &RunAC) { - DomTreeUpdater DTU(RunDT, DomTreeUpdater::UpdateStrategy::Lazy); - return runImpl(F, DTU, RunAC); + return {Changed, CFGChanged}; } PreservedAnalyses SROAPass::run(Function &F, FunctionAnalysisManager &AM) { DominatorTree &DT = AM.getResult(F); AssumptionCache &AC = AM.getResult(F); - return runImpl(F, DT, AC); + DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy); + auto [Changed, CFGChanged] = + SROA(&F.getContext(), &DTU, &AC, PreserveCFG).runSROA(F); + if (!Changed) + return PreservedAnalyses::all(); + PreservedAnalyses PA; + if (!CFGChanged) + PA.preserveSet(); + PA.preserve(); + return PA; } void SROAPass::printPipeline( raw_ostream &OS, function_ref MapClassName2PassName) { static_cast *>(this)->printPipeline( OS, MapClassName2PassName); - OS << (PreserveCFG ? "" : ""); + OS << (PreserveCFG == SROAOptions::PreserveCFG ? "" + : ""); } -SROAPass::SROAPass(SROAOptions PreserveCFG_) - : PreserveCFG(PreserveCFG_ == SROAOptions::PreserveCFG) {} +SROAPass::SROAPass(SROAOptions PreserveCFG) : PreserveCFG(PreserveCFG) {} + +namespace { /// A legacy pass for the legacy pass manager that wraps the \c SROA pass. -/// -/// This is in the llvm namespace purely to allow it to be a friend of the \c -/// SROA pass. -class llvm::sroa::SROALegacyPass : public FunctionPass { - /// The SROA implementation. - SROAPass Impl; +class SROALegacyPass : public FunctionPass { + SROAOptions PreserveCFG; public: static char ID; SROALegacyPass(SROAOptions PreserveCFG = SROAOptions::PreserveCFG) - : FunctionPass(ID), Impl(PreserveCFG) { + : FunctionPass(ID), PreserveCFG(PreserveCFG) { initializeSROALegacyPassPass(*PassRegistry::getPassRegistry()); } @@ -5169,10 +5295,13 @@ class llvm::sroa::SROALegacyPass : public FunctionPass { if (skipFunction(F)) return false; - auto PA = Impl.runImpl( - F, getAnalysis().getDomTree(), - getAnalysis().getAssumptionCache(F)); - return !PA.areAllPreserved(); + DominatorTree &DT = getAnalysis().getDomTree(); + AssumptionCache &AC = + getAnalysis().getAssumptionCache(F); + DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy); + auto [Changed, _] = + SROA(&F.getContext(), &DTU, &AC, PreserveCFG).runSROA(F); + return Changed; } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -5185,6 +5314,8 @@ class llvm::sroa::SROALegacyPass : public FunctionPass { StringRef getPassName() const override { return "SROA"; } }; +} // end anonymous namespace + char SROALegacyPass::ID = 0; FunctionPass *llvm::createSROAPass(bool PreserveCFG) {