diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index 885c5985f9d23..756fd0eeeef74 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -1112,6 +1112,46 @@ class ScalarEvolution { bool isKnownOnEveryIteration(ICmpInst::Predicate Pred, const SCEVAddRecExpr *LHS, const SCEV *RHS); + class LoopGuards { + DenseMap RewriteMap; + bool PreserveNUW = false; + bool PreserveNSW = false; + ScalarEvolution &SE; + + LoopGuards(ScalarEvolution &SE) : SE(SE) {} + + /// Recursively collect loop guards in \p Guards, starting from + /// block \p Block with predecessor \p Pred. The intended starting point + /// is to collect from a loop header and its predecessor. + static void + collectFromBlock(ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards, + const BasicBlock *Block, const BasicBlock *Pred, + SmallPtrSetImpl &VisitedBlocks, + unsigned Depth = 0); + + /// Collect loop guards in \p Guards, starting from PHINode \p + /// Phi, by calling \p collectFromBlock on the incoming blocks of + /// \Phi and trying to merge the found constraints into a single + /// combined one for \p Phi. + static void collectFromPHI( + ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards, + const PHINode &Phi, SmallPtrSetImpl &VisitedBlocks, + SmallDenseMap &IncomingGuards, + unsigned Depth); + + public: + /// Collect rewrite map for loop guards for loop \p L, together with flags + /// indicating if NUW and NSW can be preserved during rewriting. + static LoopGuards collect(const Loop *L, ScalarEvolution &SE); + + /// Try to apply the collected loop guards to \p Expr. + const SCEV *rewrite(const SCEV *Expr) const; + }; + + /// Try to apply information from loop guards for \p L to \p Expr. + const SCEV *applyLoopGuards(const SCEV *Expr, const Loop *L); + const SCEV *applyLoopGuards(const SCEV *Expr, const LoopGuards &Guards); + /// Information about the number of loop iterations for which a loop exit's /// branch condition evaluates to the not-taken path. This is a temporary /// pair of exact and max expressions that are eventually summarized in @@ -1166,9 +1206,11 @@ class ScalarEvolution { /// /// If \p AllowPredicates is set, this call will try to use a minimal set of /// SCEV predicates in order to return an exact answer. - ExitLimit computeExitLimitFromCond(const Loop *L, Value *ExitCond, - bool ExitIfTrue, bool ControlsOnlyExit, - bool AllowPredicates = false); + ExitLimit + computeExitLimitFromCond(const Loop *L, Value *ExitCond, + function_ref GetLoopGuards, + bool ExitIfTrue, bool ControlsOnlyExit, + bool AllowPredicates = false); /// A predicate is said to be monotonically increasing if may go from being /// false to being true as the loop iterates, but never the other way @@ -1308,46 +1350,6 @@ class ScalarEvolution { /// sharpen it. void setNoWrapFlags(SCEVAddRecExpr *AddRec, SCEV::NoWrapFlags Flags); - class LoopGuards { - DenseMap RewriteMap; - bool PreserveNUW = false; - bool PreserveNSW = false; - ScalarEvolution &SE; - - LoopGuards(ScalarEvolution &SE) : SE(SE) {} - - /// Recursively collect loop guards in \p Guards, starting from - /// block \p Block with predecessor \p Pred. The intended starting point - /// is to collect from a loop header and its predecessor. - static void - collectFromBlock(ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards, - const BasicBlock *Block, const BasicBlock *Pred, - SmallPtrSetImpl &VisitedBlocks, - unsigned Depth = 0); - - /// Collect loop guards in \p Guards, starting from PHINode \p - /// Phi, by calling \p collectFromBlock on the incoming blocks of - /// \Phi and trying to merge the found constraints into a single - /// combined one for \p Phi. - static void collectFromPHI( - ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards, - const PHINode &Phi, SmallPtrSetImpl &VisitedBlocks, - SmallDenseMap &IncomingGuards, - unsigned Depth); - - public: - /// Collect rewrite map for loop guards for loop \p L, together with flags - /// indicating if NUW and NSW can be preserved during rewriting. - static LoopGuards collect(const Loop *L, ScalarEvolution &SE); - - /// Try to apply the collected loop guards to \p Expr. - const SCEV *rewrite(const SCEV *Expr) const; - }; - - /// Try to apply information from loop guards for \p L to \p Expr. - const SCEV *applyLoopGuards(const SCEV *Expr, const Loop *L); - const SCEV *applyLoopGuards(const SCEV *Expr, const LoopGuards &Guards); - /// Return true if the loop has no abnormal exits. That is, if the loop /// is not infinite, it must exit through an explicit edge in the CFG. /// (As opposed to either a) throwing out of the function or b) entering a @@ -1840,6 +1842,7 @@ class ScalarEvolution { /// this call will try to use a minimal set of SCEV predicates in order to /// return an exact answer. ExitLimit computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, + function_ref GetLoopGuards, bool IsOnlyExit, bool AllowPredicates = false); // Helper functions for computeExitLimitFromCond to avoid exponential time @@ -1872,17 +1875,17 @@ class ScalarEvolution { using ExitLimitCacheTy = ExitLimitCache; - ExitLimit computeExitLimitFromCondCached(ExitLimitCacheTy &Cache, - const Loop *L, Value *ExitCond, - bool ExitIfTrue, - bool ControlsOnlyExit, - bool AllowPredicates); - ExitLimit computeExitLimitFromCondImpl(ExitLimitCacheTy &Cache, const Loop *L, - Value *ExitCond, bool ExitIfTrue, - bool ControlsOnlyExit, - bool AllowPredicates); + ExitLimit computeExitLimitFromCondCached( + ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, + function_ref GetLoopGuards, bool ExitIfTrue, + bool ControlsOnlyExit, bool AllowPredicates); + ExitLimit computeExitLimitFromCondImpl( + ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, + function_ref GetLoopGuards, bool ExitIfTrue, + bool ControlsOnlyExit, bool AllowPredicates); std::optional computeExitLimitFromCondFromBinOp( - ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue, + ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, + function_ref GetLoopGuards, bool ExitIfTrue, bool ControlsOnlyExit, bool AllowPredicates); /// Compute the number of times the backedge of the specified loop will @@ -1890,27 +1893,28 @@ class ScalarEvolution { /// ExitCond and ExitIfTrue. If AllowPredicates is set, this call will try /// to use a minimal set of SCEV predicates in order to return an exact /// answer. - ExitLimit computeExitLimitFromICmp(const Loop *L, ICmpInst *ExitCond, - bool ExitIfTrue, - bool IsSubExpr, - bool AllowPredicates = false); + ExitLimit + computeExitLimitFromICmp(const Loop *L, ICmpInst *ExitCond, + function_ref GetLoopGuards, + bool ExitIfTrue, bool IsSubExpr, + bool AllowPredicates = false); /// Variant of previous which takes the components representing an ICmp /// as opposed to the ICmpInst itself. Note that the prior version can /// return more precise results in some cases and is preferred when caller /// has a materialized ICmp. - ExitLimit computeExitLimitFromICmp(const Loop *L, ICmpInst::Predicate Pred, - const SCEV *LHS, const SCEV *RHS, - bool IsSubExpr, - bool AllowPredicates = false); + ExitLimit + computeExitLimitFromICmp(const Loop *L, ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS, + function_ref GetLoopGuards, + bool IsSubExpr, bool AllowPredicates = false); /// Compute the number of times the backedge of the specified loop will /// execute if its exit condition were a switch with a single exiting case /// to ExitingBB. - ExitLimit computeExitLimitFromSingleExitSwitch(const Loop *L, - SwitchInst *Switch, - BasicBlock *ExitingBB, - bool IsSubExpr); + ExitLimit computeExitLimitFromSingleExitSwitch( + const Loop *L, SwitchInst *Switch, BasicBlock *ExitingBB, + function_ref GetLoopGuards, bool IsSubExpr); /// Compute the exit limit of a loop that is controlled by a /// "(IV >> 1) != 0" type comparison. We cannot compute the exact trip @@ -1934,8 +1938,9 @@ class ScalarEvolution { /// value to zero will execute. If not computable, return CouldNotCompute. /// If AllowPredicates is set, this call will try to use a minimal set of /// SCEV predicates in order to return an exact answer. - ExitLimit howFarToZero(const SCEV *V, const Loop *L, bool IsSubExpr, - bool AllowPredicates = false); + ExitLimit howFarToZero(const SCEV *V, const Loop *L, + function_ref GetLoopGuards, + bool IsSubExpr, bool AllowPredicates = false); /// Return the number of times an exit condition checking the specified /// value for nonzero will execute. If not computable, return @@ -1955,7 +1960,9 @@ class ScalarEvolution { /// If \p AllowPredicates is set, this call will try to use a minimal set of /// SCEV predicates in order to return an exact answer. ExitLimit howManyLessThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, - bool isSigned, bool ControlsOnlyExit, + bool isSigned, + function_ref GetLoopGuards, + bool ControlsOnlyExit, bool AllowPredicates = false); ExitLimit howManyGreaterThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 46b108606f6a6..51ac06121c9c1 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -8806,6 +8806,12 @@ ScalarEvolution::computeBackedgeTakenCount(const Loop *L, const SCEV *MayExitMaxBECount = nullptr; bool MustExitMaxOrZero = false; bool IsOnlyExit = ExitingBlocks.size() == 1; + std::optional CachedLoopGuards; + auto GetLoopGuards = [&CachedLoopGuards, &L, this]() -> const LoopGuards & { + if (!CachedLoopGuards) + CachedLoopGuards.emplace(LoopGuards::collect(L, *this)); + return *CachedLoopGuards; + }; // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts // and compute maxBECount. @@ -8821,7 +8827,8 @@ ScalarEvolution::computeBackedgeTakenCount(const Loop *L, continue; } - ExitLimit EL = computeExitLimit(L, ExitBB, IsOnlyExit, AllowPredicates); + ExitLimit EL = + computeExitLimit(L, ExitBB, GetLoopGuards, IsOnlyExit, AllowPredicates); assert((AllowPredicates || EL.Predicates.empty()) && "Predicated exit limit when predicates are not allowed!"); @@ -8894,9 +8901,10 @@ ScalarEvolution::computeBackedgeTakenCount(const Loop *L, MaxBECount, MaxOrZero); } -ScalarEvolution::ExitLimit -ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, - bool IsOnlyExit, bool AllowPredicates) { +ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimit( + const Loop *L, BasicBlock *ExitingBlock, + function_ref GetLoopGuards, bool IsOnlyExit, + bool AllowPredicates) { assert(L->contains(ExitingBlock) && "Exit count for non-loop block?"); // If our exiting block does not dominate the latch, then its connection with // loop's exit limit may be far from trivial. @@ -8911,9 +8919,9 @@ ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) && "It should have one successor in loop and one exit block!"); // Proceed to the next level to examine the exit condition expression. - return computeExitLimitFromCond(L, BI->getCondition(), ExitIfTrue, - /*ControlsOnlyExit=*/IsOnlyExit, - AllowPredicates); + return computeExitLimitFromCond( + L, BI->getCondition(), GetLoopGuards, ExitIfTrue, + /*ControlsOnlyExit=*/IsOnlyExit, AllowPredicates); } if (SwitchInst *SI = dyn_cast(Term)) { @@ -8927,18 +8935,20 @@ ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, } assert(Exit && "Exiting block must have at least one exit"); return computeExitLimitFromSingleExitSwitch( - L, SI, Exit, /*ControlsOnlyExit=*/IsOnlyExit); + L, SI, Exit, GetLoopGuards, /*ControlsOnlyExit=*/IsOnlyExit); } return getCouldNotCompute(); } ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCond( - const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit, - bool AllowPredicates) { + const Loop *L, Value *ExitCond, + function_ref GetLoopGuards, bool ExitIfTrue, + bool ControlsOnlyExit, bool AllowPredicates) { ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates); - return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue, - ControlsOnlyExit, AllowPredicates); + return computeExitLimitFromCondCached(Cache, L, ExitCond, GetLoopGuards, + ExitIfTrue, ControlsOnlyExit, + AllowPredicates); } std::optional @@ -8974,37 +8984,41 @@ void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond, } ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached( - ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue, + ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, + function_ref GetLoopGuards, bool ExitIfTrue, bool ControlsOnlyExit, bool AllowPredicates) { if (auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates)) return *MaybeEL; - ExitLimit EL = computeExitLimitFromCondImpl( - Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates); + ExitLimit EL = computeExitLimitFromCondImpl(Cache, L, ExitCond, GetLoopGuards, + ExitIfTrue, ControlsOnlyExit, + AllowPredicates); Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL); return EL; } ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl( - ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue, + ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, + function_ref GetLoopGuards, bool ExitIfTrue, bool ControlsOnlyExit, bool AllowPredicates) { // Handle BinOp conditions (And, Or). if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp( - Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates)) + Cache, L, ExitCond, GetLoopGuards, ExitIfTrue, ControlsOnlyExit, + AllowPredicates)) return *LimitFromBinOp; // With an icmp, it may be feasible to compute an exact backedge-taken count. // Proceed to the next level to examine the icmp. if (ICmpInst *ExitCondICmp = dyn_cast(ExitCond)) { - ExitLimit EL = - computeExitLimitFromICmp(L, ExitCondICmp, ExitIfTrue, ControlsOnlyExit); + ExitLimit EL = computeExitLimitFromICmp(L, ExitCondICmp, GetLoopGuards, + ExitIfTrue, ControlsOnlyExit); if (EL.hasFullInfo() || !AllowPredicates) return EL; // Try again, but use SCEV predicates this time. - return computeExitLimitFromICmp(L, ExitCondICmp, ExitIfTrue, + return computeExitLimitFromICmp(L, ExitCondICmp, GetLoopGuards, ExitIfTrue, ControlsOnlyExit, /*AllowPredicates=*/true); } @@ -9040,7 +9054,8 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl( if (Offset != 0) LHS = getAddExpr(LHS, getConstant(Offset)); auto EL = computeExitLimitFromICmp(L, Pred, LHS, getConstant(NewRHSC), - ControlsOnlyExit, AllowPredicates); + GetLoopGuards, ControlsOnlyExit, + AllowPredicates); if (EL.hasAnyInfo()) return EL; } @@ -9051,7 +9066,8 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl( std::optional ScalarEvolution::computeExitLimitFromCondFromBinOp( - ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue, + ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, + function_ref GetLoopGuards, bool ExitIfTrue, bool ControlsOnlyExit, bool AllowPredicates) { // Check if the controlling expression for this loop is an And or Or. Value *Op0, *Op1; @@ -9068,11 +9084,11 @@ ScalarEvolution::computeExitLimitFromCondFromBinOp( // br (or Op0 Op1), exit, loop bool EitherMayExit = IsAnd ^ ExitIfTrue; ExitLimit EL0 = computeExitLimitFromCondCached( - Cache, L, Op0, ExitIfTrue, ControlsOnlyExit && !EitherMayExit, - AllowPredicates); + Cache, L, Op0, GetLoopGuards, ExitIfTrue, + ControlsOnlyExit && !EitherMayExit, AllowPredicates); ExitLimit EL1 = computeExitLimitFromCondCached( - Cache, L, Op1, ExitIfTrue, ControlsOnlyExit && !EitherMayExit, - AllowPredicates); + Cache, L, Op1, GetLoopGuards, ExitIfTrue, + ControlsOnlyExit && !EitherMayExit, AllowPredicates); // Be robust against unsimplified IR for the form "op i1 X, NeutralElement" const Constant *NeutralElement = ConstantInt::get(ExitCond->getType(), IsAnd); @@ -9131,8 +9147,9 @@ ScalarEvolution::computeExitLimitFromCondFromBinOp( } ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp( - const Loop *L, ICmpInst *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit, - bool AllowPredicates) { + const Loop *L, ICmpInst *ExitCond, + function_ref GetLoopGuards, bool ExitIfTrue, + bool ControlsOnlyExit, bool AllowPredicates) { // If the condition was exit on true, convert the condition to exit on false ICmpInst::Predicate Pred; if (!ExitIfTrue) @@ -9144,8 +9161,8 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp( const SCEV *LHS = getSCEV(ExitCond->getOperand(0)); const SCEV *RHS = getSCEV(ExitCond->getOperand(1)); - ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, ControlsOnlyExit, - AllowPredicates); + ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, GetLoopGuards, + ControlsOnlyExit, AllowPredicates); if (EL.hasAnyInfo()) return EL; @@ -9160,7 +9177,8 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp( } ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp( const Loop *L, ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, - bool ControlsOnlyExit, bool AllowPredicates) { + function_ref GetLoopGuards, bool ControlsOnlyExit, + bool AllowPredicates) { // Try to evaluate any dependencies out of the loop. LHS = getSCEVAtScope(LHS, L); @@ -9248,8 +9266,8 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp( if (isa(RHS)) return RHS; } - ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsOnlyExit, - AllowPredicates); + ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, GetLoopGuards, + ControlsOnlyExit, AllowPredicates); if (EL.hasAnyInfo()) return EL; break; @@ -9299,8 +9317,8 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp( case ICmpInst::ICMP_SLT: case ICmpInst::ICMP_ULT: { // while (X < Y) bool IsSigned = ICmpInst::isSigned(Pred); - ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, ControlsOnlyExit, - AllowPredicates); + ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, GetLoopGuards, + ControlsOnlyExit, AllowPredicates); if (EL.hasAnyInfo()) return EL; break; @@ -9331,10 +9349,9 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp( } ScalarEvolution::ExitLimit -ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L, - SwitchInst *Switch, - BasicBlock *ExitingBlock, - bool ControlsOnlyExit) { +ScalarEvolution::computeExitLimitFromSingleExitSwitch( + const Loop *L, SwitchInst *Switch, BasicBlock *ExitingBlock, + function_ref GetLoopGuards, bool ControlsOnlyExit) { assert(!L->contains(ExitingBlock) && "Not an exiting block!"); // Give up if the exit is the default dest of a switch. @@ -9347,7 +9364,8 @@ ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L, const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock)); // while (X != Y) --> while (X-Y != 0) - ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsOnlyExit); + ExitLimit EL = + howFarToZero(getMinusSCEV(LHS, RHS), L, GetLoopGuards, ControlsOnlyExit); if (EL.hasAnyInfo()) return EL; @@ -10485,10 +10503,10 @@ SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec, return TruncIfPossible(MinOptional(SL.first, SU.first), BitWidth); } -ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V, - const Loop *L, - bool ControlsOnlyExit, - bool AllowPredicates) { +ScalarEvolution::ExitLimit +ScalarEvolution::howFarToZero(const SCEV *V, const Loop *L, + function_ref GetLoopGuards, + bool ControlsOnlyExit, bool AllowPredicates) { // This is only used for loops with a "x != y" exit test. The exit condition // is now expressed as a single expression, V = x-y. So the exit test is @@ -10551,7 +10569,7 @@ ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V, if (!isLoopInvariant(Step, L)) return getCouldNotCompute(); - LoopGuards Guards = LoopGuards::collect(L, *this); + const LoopGuards &Guards = GetLoopGuards(); // Specialize step for this loop so we get context sensitive facts below. const SCEV *StepWLG = applyLoopGuards(Step, Guards); @@ -12910,10 +12928,10 @@ const SCEV *ScalarEvolution::computeMaxBECountForLT(const SCEV *Start, getConstant(StrideForMaxBECount) /* Step */); } -ScalarEvolution::ExitLimit -ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS, - const Loop *L, bool IsSigned, - bool ControlsOnlyExit, bool AllowPredicates) { +ScalarEvolution::ExitLimit ScalarEvolution::howManyLessThans( + const SCEV *LHS, const SCEV *RHS, const Loop *L, bool IsSigned, + function_ref GetLoopGuards, bool ControlsOnlyExit, + bool AllowPredicates) { SmallVector Predicates; const SCEVAddRecExpr *IV = dyn_cast(LHS); @@ -12947,7 +12965,8 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS, APInt StrideMax = getUnsignedRangeMax(AR->getStepRecurrence(*this)); APInt Limit = APInt::getMaxValue(InnerBitWidth) - (StrideMax - 1); Limit = Limit.zext(OuterBitWidth); - return getUnsignedRangeMax(applyLoopGuards(RHS, L)).ule(Limit); + return getUnsignedRangeMax(applyLoopGuards(RHS, GetLoopGuards())) + .ule(Limit); }; auto Flags = AR->getNoWrapFlags(); if (!hasFlags(Flags, SCEV::FlagNUW) && canProveNUW()) @@ -13198,8 +13217,8 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS, if (!BECount) { auto canProveRHSGreaterThanEqualStart = [&]() { auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; - const SCEV *GuardedRHS = applyLoopGuards(OrigRHS, L); - const SCEV *GuardedStart = applyLoopGuards(OrigStart, L); + const SCEV *GuardedRHS = applyLoopGuards(OrigRHS, GetLoopGuards()); + const SCEV *GuardedStart = applyLoopGuards(OrigStart, GetLoopGuards()); if (isLoopEntryGuardedByCond(L, CondGE, OrigRHS, OrigStart) || isKnownPredicate(CondGE, GuardedRHS, GuardedStart)) diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp index 8a3e0bc3eb971..1cf5aca2266c6 100644 --- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp +++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp @@ -1335,6 +1335,14 @@ static bool optimizeLoopExitWithUnknownExitCount( Visited.insert(OldCond); Worklist.push_back(OldCond); + std::optional CachedLoopGuards; + auto GetLoopGuards = [&CachedLoopGuards, &L, + &SE]() -> const ScalarEvolution::LoopGuards & { + if (!CachedLoopGuards) + CachedLoopGuards.emplace(ScalarEvolution::LoopGuards::collect(L, *SE)); + return *CachedLoopGuards; + }; + auto GoThrough = [&](Value *V) { Value *LHS = nullptr, *RHS = nullptr; if (Inverted) { @@ -1371,7 +1379,7 @@ static bool optimizeLoopExitWithUnknownExitCount( ScalarEvolution::ExitCountKind::SymbolicMaximum) == MaxIter) for (auto *ICmp : LeafConditions) { - auto EL = SE->computeExitLimitFromCond(L, ICmp, Inverted, + auto EL = SE->computeExitLimitFromCond(L, ICmp, GetLoopGuards, Inverted, /*ControlsExit*/ false); const SCEV *ExitMax = EL.SymbolicMaxNotTaken; if (isa(ExitMax))