@@ -5907,6 +5907,90 @@ const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
59075907 return nullptr;
59085908}
59095909
5910+ // Checks if the SCEV S is available at BB. S is considered available at BB
5911+ // if S can be materialized at BB without introducing a fault.
5912+ static bool IsAvailableOnEntry(const Loop *L, DominatorTree &DT, const SCEV *S,
5913+ BasicBlock *BB) {
5914+ struct CheckAvailable {
5915+ bool TraversalDone = false;
5916+ bool Available = true;
5917+
5918+ const Loop *L = nullptr; // The loop BB is in (can be nullptr)
5919+ BasicBlock *BB = nullptr;
5920+ DominatorTree &DT;
5921+
5922+ CheckAvailable(const Loop *L, BasicBlock *BB, DominatorTree &DT)
5923+ : L(L), BB(BB), DT(DT) {}
5924+
5925+ bool setUnavailable() {
5926+ TraversalDone = true;
5927+ Available = false;
5928+ return false;
5929+ }
5930+
5931+ bool follow(const SCEV *S) {
5932+ switch (S->getSCEVType()) {
5933+ case scConstant:
5934+ case scVScale:
5935+ case scPtrToInt:
5936+ case scTruncate:
5937+ case scZeroExtend:
5938+ case scSignExtend:
5939+ case scAddExpr:
5940+ case scMulExpr:
5941+ case scUMaxExpr:
5942+ case scSMaxExpr:
5943+ case scUMinExpr:
5944+ case scSMinExpr:
5945+ case scSequentialUMinExpr:
5946+ // These expressions are available if their operand(s) is/are.
5947+ return true;
5948+
5949+ case scAddRecExpr: {
5950+ // We allow add recurrences that are on the loop BB is in, or some
5951+ // outer loop. This guarantees availability because the value of the
5952+ // add recurrence at BB is simply the "current" value of the induction
5953+ // variable. We can relax this in the future; for instance an add
5954+ // recurrence on a sibling dominating loop is also available at BB.
5955+ const auto *ARLoop = cast<SCEVAddRecExpr>(S)->getLoop();
5956+ if (L && (ARLoop == L || ARLoop->contains(L)))
5957+ return true;
5958+
5959+ return setUnavailable();
5960+ }
5961+
5962+ case scUnknown: {
5963+ // For SCEVUnknown, we check for simple dominance.
5964+ const auto *SU = cast<SCEVUnknown>(S);
5965+ Value *V = SU->getValue();
5966+
5967+ if (isa<Argument>(V))
5968+ return false;
5969+
5970+ if (isa<Instruction>(V) && DT.dominates(cast<Instruction>(V), BB))
5971+ return false;
5972+
5973+ return setUnavailable();
5974+ }
5975+
5976+ case scUDivExpr:
5977+ case scCouldNotCompute:
5978+ // We do not try to smart about these at all.
5979+ return setUnavailable();
5980+ }
5981+ llvm_unreachable("Unknown SCEV kind!");
5982+ }
5983+
5984+ bool isDone() { return TraversalDone; }
5985+ };
5986+
5987+ CheckAvailable CA(L, BB, DT);
5988+ SCEVTraversal<CheckAvailable> ST(CA);
5989+
5990+ ST.visitAll(S);
5991+ return CA.Available;
5992+ }
5993+
59105994// Try to match a control flow sequence that branches out at BI and merges back
59115995// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
59125996// match.
@@ -5944,6 +6028,8 @@ const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
59446028 auto IsReachable =
59456029 [&](BasicBlock *BB) { return DT.isReachableFromEntry(BB); };
59466030 if (PN->getNumIncomingValues() == 2 && all_of(PN->blocks(), IsReachable)) {
6031+ const Loop *L = LI.getLoopFor(PN->getParent());
6032+
59476033 // Try to match
59486034 //
59496035 // br %cond, label %left, label %right
@@ -5964,8 +6050,8 @@ const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
59646050
59656051 if (BI && BI->isConditional() &&
59666052 BrPHIToSelect(DT, BI, PN, Cond, LHS, RHS) &&
5967- properlyDominates( getSCEV(LHS), PN->getParent()) &&
5968- properlyDominates( getSCEV(RHS), PN->getParent()))
6053+ IsAvailableOnEntry(L, DT, getSCEV(LHS), PN->getParent()) &&
6054+ IsAvailableOnEntry(L, DT, getSCEV(RHS), PN->getParent()))
59696055 return createNodeForSelectOrPHI(PN, Cond, LHS, RHS);
59706056 }
59716057
0 commit comments