Skip to content

Commit 2bc99bf

Browse files
artagnonartagnon
committed
LICM: hoist BO assoc when BinOp is in RHS
Extend hoistBOAssociation smoothly to handle the case when the inner BinaryOperator is in the RHS of the outer BinaryOperator. This completes the generalization of hoistBOAssociation, and the only limitation after this patch is the fact that only Add and Mul are hoisted.
1 parent 55a2473 commit 2bc99bf

File tree

2 files changed

+129
-15
lines changed

2 files changed

+129
-15
lines changed

‎llvm/lib/Transforms/Scalar/LICM.cpp

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2805,13 +2805,11 @@ static bool hoistMulAddAssociation(Instruction &I, Loop &L,
28052805
///
28062806
/// 1. "(LV op C1) op C2" ==> "LV op (C1 op C2)" if op is an associative BinOp
28072807
/// 2. "(C1 op LV) op C2" ==> "LV op (C1 op C2)" if op is a commutative BinOp
2808+
/// 3. "C2 op (C1 op LV)" ==> "(C2 op C1) op LV" if op an associative BinOp
2809+
/// 4. "C2 op (LV op C1)" ==> "(C2 op C1) op LV" if op is a commutative BinOp
28082810
///
28092811
/// where LV is a loop variant, and C1 and C2 are loop invariants that we want
28102812
/// to hoist.
2811-
///
2812-
/// TODO: This can be extended to more cases such as
2813-
/// 1. "C1 op (C2 op LV)" ==> "(C1 op C2) op LV" if op an associative BinOp
2814-
/// 2. "C1 op (LV op C2)" ==> "(C1 op C2) op LV" if op is a commutative BinOp
28152813
static bool hoistBOAssociation(Instruction &I, Loop &L,
28162814
ICFLoopSafetyInfo &SafetyInfo,
28172815
MemorySSAUpdater &MSSAU, AssumptionCache *AC,
@@ -2825,30 +2823,45 @@ static bool hoistBOAssociation(Instruction &I, Loop &L,
28252823
if (Opcode != Instruction::Add && Opcode != Instruction::Mul)
28262824
return false;
28272825

2828-
auto *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(0));
2826+
bool BinOpInRHS = isa<BinaryOperator>(BO->getOperand(1));
2827+
auto *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(BinOpInRHS));
28292828
if (!BO0 || BO0->getOpcode() != Opcode || !BO0->isAssociative() ||
28302829
BO0->hasNUsesOrMore(3))
28312830
return false;
28322831

28332832
Value *LV = BO0->getOperand(0);
28342833
Value *C1 = BO0->getOperand(1);
2835-
Value *C2 = BO->getOperand(1);
2834+
Value *C2 = BO->getOperand(!BinOpInRHS);
28362835

2837-
if (L.isLoopInvariant(LV) && !L.isLoopInvariant(C1)) {
2838-
assert(BO0->isCommutative() && "Associativity implies commutativity");
2836+
if (!L.isLoopInvariant(C2))
2837+
return false;
2838+
if (!L.isLoopInvariant(LV) && L.isLoopInvariant(C1)) {
2839+
if (BinOpInRHS)
2840+
assert(BO0->isCommutative() && "Associativity implies commutativity");
2841+
} else if (L.isLoopInvariant(LV) && !L.isLoopInvariant(C1)) {
2842+
if (!BinOpInRHS)
2843+
assert(BO0->isCommutative() && "Associativity implies commutativity");
28392844
std::swap(LV, C1);
2840-
}
2841-
if (L.isLoopInvariant(LV) || !L.isLoopInvariant(C1) || !L.isLoopInvariant(C2))
2845+
} else {
28422846
return false;
2847+
}
28432848

28442849
auto *Preheader = L.getLoopPreheader();
28452850
assert(Preheader && "Loop is not in simplify form?");
28462851

2852+
// To create C2 op C1, instead of C1 op C2.
2853+
if (BinOpInRHS)
2854+
std::swap(C1, C2);
2855+
28472856
IRBuilder<> Builder(Preheader->getTerminator());
28482857
auto *Inv = Builder.CreateBinOp(Opcode, C1, C2, "invariant.op");
28492858

2850-
auto *NewBO = BinaryOperator::Create(
2851-
Opcode, LV, Inv, BO->getName() + ".reass", BO->getIterator());
2859+
auto *NewBO =
2860+
BinOpInRHS
2861+
? BinaryOperator::Create(Opcode, Inv, LV, BO->getName() + ".reass",
2862+
BO->getIterator())
2863+
: BinaryOperator::Create(Opcode, LV, Inv, BO->getName() + ".reass",
2864+
BO->getIterator());
28522865

28532866
// Copy NUW for ADDs if both instructions have it.
28542867
if (Opcode == Instruction::Add && BO->hasNoUnsignedWrap() &&

‎llvm/test/Transforms/LICM/hoist-binop.ll

Lines changed: 104 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ loop:
6767
br label %loop
6868
}
6969

70-
71-
; Hoist ADD and copy NUW if both ops have it. Commutative version.
70+
; Hoist ADD and copy NUW if both ops have it.
71+
; Version where operands are commuted.
7272
define void @add_nuw_comm(i64 %c1, i64 %c2) {
7373
; CHECK-LABEL: @add_nuw_comm(
7474
; CHECK-NEXT: entry:
@@ -92,6 +92,56 @@ loop:
9292
br label %loop
9393
}
9494

95+
; Hoist ADD and copy NUW if both ops have it.
96+
; Another version where operands are commuted.
97+
define void @add_nuw_comm2(i64 %c1, i64 %c2) {
98+
; CHECK-LABEL: @add_nuw_comm2(
99+
; CHECK-NEXT: entry:
100+
; CHECK-NEXT: [[INVARIANT_OP:%.*]] = add nuw i64 [[C2:%.*]], [[C1:%.*]]
101+
; CHECK-NEXT: br label [[LOOP:%.*]]
102+
; CHECK: loop:
103+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
104+
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw i64 [[INDEX]], [[C1]]
105+
; CHECK-NEXT: call void @use(i64 [[STEP_ADD]])
106+
; CHECK-NEXT: [[INDEX_NEXT_REASS]] = add nuw i64 [[INVARIANT_OP]], [[INDEX]]
107+
; CHECK-NEXT: br label [[LOOP]]
108+
;
109+
entry:
110+
br label %loop
111+
112+
loop:
113+
%index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
114+
%step.add = add nuw i64 %index, %c1
115+
call void @use(i64 %step.add)
116+
%index.next = add nuw i64 %c2, %step.add
117+
br label %loop
118+
}
119+
120+
; Hoist ADD and copy NUW if both ops have it.
121+
; Another version where operands are commuted.
122+
define void @add_nuw_comm3(i64 %c1, i64 %c2) {
123+
; CHECK-LABEL: @add_nuw_comm3(
124+
; CHECK-NEXT: entry:
125+
; CHECK-NEXT: [[INVARIANT_OP:%.*]] = add nuw i64 [[C2:%.*]], [[C1:%.*]]
126+
; CHECK-NEXT: br label [[LOOP:%.*]]
127+
; CHECK: loop:
128+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
129+
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw i64 [[C1]], [[INDEX]]
130+
; CHECK-NEXT: call void @use(i64 [[STEP_ADD]])
131+
; CHECK-NEXT: [[INDEX_NEXT_REASS]] = add nuw i64 [[INVARIANT_OP]], [[INDEX]]
132+
; CHECK-NEXT: br label [[LOOP]]
133+
;
134+
entry:
135+
br label %loop
136+
137+
loop:
138+
%index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
139+
%step.add = add nuw i64 %c1, %index
140+
call void @use(i64 %step.add)
141+
%index.next = add nuw i64 %c2, %step.add
142+
br label %loop
143+
}
144+
95145
; Hoist MUL and drop NUW even if both ops have it.
96146
define void @mul_nuw(i64 %c1, i64 %c2) {
97147
; CHECK-LABEL: @mul_nuw(
@@ -116,7 +166,8 @@ loop:
116166
br label %loop
117167
}
118168

119-
; Hoist MUL and drop NUW even if both ops have it. Commutative version.
169+
; Hoist MUL and drop NUW even if both ops have it.
170+
; Version where operands are commuted.
120171
define void @mul_nuw_comm(i64 %c1, i64 %c2) {
121172
; CHECK-LABEL: @mul_nuw_comm(
122173
; CHECK-NEXT: entry:
@@ -140,6 +191,56 @@ loop:
140191
br label %loop
141192
}
142193

194+
; Hoist MUL and drop NUW even if both ops have it.
195+
; Another version where operands are commuted.
196+
define void @mul_nuw_comm2(i64 %c1, i64 %c2) {
197+
; CHECK-LABEL: @mul_nuw_comm2(
198+
; CHECK-NEXT: entry:
199+
; CHECK-NEXT: [[INVARIANT_OP:%.*]] = mul i64 [[C2:%.*]], [[C1:%.*]]
200+
; CHECK-NEXT: br label [[LOOP:%.*]]
201+
; CHECK: loop:
202+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
203+
; CHECK-NEXT: [[STEP_ADD:%.*]] = mul nuw i64 [[INDEX]], [[C1]]
204+
; CHECK-NEXT: call void @use(i64 [[STEP_ADD]])
205+
; CHECK-NEXT: [[INDEX_NEXT_REASS]] = mul i64 [[INVARIANT_OP]], [[INDEX]]
206+
; CHECK-NEXT: br label [[LOOP]]
207+
;
208+
entry:
209+
br label %loop
210+
211+
loop:
212+
%index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
213+
%step.add = mul nuw i64 %index, %c1
214+
call void @use(i64 %step.add)
215+
%index.next = mul nuw i64 %c2, %step.add
216+
br label %loop
217+
}
218+
219+
; Hoist MUL and drop NUW even if both ops have it.
220+
; Another version where operands are commuted.
221+
define void @mul_nuw_comm3(i64 %c1, i64 %c2) {
222+
; CHECK-LABEL: @mul_nuw_comm3(
223+
; CHECK-NEXT: entry:
224+
; CHECK-NEXT: [[INVARIANT_OP:%.*]] = mul i64 [[C2:%.*]], [[C1:%.*]]
225+
; CHECK-NEXT: br label [[LOOP:%.*]]
226+
; CHECK: loop:
227+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
228+
; CHECK-NEXT: [[STEP_ADD:%.*]] = mul nuw i64 [[C1]], [[INDEX]]
229+
; CHECK-NEXT: call void @use(i64 [[STEP_ADD]])
230+
; CHECK-NEXT: [[INDEX_NEXT_REASS]] = mul i64 [[INVARIANT_OP]], [[INDEX]]
231+
; CHECK-NEXT: br label [[LOOP]]
232+
;
233+
entry:
234+
br label %loop
235+
236+
loop:
237+
%index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
238+
%step.add = mul nuw i64 %c1, %index
239+
call void @use(i64 %step.add)
240+
%index.next = mul nuw i64 %c2, %step.add
241+
br label %loop
242+
}
243+
143244
; Hoist ADD but don't copy NUW if only one op has it.
144245
define void @add_no_nuw(i64 %c1, i64 %c2) {
145246
; CHECK-LABEL: @add_no_nuw(

0 commit comments

Comments
 (0)