Skip to content

Commit 16900d3

Browse files
artagnonartagnon
authored
LICM: hoist BO assoc when BinOp is in RHS (#107072)
Extend hoistBOAssociation smoothly to handle the case when the inner BinaryOperator is in the RHS of the outer BinaryOperator. This completes the generalization of hoistBOAssociation, and the only limitation after this patch is the fact that only Add and Mul are hoisted.
1 parent 52dc491 commit 16900d3

File tree

2 files changed

+171
-16
lines changed

2 files changed

+171
-16
lines changed

‎llvm/lib/Transforms/Scalar/LICM.cpp

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2803,15 +2803,14 @@ static bool hoistMulAddAssociation(Instruction &I, Loop &L,
28032803

28042804
/// Reassociate associative binary expressions of the form
28052805
///
2806-
/// 1. "(LV op C1) op C2" ==> "LV op (C1 op C2)" if op is an associative BinOp
2807-
/// 2. "(C1 op LV) op C2" ==> "LV op (C1 op C2)" if op is a commutative BinOp
2806+
/// 1. "(LV op C1) op C2" ==> "LV op (C1 op C2)"
2807+
/// 2. "(C1 op LV) op C2" ==> "LV op (C1 op C2)"
2808+
/// 3. "C2 op (C1 op LV)" ==> "LV op (C1 op C2)"
2809+
/// 4. "C2 op (LV op C1)" ==> "LV op (C1 op C2)"
28082810
///
2809-
/// where LV is a loop variant, and C1 and C2 are loop invariants that we want
2810-
/// to hoist.
2811-
///
2812-
/// TODO: This can be extended to more cases such as
2813-
/// 1. "C1 op (C2 op LV)" ==> "(C1 op C2) op LV" if op an associative BinOp
2814-
/// 2. "C1 op (LV op C2)" ==> "(C1 op C2) op LV" if op is a commutative BinOp
2811+
/// where op is an associative BinOp, LV is a loop variant, and C1 and C2 are
2812+
/// loop invariants that we want to hoist, noting that associativity implies
2813+
/// commutativity.
28152814
static bool hoistBOAssociation(Instruction &I, Loop &L,
28162815
ICFLoopSafetyInfo &SafetyInfo,
28172816
MemorySSAUpdater &MSSAU, AssumptionCache *AC,
@@ -2825,19 +2824,20 @@ static bool hoistBOAssociation(Instruction &I, Loop &L,
28252824
if (Opcode != Instruction::Add && Opcode != Instruction::Mul)
28262825
return false;
28272826

2828-
auto *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(0));
2827+
bool LVInRHS = L.isLoopInvariant(BO->getOperand(0));
2828+
auto *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(LVInRHS));
28292829
if (!BO0 || BO0->getOpcode() != Opcode || !BO0->isAssociative() ||
28302830
BO0->hasNUsesOrMore(3))
28312831
return false;
28322832

28332833
Value *LV = BO0->getOperand(0);
28342834
Value *C1 = BO0->getOperand(1);
2835-
Value *C2 = BO->getOperand(1);
2835+
Value *C2 = BO->getOperand(!LVInRHS);
28362836

2837-
if (L.isLoopInvariant(LV) && !L.isLoopInvariant(C1)) {
2838-
assert(BO0->isCommutative() && "Associativity implies commutativity");
2837+
assert(BO->isCommutative() && BO0->isCommutative() &&
2838+
"Associativity implies commutativity");
2839+
if (L.isLoopInvariant(LV) && !L.isLoopInvariant(C1))
28392840
std::swap(LV, C1);
2840-
}
28412841
if (L.isLoopInvariant(LV) || !L.isLoopInvariant(C1) || !L.isLoopInvariant(C2))
28422842
return false;
28432843

‎llvm/test/Transforms/LICM/hoist-binop.ll

Lines changed: 158 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ loop:
6767
br label %loop
6868
}
6969

70-
71-
; Hoist ADD and copy NUW if both ops have it. Commutative version.
70+
; Hoist ADD and copy NUW if both ops have it.
71+
; Version where operands are commuted.
7272
define void @add_nuw_comm(i64 %c1, i64 %c2) {
7373
; CHECK-LABEL: @add_nuw_comm(
7474
; CHECK-NEXT: entry:
@@ -92,6 +92,83 @@ loop:
9292
br label %loop
9393
}
9494

95+
; Hoist ADD and copy NUW if both ops have it.
96+
; Another version where operands are commuted.
97+
define void @add_nuw_comm2(i64 %c1, i64 %c2) {
98+
; CHECK-LABEL: @add_nuw_comm2(
99+
; CHECK-NEXT: entry:
100+
; CHECK-NEXT: [[INVARIANT_OP:%.*]] = add nuw i64 [[C1:%.*]], [[C2:%.*]]
101+
; CHECK-NEXT: br label [[LOOP:%.*]]
102+
; CHECK: loop:
103+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
104+
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw i64 [[INDEX]], [[C1]]
105+
; CHECK-NEXT: call void @use(i64 [[STEP_ADD]])
106+
; CHECK-NEXT: [[INDEX_NEXT_REASS]] = add nuw i64 [[INDEX]], [[INVARIANT_OP]]
107+
; CHECK-NEXT: br label [[LOOP]]
108+
;
109+
entry:
110+
br label %loop
111+
112+
loop:
113+
%index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
114+
%step.add = add nuw i64 %index, %c1
115+
call void @use(i64 %step.add)
116+
%index.next = add nuw i64 %c2, %step.add
117+
br label %loop
118+
}
119+
120+
; Hoist ADD and copy NUW if both ops have it.
121+
; Another version where operands are commuted.
122+
define void @add_nuw_comm3(i64 %c1, i64 %c2) {
123+
; CHECK-LABEL: @add_nuw_comm3(
124+
; CHECK-NEXT: entry:
125+
; CHECK-NEXT: [[INVARIANT_OP:%.*]] = add nuw i64 [[C1:%.*]], [[C2:%.*]]
126+
; CHECK-NEXT: br label [[LOOP:%.*]]
127+
; CHECK: loop:
128+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
129+
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw i64 [[C1]], [[INDEX]]
130+
; CHECK-NEXT: call void @use(i64 [[STEP_ADD]])
131+
; CHECK-NEXT: [[INDEX_NEXT_REASS]] = add nuw i64 [[INDEX]], [[INVARIANT_OP]]
132+
; CHECK-NEXT: br label [[LOOP]]
133+
;
134+
entry:
135+
br label %loop
136+
137+
loop:
138+
%index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
139+
%step.add = add nuw i64 %c1, %index
140+
call void @use(i64 %step.add)
141+
%index.next = add nuw i64 %c2, %step.add
142+
br label %loop
143+
}
144+
145+
; Hoist ADD and copy NUW if both ops have it.
146+
; A version where the LHS and RHS of the outer BinOp are BinOps.
147+
define void @add_nuw_twobinops(i64 %c1, i64 %c2) {
148+
; CHECK-LABEL: @add_nuw_twobinops(
149+
; CHECK-NEXT: entry:
150+
; CHECK-NEXT: [[C2_PLUS_2:%.*]] = add nuw i64 [[C2:%.*]], 2
151+
; CHECK-NEXT: [[INVARIANT_OP:%.*]] = add nuw i64 [[C1:%.*]], [[C2_PLUS_2]]
152+
; CHECK-NEXT: br label [[LOOP:%.*]]
153+
; CHECK: loop:
154+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
155+
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw i64 [[C1]], [[INDEX]]
156+
; CHECK-NEXT: call void @use(i64 [[STEP_ADD]])
157+
; CHECK-NEXT: [[INDEX_NEXT_REASS]] = add nuw i64 [[INDEX]], [[INVARIANT_OP]]
158+
; CHECK-NEXT: br label [[LOOP]]
159+
;
160+
entry:
161+
br label %loop
162+
163+
loop:
164+
%index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
165+
%step.add = add nuw i64 %c1, %index
166+
call void @use(i64 %step.add)
167+
%c2.plus.2 = add nuw i64 %c2, 2
168+
%index.next = add nuw i64 %step.add, %c2.plus.2
169+
br label %loop
170+
}
171+
95172
; Hoist MUL and drop NUW even if both ops have it.
96173
define void @mul_nuw(i64 %c1, i64 %c2) {
97174
; CHECK-LABEL: @mul_nuw(
@@ -116,7 +193,8 @@ loop:
116193
br label %loop
117194
}
118195

119-
; Hoist MUL and drop NUW even if both ops have it. Commutative version.
196+
; Hoist MUL and drop NUW even if both ops have it.
197+
; Version where operands are commuted.
120198
define void @mul_nuw_comm(i64 %c1, i64 %c2) {
121199
; CHECK-LABEL: @mul_nuw_comm(
122200
; CHECK-NEXT: entry:
@@ -140,6 +218,83 @@ loop:
140218
br label %loop
141219
}
142220

221+
; Hoist MUL and drop NUW even if both ops have it.
222+
; Another version where operands are commuted.
223+
define void @mul_nuw_comm2(i64 %c1, i64 %c2) {
224+
; CHECK-LABEL: @mul_nuw_comm2(
225+
; CHECK-NEXT: entry:
226+
; CHECK-NEXT: [[INVARIANT_OP:%.*]] = mul i64 [[C1:%.*]], [[C2:%.*]]
227+
; CHECK-NEXT: br label [[LOOP:%.*]]
228+
; CHECK: loop:
229+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
230+
; CHECK-NEXT: [[STEP_ADD:%.*]] = mul nuw i64 [[INDEX]], [[C1]]
231+
; CHECK-NEXT: call void @use(i64 [[STEP_ADD]])
232+
; CHECK-NEXT: [[INDEX_NEXT_REASS]] = mul i64 [[INDEX]], [[INVARIANT_OP]]
233+
; CHECK-NEXT: br label [[LOOP]]
234+
;
235+
entry:
236+
br label %loop
237+
238+
loop:
239+
%index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
240+
%step.add = mul nuw i64 %index, %c1
241+
call void @use(i64 %step.add)
242+
%index.next = mul nuw i64 %c2, %step.add
243+
br label %loop
244+
}
245+
246+
; Hoist MUL and drop NUW even if both ops have it.
247+
; Another version where operands are commuted.
248+
define void @mul_nuw_comm3(i64 %c1, i64 %c2) {
249+
; CHECK-LABEL: @mul_nuw_comm3(
250+
; CHECK-NEXT: entry:
251+
; CHECK-NEXT: [[INVARIANT_OP:%.*]] = mul i64 [[C1:%.*]], [[C2:%.*]]
252+
; CHECK-NEXT: br label [[LOOP:%.*]]
253+
; CHECK: loop:
254+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
255+
; CHECK-NEXT: [[STEP_ADD:%.*]] = mul nuw i64 [[C1]], [[INDEX]]
256+
; CHECK-NEXT: call void @use(i64 [[STEP_ADD]])
257+
; CHECK-NEXT: [[INDEX_NEXT_REASS]] = mul i64 [[INDEX]], [[INVARIANT_OP]]
258+
; CHECK-NEXT: br label [[LOOP]]
259+
;
260+
entry:
261+
br label %loop
262+
263+
loop:
264+
%index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
265+
%step.add = mul nuw i64 %c1, %index
266+
call void @use(i64 %step.add)
267+
%index.next = mul nuw i64 %c2, %step.add
268+
br label %loop
269+
}
270+
271+
; Hoist MUL and drop NUW even if both ops have it.
272+
; A version where the LHS and RHS of the outer BinOp are BinOps.
273+
define void @mul_nuw_twobinops(i64 %c1, i64 %c2) {
274+
; CHECK-LABEL: @mul_nuw_twobinops(
275+
; CHECK-NEXT: entry:
276+
; CHECK-NEXT: [[C2_PLUS_2:%.*]] = add nuw i64 [[C2:%.*]], 2
277+
; CHECK-NEXT: [[INVARIANT_OP:%.*]] = mul i64 [[C1:%.*]], [[C2_PLUS_2]]
278+
; CHECK-NEXT: br label [[LOOP:%.*]]
279+
; CHECK: loop:
280+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
281+
; CHECK-NEXT: [[STEP_ADD:%.*]] = mul nuw i64 [[C1]], [[INDEX]]
282+
; CHECK-NEXT: call void @use(i64 [[STEP_ADD]])
283+
; CHECK-NEXT: [[INDEX_NEXT_REASS]] = mul i64 [[INDEX]], [[INVARIANT_OP]]
284+
; CHECK-NEXT: br label [[LOOP]]
285+
;
286+
entry:
287+
br label %loop
288+
289+
loop:
290+
%index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
291+
%step.add = mul nuw i64 %c1, %index
292+
call void @use(i64 %step.add)
293+
%c2.plus.2 = add nuw i64 %c2, 2
294+
%index.next = mul nuw i64 %step.add, %c2.plus.2
295+
br label %loop
296+
}
297+
143298
; Hoist ADD but don't copy NUW if only one op has it.
144299
define void @add_no_nuw(i64 %c1, i64 %c2) {
145300
; CHECK-LABEL: @add_no_nuw(

0 commit comments

Comments
 (0)