Skip to content

Commit be50a25

Browse files
author
Andy Kaylor
authored
Update foldFMulReassoc to respect absent fast-math flags (#88589)
This change updates a few of the transformations in foldFMulReassoc to respect absent fast-math flags in cases where fmul and fdiv, fadd, or fsub instructions were being folded but the code was only checking for fast-math flags on the fmul instruction and was transferring flags to the folded instruction that were not present on the other original instructions. This fixes #82857
1 parent 1bc0921 commit be50a25

File tree

5 files changed

+141
-62
lines changed

5 files changed

+141
-62
lines changed

‎llvm/include/llvm/IR/InstrTypes.h

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "llvm/IR/Attributes.h"
2525
#include "llvm/IR/CallingConv.h"
2626
#include "llvm/IR/DerivedTypes.h"
27+
#include "llvm/IR/FMF.h"
2728
#include "llvm/IR/Function.h"
2829
#include "llvm/IR/Instruction.h"
2930
#include "llvm/IR/LLVMContext.h"
@@ -311,6 +312,32 @@ class BinaryOperator : public Instruction {
311312
return BO;
312313
}
313314

315+
static BinaryOperator *CreateWithFMF(BinaryOps Opc, Value *V1, Value *V2,
316+
FastMathFlags FMF,
317+
const Twine &Name = "",
318+
Instruction *InsertBefore = nullptr) {
319+
BinaryOperator *BO = Create(Opc, V1, V2, Name, InsertBefore);
320+
BO->setFastMathFlags(FMF);
321+
return BO;
322+
}
323+
324+
static BinaryOperator *CreateFAddFMF(Value *V1, Value *V2, FastMathFlags FMF,
325+
const Twine &Name = "") {
326+
return CreateWithFMF(Instruction::FAdd, V1, V2, FMF, Name);
327+
}
328+
static BinaryOperator *CreateFSubFMF(Value *V1, Value *V2, FastMathFlags FMF,
329+
const Twine &Name = "") {
330+
return CreateWithFMF(Instruction::FSub, V1, V2, FMF, Name);
331+
}
332+
static BinaryOperator *CreateFMulFMF(Value *V1, Value *V2, FastMathFlags FMF,
333+
const Twine &Name = "") {
334+
return CreateWithFMF(Instruction::FMul, V1, V2, FMF, Name);
335+
}
336+
static BinaryOperator *CreateFDivFMF(Value *V1, Value *V2, FastMathFlags FMF,
337+
const Twine &Name = "") {
338+
return CreateWithFMF(Instruction::FDiv, V1, V2, FMF, Name);
339+
}
340+
314341
static BinaryOperator *CreateFAddFMF(Value *V1, Value *V2,
315342
Instruction *FMFSource,
316343
const Twine &Name = "") {

‎llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -624,31 +624,38 @@ Instruction *InstCombinerImpl::foldFMulReassoc(BinaryOperator &I) {
624624
Value *Op1 = I.getOperand(1);
625625
Value *X, *Y;
626626
Constant *C;
627+
BinaryOperator *Op0BinOp;
627628

628629
// Reassociate constant RHS with another constant to form constant
629630
// expression.
630-
if (match(Op1, m_Constant(C)) && C->isFiniteNonZeroFP()) {
631+
if (match(Op1, m_Constant(C)) && C->isFiniteNonZeroFP() &&
632+
match(Op0, m_AllowReassoc(m_BinOp(Op0BinOp)))) {
633+
// Everything in this scope folds I with Op0, intersecting their FMF.
634+
FastMathFlags FMF = I.getFastMathFlags() & Op0BinOp->getFastMathFlags();
635+
IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
636+
Builder.setFastMathFlags(FMF);
631637
Constant *C1;
632638
if (match(Op0, m_OneUse(m_FDiv(m_Constant(C1), m_Value(X))))) {
633639
// (C1 / X) * C --> (C * C1) / X
634640
Constant *CC1 =
635641
ConstantFoldBinaryOpOperands(Instruction::FMul, C, C1, DL);
636642
if (CC1 && CC1->isNormalFP())
637-
return BinaryOperator::CreateFDivFMF(CC1, X, &I);
643+
return BinaryOperator::CreateFDivFMF(CC1, X, FMF);
638644
}
639645
if (match(Op0, m_FDiv(m_Value(X), m_Constant(C1)))) {
646+
// FIXME: This seems like it should also be checking for arcp
640647
// (X / C1) * C --> X * (C / C1)
641648
Constant *CDivC1 =
642649
ConstantFoldBinaryOpOperands(Instruction::FDiv, C, C1, DL);
643650
if (CDivC1 && CDivC1->isNormalFP())
644-
return BinaryOperator::CreateFMulFMF(X, CDivC1, &I);
651+
return BinaryOperator::CreateFMulFMF(X, CDivC1, FMF);
645652

646653
// If the constant was a denormal, try reassociating differently.
647654
// (X / C1) * C --> X / (C1 / C)
648655
Constant *C1DivC =
649656
ConstantFoldBinaryOpOperands(Instruction::FDiv, C1, C, DL);
650657
if (C1DivC && Op0->hasOneUse() && C1DivC->isNormalFP())
651-
return BinaryOperator::CreateFDivFMF(X, C1DivC, &I);
658+
return BinaryOperator::CreateFDivFMF(X, C1DivC, FMF);
652659
}
653660

654661
// We do not need to match 'fadd C, X' and 'fsub X, C' because they are
@@ -658,26 +665,33 @@ Instruction *InstCombinerImpl::foldFMulReassoc(BinaryOperator &I) {
658665
// (X + C1) * C --> (X * C) + (C * C1)
659666
if (Constant *CC1 =
660667
ConstantFoldBinaryOpOperands(Instruction::FMul, C, C1, DL)) {
661-
Value *XC = Builder.CreateFMulFMF(X, C, &I);
662-
return BinaryOperator::CreateFAddFMF(XC, CC1, &I);
668+
Value *XC = Builder.CreateFMul(X, C);
669+
return BinaryOperator::CreateFAddFMF(XC, CC1, FMF);
663670
}
664671
}
665672
if (match(Op0, m_OneUse(m_FSub(m_Constant(C1), m_Value(X))))) {
666673
// (C1 - X) * C --> (C * C1) - (X * C)
667674
if (Constant *CC1 =
668675
ConstantFoldBinaryOpOperands(Instruction::FMul, C, C1, DL)) {
669-
Value *XC = Builder.CreateFMulFMF(X, C, &I);
670-
return BinaryOperator::CreateFSubFMF(CC1, XC, &I);
676+
Value *XC = Builder.CreateFMul(X, C);
677+
return BinaryOperator::CreateFSubFMF(CC1, XC, FMF);
671678
}
672679
}
673680
}
674681

675682
Value *Z;
676683
if (match(&I,
677-
m_c_FMul(m_OneUse(m_FDiv(m_Value(X), m_Value(Y))), m_Value(Z)))) {
678-
// Sink division: (X / Y) * Z --> (X * Z) / Y
679-
Value *NewFMul = Builder.CreateFMulFMF(X, Z, &I);
680-
return BinaryOperator::CreateFDivFMF(NewFMul, Y, &I);
684+
m_c_FMul(m_AllowReassoc(m_OneUse(m_FDiv(m_Value(X), m_Value(Y)))),
685+
m_Value(Z)))) {
686+
BinaryOperator *DivOp = cast<BinaryOperator>(((Z == Op0) ? Op1 : Op0));
687+
FastMathFlags FMF = I.getFastMathFlags() & DivOp->getFastMathFlags();
688+
if (FMF.allowReassoc()) {
689+
// Sink division: (X / Y) * Z --> (X * Z) / Y
690+
IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
691+
Builder.setFastMathFlags(FMF);
692+
auto *NewFMul = Builder.CreateFMul(X, Z);
693+
return BinaryOperator::CreateFDivFMF(NewFMul, Y, FMF);
694+
}
681695
}
682696

683697
// sqrt(X) * sqrt(Y) -> sqrt(X * Y)

‎llvm/test/Transforms/InstCombine/fast-math.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -562,7 +562,7 @@ define float @fdiv1(float %x) {
562562
; CHECK-NEXT: [[DIV1:%.*]] = fmul fast float [[X:%.*]], 0x3FD7303B60000000
563563
; CHECK-NEXT: ret float [[DIV1]]
564564
;
565-
%div = fdiv float %x, 0x3FF3333340000000
565+
%div = fdiv fast float %x, 0x3FF3333340000000
566566
%div1 = fdiv fast float %div, 0x4002666660000000
567567
ret float %div1
568568
; 0x3FF3333340000000 = 1.2f
@@ -603,7 +603,7 @@ define float @fdiv3(float %x) {
603603
; CHECK-NEXT: [[DIV1:%.*]] = fdiv fast float [[TMP1]], 0x47EFFFFFE0000000
604604
; CHECK-NEXT: ret float [[DIV1]]
605605
;
606-
%div = fdiv float %x, 0x47EFFFFFE0000000
606+
%div = fdiv fast float %x, 0x47EFFFFFE0000000
607607
%div1 = fdiv fast float %div, 0x4002666660000000
608608
ret float %div1
609609
}

‎llvm/test/Transforms/InstCombine/fmul-pow.ll

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,8 @@ define double @pow_ab_recip_a_reassoc(double %a, double %b) {
8585
; CHECK-NEXT: [[M:%.*]] = call reassoc double @llvm.pow.f64(double [[A:%.*]], double [[TMP1]])
8686
; CHECK-NEXT: ret double [[M]]
8787
;
88-
%r = fdiv double 1.0, %a
89-
%p = call double @llvm.pow.f64(double %a, double %b)
88+
%r = fdiv reassoc double 1.0, %a
89+
%p = call reassoc double @llvm.pow.f64(double %a, double %b)
9090
%m = fmul reassoc double %r, %p
9191
ret double %m
9292
}
@@ -99,8 +99,8 @@ define double @pow_ab_recip_a_reassoc_commute(double %a, double %b) {
9999
; CHECK-NEXT: [[M:%.*]] = call reassoc double @llvm.pow.f64(double [[A:%.*]], double [[TMP1]])
100100
; CHECK-NEXT: ret double [[M]]
101101
;
102-
%r = fdiv double 1.0, %a
103-
%p = call double @llvm.pow.f64(double %a, double %b)
102+
%r = fdiv reassoc double 1.0, %a
103+
%p = call reassoc double @llvm.pow.f64(double %a, double %b)
104104
%m = fmul reassoc double %p, %r
105105
ret double %m
106106
}
@@ -109,14 +109,14 @@ define double @pow_ab_recip_a_reassoc_commute(double %a, double %b) {
109109

110110
define double @pow_ab_recip_a_reassoc_use1(double %a, double %b) {
111111
; CHECK-LABEL: @pow_ab_recip_a_reassoc_use1(
112-
; CHECK-NEXT: [[R:%.*]] = fdiv double 1.000000e+00, [[A:%.*]]
113-
; CHECK-NEXT: [[P:%.*]] = call double @llvm.pow.f64(double [[A]], double [[B:%.*]])
112+
; CHECK-NEXT: [[R:%.*]] = fdiv reassoc double 1.000000e+00, [[A:%.*]]
113+
; CHECK-NEXT: [[P:%.*]] = call reassoc double @llvm.pow.f64(double [[A]], double [[B:%.*]])
114114
; CHECK-NEXT: [[M:%.*]] = fmul reassoc double [[R]], [[P]]
115115
; CHECK-NEXT: call void @use(double [[R]])
116116
; CHECK-NEXT: ret double [[M]]
117117
;
118-
%r = fdiv double 1.0, %a
119-
%p = call double @llvm.pow.f64(double %a, double %b)
118+
%r = fdiv reassoc double 1.0, %a
119+
%p = call reassoc double @llvm.pow.f64(double %a, double %b)
120120
%m = fmul reassoc double %r, %p
121121
call void @use(double %r)
122122
ret double %m
@@ -126,13 +126,13 @@ define double @pow_ab_recip_a_reassoc_use1(double %a, double %b) {
126126

127127
define double @pow_ab_recip_a_reassoc_use2(double %a, double %b) {
128128
; CHECK-LABEL: @pow_ab_recip_a_reassoc_use2(
129-
; CHECK-NEXT: [[P:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
129+
; CHECK-NEXT: [[P:%.*]] = call reassoc double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
130130
; CHECK-NEXT: [[M:%.*]] = fdiv reassoc double [[P]], [[A]]
131131
; CHECK-NEXT: call void @use(double [[P]])
132132
; CHECK-NEXT: ret double [[M]]
133133
;
134-
%r = fdiv double 1.0, %a
135-
%p = call double @llvm.pow.f64(double %a, double %b)
134+
%r = fdiv reassoc double 1.0, %a
135+
%p = call reassoc double @llvm.pow.f64(double %a, double %b)
136136
%m = fmul reassoc double %r, %p
137137
call void @use(double %p)
138138
ret double %m
@@ -142,15 +142,15 @@ define double @pow_ab_recip_a_reassoc_use2(double %a, double %b) {
142142

143143
define double @pow_ab_recip_a_reassoc_use3(double %a, double %b) {
144144
; CHECK-LABEL: @pow_ab_recip_a_reassoc_use3(
145-
; CHECK-NEXT: [[R:%.*]] = fdiv double 1.000000e+00, [[A:%.*]]
146-
; CHECK-NEXT: [[P:%.*]] = call double @llvm.pow.f64(double [[A]], double [[B:%.*]])
145+
; CHECK-NEXT: [[R:%.*]] = fdiv reassoc double 1.000000e+00, [[A:%.*]]
146+
; CHECK-NEXT: [[P:%.*]] = call reassoc double @llvm.pow.f64(double [[A]], double [[B:%.*]])
147147
; CHECK-NEXT: [[M:%.*]] = fmul reassoc double [[R]], [[P]]
148148
; CHECK-NEXT: call void @use(double [[R]])
149149
; CHECK-NEXT: call void @use(double [[P]])
150150
; CHECK-NEXT: ret double [[M]]
151151
;
152-
%r = fdiv double 1.0, %a
153-
%p = call double @llvm.pow.f64(double %a, double %b)
152+
%r = fdiv reassoc double 1.0, %a
153+
%p = call reassoc double @llvm.pow.f64(double %a, double %b)
154154
%m = fmul reassoc double %r, %p
155155
call void @use(double %r)
156156
call void @use(double %p)

0 commit comments

Comments
 (0)