Skip to content

Commit f7c7db9

Browse files
fhahntstellar
fhahn
authored and
tstellar
committed
[VPlan] Check VPWidenIntrinsicSC in VPRecipeWithIRFlags::classof.
When VPWidenIntrinsicRecipe was changed to inhert from VPRecipeWithIRFlags, VPRecipeWithIRFlags::classof wasn't updated accordingly. Also check for VPWidenIntrinsicSC in VPRecipeWithIRFlags::classof. Fixes #125301. (cherry picked from commit 75b922d)
1 parent 5b48526 commit f7c7db9

File tree

2 files changed

+152
-0
lines changed

2 files changed

+152
-0
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1058,6 +1058,7 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
10581058
R->getVPDefID() == VPRecipeBase::VPWidenEVLSC ||
10591059
R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
10601060
R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
1061+
R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC ||
10611062
R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
10621063
R->getVPDefID() == VPRecipeBase::VPReverseVectorPointerSC ||
10631064
R->getVPDefID() == VPRecipeBase::VPVectorPointerSC;

‎llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
2+
; RUN: opt -p loop-vectorize -mcpu=neoverse-v2 -force-vector-width=4 -S %s | FileCheck %s
3+
4+
target triple = "aarch64-unknown-linux"
5+
6+
; Test case where we visit a VPWidenIntrinsic (for @llvm.fabs) with nnan flags.
7+
; For https://github.com/llvm/llvm-project/issues/125301.
8+
define void @check_widen_intrinsic_with_nnan(ptr noalias %dst.0, ptr noalias %dst.1, ptr noalias %src.1, ptr %src.2) {
9+
; CHECK-LABEL: define void @check_widen_intrinsic_with_nnan(
10+
; CHECK-SAME: ptr noalias [[DST_0:%.*]], ptr noalias [[DST_1:%.*]], ptr noalias [[SRC_1:%.*]], ptr [[SRC_2:%.*]]) #[[ATTR0:[0-9]+]] {
11+
; CHECK-NEXT: [[ENTRY:.*]]:
12+
; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
13+
; CHECK: [[VECTOR_PH]]:
14+
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
15+
; CHECK: [[VECTOR_BODY]]:
16+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
17+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
18+
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[SRC_1]], i64 [[TMP0]]
19+
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i32 0
20+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP2]], align 8
21+
; CHECK-NEXT: [[TMP3:%.*]] = call <4 x double> @llvm.fabs.v4f64(<4 x double> [[WIDE_LOAD]])
22+
; CHECK-NEXT: [[TMP4:%.*]] = fcmp olt <4 x double> [[TMP3]], splat (double 1.000000e+00)
23+
; CHECK-NEXT: [[TMP5:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
24+
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP0]], -1
25+
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[DST_0]], i64 [[TMP6]]
26+
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr double, ptr [[TMP7]], i32 0
27+
; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> zeroinitializer, ptr [[TMP8]], i32 8, <4 x i1> [[TMP5]])
28+
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
29+
; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
30+
; CHECK: [[PRED_LOAD_IF]]:
31+
; CHECK-NEXT: [[TMP10:%.*]] = load double, ptr [[SRC_2]], align 8
32+
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> poison, double [[TMP10]], i32 0
33+
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
34+
; CHECK: [[PRED_LOAD_CONTINUE]]:
35+
; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x double> [ poison, %[[VECTOR_BODY]] ], [ [[TMP11]], %[[PRED_LOAD_IF]] ]
36+
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
37+
; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
38+
; CHECK: [[PRED_LOAD_IF1]]:
39+
; CHECK-NEXT: [[TMP14:%.*]] = load double, ptr [[SRC_2]], align 8
40+
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> [[TMP12]], double [[TMP14]], i32 1
41+
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
42+
; CHECK: [[PRED_LOAD_CONTINUE2]]:
43+
; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x double> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], %[[PRED_LOAD_IF1]] ]
44+
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
45+
; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
46+
; CHECK: [[PRED_LOAD_IF3]]:
47+
; CHECK-NEXT: [[TMP18:%.*]] = load double, ptr [[SRC_2]], align 8
48+
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP16]], double [[TMP18]], i32 2
49+
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
50+
; CHECK: [[PRED_LOAD_CONTINUE4]]:
51+
; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x double> [ [[TMP16]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], %[[PRED_LOAD_IF3]] ]
52+
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
53+
; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
54+
; CHECK: [[PRED_LOAD_IF5]]:
55+
; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[SRC_2]], align 8
56+
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x double> [[TMP20]], double [[TMP22]], i32 3
57+
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
58+
; CHECK: [[PRED_LOAD_CONTINUE6]]:
59+
; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x double> [ [[TMP20]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], %[[PRED_LOAD_IF5]] ]
60+
; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[TMP0]], -1
61+
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[DST_0]], i64 [[TMP25]]
62+
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr double, ptr [[TMP26]], i32 0
63+
; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> zeroinitializer, ptr [[TMP27]], i32 8, <4 x i1> [[TMP4]])
64+
; CHECK-NEXT: [[TMP28:%.*]] = fcmp oeq <4 x double> [[TMP24]], zeroinitializer
65+
; CHECK-NEXT: [[TMP29:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP28]], <4 x i1> zeroinitializer
66+
; CHECK-NEXT: [[TMP30:%.*]] = or <4 x i1> [[TMP5]], [[TMP29]]
67+
; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP29]], i32 0
68+
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP31]], i64 [[TMP25]], i64 [[TMP6]]
69+
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[DST_1]], i64 [[PREDPHI]]
70+
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[TMP32]], i32 0
71+
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> splat (i32 10), ptr [[TMP33]], i32 4, <4 x i1> [[TMP30]])
72+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
73+
; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
74+
; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
75+
; CHECK: [[MIDDLE_BLOCK]]:
76+
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
77+
; CHECK: [[SCALAR_PH]]:
78+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
79+
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
80+
; CHECK: [[LOOP_HEADER]]:
81+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
82+
; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds double, ptr [[SRC_1]], i64 [[IV]]
83+
; CHECK-NEXT: [[L_1:%.*]] = load double, ptr [[GEP_SRC_1]], align 8
84+
; CHECK-NEXT: [[ABS:%.*]] = tail call nnan double @llvm.fabs.f64(double [[L_1]])
85+
; CHECK-NEXT: [[C_0:%.*]] = fcmp olt double [[ABS]], 1.000000e+00
86+
; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]]
87+
; CHECK: [[THEN]]:
88+
; CHECK-NEXT: [[L_2:%.*]] = load double, ptr [[SRC_2]], align 8
89+
; CHECK-NEXT: [[IV_SUB_1:%.*]] = add nsw i64 [[IV]], -1
90+
; CHECK-NEXT: [[GEP_IV_SUB_1:%.*]] = getelementptr double, ptr [[DST_0]], i64 [[IV_SUB_1]]
91+
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_IV_SUB_1]], align 8
92+
; CHECK-NEXT: [[C_1:%.*]] = fcmp oeq double [[L_2]], 0.000000e+00
93+
; CHECK-NEXT: br i1 [[C_1]], label %[[MERGE:.*]], label %[[LOOP_LATCH]]
94+
; CHECK: [[ELSE]]:
95+
; CHECK-NEXT: [[IV_SUB_2:%.*]] = add nsw i64 [[IV]], -1
96+
; CHECK-NEXT: [[GEP_IV_SUB_2:%.*]] = getelementptr double, ptr [[DST_0]], i64 [[IV_SUB_2]]
97+
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_IV_SUB_2]], align 8
98+
; CHECK-NEXT: br label %[[MERGE]]
99+
; CHECK: [[MERGE]]:
100+
; CHECK-NEXT: [[MERGE_IV:%.*]] = phi i64 [ [[IV_SUB_2]], %[[ELSE]] ], [ [[IV_SUB_1]], %[[THEN]] ]
101+
; CHECK-NEXT: [[GEP_DST_1:%.*]] = getelementptr inbounds i32, ptr [[DST_1]], i64 [[MERGE_IV]]
102+
; CHECK-NEXT: store i32 10, ptr [[GEP_DST_1]], align 4
103+
; CHECK-NEXT: br label %[[LOOP_LATCH]]
104+
; CHECK: [[LOOP_LATCH]]:
105+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
106+
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
107+
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
108+
; CHECK: [[EXIT]]:
109+
; CHECK-NEXT: ret void
110+
;
111+
entry:
112+
br label %loop.header
113+
114+
loop.header:
115+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
116+
%gep.src.1 = getelementptr inbounds double, ptr %src.1, i64 %iv
117+
%l.1 = load double, ptr %gep.src.1, align 8
118+
%abs = tail call nnan double @llvm.fabs.f64(double %l.1)
119+
%c.0 = fcmp olt double %abs, 1.000000e+00
120+
br i1 %c.0, label %then, label %else
121+
122+
then:
123+
%l.2 = load double, ptr %src.2, align 8
124+
%iv.sub.1 = add nsw i64 %iv, -1
125+
%gep.iv.sub.1 = getelementptr double, ptr %dst.0, i64 %iv.sub.1
126+
store double 0.000000e+00, ptr %gep.iv.sub.1, align 8
127+
%c.1 = fcmp oeq double %l.2, 0.000000e+00
128+
br i1 %c.1, label %merge, label %loop.latch
129+
130+
else:
131+
%iv.sub.2 = add nsw i64 %iv, -1
132+
%gep.iv.sub.2 = getelementptr double, ptr %dst.0, i64 %iv.sub.2
133+
store double 0.000000e+00, ptr %gep.iv.sub.2, align 8
134+
br label %merge
135+
136+
merge:
137+
%merge.iv = phi i64 [ %iv.sub.2, %else ], [ %iv.sub.1, %then ]
138+
%gep.dst.1 = getelementptr inbounds i32, ptr %dst.1, i64 %merge.iv
139+
store i32 10, ptr %gep.dst.1, align 4
140+
br label %loop.latch
141+
142+
loop.latch:
143+
%iv.next = add nuw nsw i64 %iv, 1
144+
%exitcond.not = icmp eq i64 %iv.next, 1000
145+
br i1 %exitcond.not, label %exit, label %loop.header
146+
147+
exit:
148+
ret void
149+
}
150+
151+
declare double @llvm.fabs.f64(double)

0 commit comments

Comments
 (0)