Skip to content

Commit 01cbbc5

Browse files
fhahnfhahn
committed
[VPlan] Request lane 0 for pointer arg in PtrAdd.
After 7f74651, the pointer operand may be replicated of a PtrAdd. Instead of requesting a single scalar, request lane 0, which correctly handles the case when there is a scalar-per-lane. Fixes #111606.
1 parent 374fffe commit 01cbbc5

File tree

2 files changed

+174
-2
lines changed

2 files changed

+174
-2
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -645,8 +645,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
645645
case VPInstruction::PtrAdd: {
646646
assert(vputils::onlyFirstLaneUsed(this) &&
647647
"can only generate first lane for PtrAdd");
648-
Value *Ptr = State.get(getOperand(0), /* IsScalar */ true);
649-
Value *Addend = State.get(getOperand(1), /* IsScalar */ true);
648+
Value *Ptr = State.get(getOperand(0), VPLane(0));
649+
Value *Addend = State.get(getOperand(1), VPLane(0));
650650
return isInBounds() ? Builder.CreateInBoundsPtrAdd(Ptr, Addend, Name)
651651
: Builder.CreatePtrAdd(Ptr, Addend, Name);
652652
}

‎llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll

+172
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
2+
; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
3+
4+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
5+
target triple = "x86_64-unknown-linux-gnu"
6+
7+
; Test for https://github.com/llvm/llvm-project/issues/111606.
8+
define ptr @test_interleave_ptradd_with_replicated_op(ptr %m) #0 {
9+
; CHECK-LABEL: define ptr @test_interleave_ptradd_with_replicated_op(
10+
; CHECK-SAME: ptr [[M:%.*]]) #[[ATTR0:[0-9]+]] {
11+
; CHECK-NEXT: [[ENTRY:.*]]:
12+
; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
13+
; CHECK: [[VECTOR_PH]]:
14+
; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[M]], i64 768
15+
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
16+
; CHECK: [[VECTOR_BODY]]:
17+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
18+
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
19+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
20+
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 8
21+
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 16
22+
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 24
23+
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 32
24+
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 40
25+
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 48
26+
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 56
27+
; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 64
28+
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 72
29+
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 80
30+
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 88
31+
; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 96
32+
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 104
33+
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 112
34+
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 120
35+
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP0]]
36+
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP1]]
37+
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP2]]
38+
; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP3]]
39+
; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP4]]
40+
; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP5]]
41+
; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP6]]
42+
; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP7]]
43+
; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP8]]
44+
; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP9]]
45+
; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP10]]
46+
; CHECK-NEXT: [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP11]]
47+
; CHECK-NEXT: [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP12]]
48+
; CHECK-NEXT: [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP13]]
49+
; CHECK-NEXT: [[NEXT_GEP15:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP14]]
50+
; CHECK-NEXT: [[NEXT_GEP16:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP15]]
51+
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
52+
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 4
53+
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 4
54+
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 4
55+
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 4
56+
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 4
57+
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 4
58+
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 4
59+
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 4
60+
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 4
61+
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 4
62+
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 4
63+
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 4
64+
; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 4
65+
; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 4
66+
; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 4
67+
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP16]], i32 -4
68+
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[TMP20]], i32 -4
69+
; CHECK-NEXT: [[TMP34:%.*]] = getelementptr i8, ptr [[TMP24]], i32 -4
70+
; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[TMP28]], i32 -4
71+
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP32]], align 4
72+
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
73+
; CHECK-NEXT: [[STRIDED_VEC17:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
74+
; CHECK-NEXT: [[WIDE_VEC18:%.*]] = load <8 x i32>, ptr [[TMP33]], align 4
75+
; CHECK-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <8 x i32> [[WIDE_VEC18]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
76+
; CHECK-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <8 x i32> [[WIDE_VEC18]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
77+
; CHECK-NEXT: [[WIDE_VEC21:%.*]] = load <8 x i32>, ptr [[TMP34]], align 4
78+
; CHECK-NEXT: [[STRIDED_VEC22:%.*]] = shufflevector <8 x i32> [[WIDE_VEC21]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
79+
; CHECK-NEXT: [[STRIDED_VEC23:%.*]] = shufflevector <8 x i32> [[WIDE_VEC21]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
80+
; CHECK-NEXT: [[WIDE_VEC24:%.*]] = load <8 x i32>, ptr [[TMP35]], align 4
81+
; CHECK-NEXT: [[STRIDED_VEC25:%.*]] = shufflevector <8 x i32> [[WIDE_VEC24]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
82+
; CHECK-NEXT: [[STRIDED_VEC26:%.*]] = shufflevector <8 x i32> [[WIDE_VEC24]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
83+
; CHECK-NEXT: [[TMP36:%.*]] = add <4 x i32> [[STRIDED_VEC17]], [[STRIDED_VEC]]
84+
; CHECK-NEXT: [[TMP37:%.*]] = add <4 x i32> [[STRIDED_VEC20]], [[STRIDED_VEC19]]
85+
; CHECK-NEXT: [[TMP38:%.*]] = add <4 x i32> [[STRIDED_VEC23]], [[STRIDED_VEC22]]
86+
; CHECK-NEXT: [[TMP39:%.*]] = add <4 x i32> [[STRIDED_VEC26]], [[STRIDED_VEC25]]
87+
; CHECK-NEXT: [[TMP40:%.*]] = extractelement <4 x i32> [[TMP36]], i32 0
88+
; CHECK-NEXT: store i32 [[TMP40]], ptr [[NEXT_GEP]], align 4
89+
; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i32> [[TMP36]], i32 1
90+
; CHECK-NEXT: store i32 [[TMP41]], ptr [[NEXT_GEP2]], align 4
91+
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i32> [[TMP36]], i32 2
92+
; CHECK-NEXT: store i32 [[TMP42]], ptr [[NEXT_GEP3]], align 4
93+
; CHECK-NEXT: [[TMP43:%.*]] = extractelement <4 x i32> [[TMP36]], i32 3
94+
; CHECK-NEXT: store i32 [[TMP43]], ptr [[NEXT_GEP4]], align 4
95+
; CHECK-NEXT: [[TMP44:%.*]] = extractelement <4 x i32> [[TMP37]], i32 0
96+
; CHECK-NEXT: store i32 [[TMP44]], ptr [[NEXT_GEP5]], align 4
97+
; CHECK-NEXT: [[TMP45:%.*]] = extractelement <4 x i32> [[TMP37]], i32 1
98+
; CHECK-NEXT: store i32 [[TMP45]], ptr [[NEXT_GEP6]], align 4
99+
; CHECK-NEXT: [[TMP46:%.*]] = extractelement <4 x i32> [[TMP37]], i32 2
100+
; CHECK-NEXT: store i32 [[TMP46]], ptr [[NEXT_GEP7]], align 4
101+
; CHECK-NEXT: [[TMP47:%.*]] = extractelement <4 x i32> [[TMP37]], i32 3
102+
; CHECK-NEXT: store i32 [[TMP47]], ptr [[NEXT_GEP8]], align 4
103+
; CHECK-NEXT: [[TMP48:%.*]] = extractelement <4 x i32> [[TMP38]], i32 0
104+
; CHECK-NEXT: store i32 [[TMP48]], ptr [[NEXT_GEP9]], align 4
105+
; CHECK-NEXT: [[TMP49:%.*]] = extractelement <4 x i32> [[TMP38]], i32 1
106+
; CHECK-NEXT: store i32 [[TMP49]], ptr [[NEXT_GEP10]], align 4
107+
; CHECK-NEXT: [[TMP50:%.*]] = extractelement <4 x i32> [[TMP38]], i32 2
108+
; CHECK-NEXT: store i32 [[TMP50]], ptr [[NEXT_GEP11]], align 4
109+
; CHECK-NEXT: [[TMP51:%.*]] = extractelement <4 x i32> [[TMP38]], i32 3
110+
; CHECK-NEXT: store i32 [[TMP51]], ptr [[NEXT_GEP12]], align 4
111+
; CHECK-NEXT: [[TMP52:%.*]] = extractelement <4 x i32> [[TMP39]], i32 0
112+
; CHECK-NEXT: store i32 [[TMP52]], ptr [[NEXT_GEP13]], align 4
113+
; CHECK-NEXT: [[TMP53:%.*]] = extractelement <4 x i32> [[TMP39]], i32 1
114+
; CHECK-NEXT: store i32 [[TMP53]], ptr [[NEXT_GEP14]], align 4
115+
; CHECK-NEXT: [[TMP54:%.*]] = extractelement <4 x i32> [[TMP39]], i32 2
116+
; CHECK-NEXT: store i32 [[TMP54]], ptr [[NEXT_GEP15]], align 4
117+
; CHECK-NEXT: [[TMP55:%.*]] = extractelement <4 x i32> [[TMP39]], i32 3
118+
; CHECK-NEXT: store i32 [[TMP55]], ptr [[NEXT_GEP16]], align 4
119+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
120+
; CHECK-NEXT: [[TMP56:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
121+
; CHECK-NEXT: br i1 [[TMP56]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
122+
; CHECK: [[MIDDLE_BLOCK]]:
123+
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
124+
; CHECK: [[SCALAR_PH]]:
125+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[M]], %[[ENTRY]] ]
126+
; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 97, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
127+
; CHECK-NEXT: br label %[[LOOP:.*]]
128+
; CHECK: [[LOOP]]:
129+
; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
130+
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
131+
; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 8
132+
; CHECK-NEXT: [[P_4:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 4
133+
; CHECK-NEXT: [[L_1:%.*]] = load i32, ptr [[P_4]], align 4
134+
; CHECK-NEXT: [[P_0:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 0
135+
; CHECK-NEXT: [[L_2:%.*]] = load i32, ptr [[P_0]], align 4
136+
; CHECK-NEXT: [[ADD:%.*]] = add i32 [[L_1]], [[L_2]]
137+
; CHECK-NEXT: store i32 [[ADD]], ptr [[PTR_IV]], align 4
138+
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
139+
; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[IV]], 100
140+
; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
141+
; CHECK: [[EXIT]]:
142+
; CHECK-NEXT: [[P_4_LCSSA:%.*]] = phi ptr [ [[P_4]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ]
143+
; CHECK-NEXT: ret ptr [[P_4_LCSSA]]
144+
;
145+
entry:
146+
br label %loop
147+
148+
loop: ; preds = %loop, %entry
149+
%ptr.iv = phi ptr [ %m, %entry ], [ %ptr.iv.next, %loop ]
150+
%iv = phi i32 [ 1, %entry ], [ %iv.next, %loop ]
151+
%ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 8
152+
%p.4 = getelementptr i8, ptr %ptr.iv, i64 4
153+
%l.1 = load i32, ptr %p.4, align 4
154+
%p.0 = getelementptr i8, ptr %ptr.iv, i64 0
155+
%l.2 = load i32, ptr %p.0, align 4
156+
%add = add i32 %l.1, %l.2
157+
store i32 %add, ptr %ptr.iv, align 4
158+
%iv.next = add i32 %iv, 1
159+
%tobool.not = icmp eq i32 %iv, 100
160+
br i1 %tobool.not, label %exit, label %loop
161+
162+
exit: ; preds = %loop
163+
ret ptr %p.4
164+
}
165+
166+
attributes #0 = { "target-cpu"="znver2" }
167+
;.
168+
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
169+
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
170+
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
171+
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
172+
;.

0 commit comments

Comments
 (0)