Skip to content

Commit 3a7f4be

Browse files
artagnonartagnon
committed
LoopVectorize: optimize VF for low TC, when tail-folding
9a087a3 (LoopVectorize: MaxVF should not be larger than the loop trip count) was the first commit to add the condition PowerOf2_32() of the trip-count to, what is now getMaximizedVFForTarget(). It made sense at the time, as there was no tail-folding support. Much later, 2025e09 ([LV] Make sure VF doesn't exceed compile time known TC) came along to patch this with an extra condition on FoldTailByMasking, in order to ensure that that the VF doesn't exceed the trip-count. However, it didn't go far enough, and we can do better, as there is existing code to clamp the trip-count, and do tail-folding. Fixes #82626.
1 parent c1622ca commit 3a7f4be

File tree

3 files changed

+105
-104
lines changed

3 files changed

+105
-104
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4202,26 +4202,21 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
42024202
WidestRegisterMinEC *= Min;
42034203
}
42044204

4205-
// When a scalar epilogue is required, at least one iteration of the scalar
4206-
// loop has to execute. Adjust MaxTripCount accordingly to avoid picking a
4207-
// max VF that results in a dead vector loop.
4208-
if (MaxTripCount > 0 && requiresScalarEpilogue(true))
4209-
MaxTripCount -= 1;
4210-
4211-
if (MaxTripCount && MaxTripCount <= WidestRegisterMinEC &&
4212-
(!FoldTailByMasking || isPowerOf2_32(MaxTripCount))) {
4213-
// If upper bound loop trip count (TC) is known at compile time there is no
4214-
// point in choosing VF greater than TC (as done in the loop below). Select
4215-
// maximum power of two which doesn't exceed TC. If MaxVectorElementCount is
4216-
// scalable, we only fall back on a fixed VF when the TC is less than or
4217-
// equal to the known number of lanes.
4205+
if (MaxTripCount > 0 && MaxTripCount <= WidestRegisterMinEC &&
4206+
requiresScalarEpilogue(true)) {
4207+
// When a scalar epilogue is required, at least one iteration of the scalar
4208+
// loop has to execute. Adjust MaxTripCount accordingly to avoid picking a
4209+
// max VF that results in a dead vector loop.
4210+
--MaxTripCount;
4211+
4212+
// When a scalar epilogue is required, if upper bound loop trip count (TC)
4213+
// is known at compile time, clamp the VF to a maximum power of two which
4214+
// doesn't exceed TC, and vectorize using a fixed-length vector.
42184215
auto ClampedUpperTripCount = llvm::bit_floor(MaxTripCount);
42194216
LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to maximum power of two not "
42204217
"exceeding the constant trip count: "
42214218
<< ClampedUpperTripCount << "\n");
4222-
return ElementCount::get(
4223-
ClampedUpperTripCount,
4224-
FoldTailByMasking ? MaxVectorElementCount.isScalable() : false);
4219+
return ElementCount::getFixed(ClampedUpperTripCount);
42254220
}
42264221

42274222
TargetTransformInfo::RegisterKind RegKind =

‎llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll

Lines changed: 86 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -252,36 +252,46 @@ for.end: ; preds = %for.body
252252
define void @trip16_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
253253
; CHECK-LABEL: @trip16_i8(
254254
; CHECK-NEXT: entry:
255-
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
255+
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
256+
; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 8
257+
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 16, [[TMP1]]
258+
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
256259
; CHECK: vector.ph:
260+
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
261+
; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8
262+
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 16, [[TMP3]]
263+
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 16, [[N_MOD_VF]]
264+
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
265+
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8
257266
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
258267
; CHECK: vector.body:
259268
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
260-
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
261-
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP0]]
262-
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
263-
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
264-
; CHECK-NEXT: [[TMP3:%.*]] = shl <16 x i8> [[WIDE_LOAD]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
265-
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP0]]
266-
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
267-
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
268-
; CHECK-NEXT: [[TMP6:%.*]] = add <16 x i8> [[TMP3]], [[WIDE_LOAD1]]
269-
; CHECK-NEXT: store <16 x i8> [[TMP6]], ptr [[TMP5]], align 1
270-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
269+
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0
270+
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP6]]
271+
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0
272+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
273+
; CHECK-NEXT: [[TMP9:%.*]] = shl <vscale x 8 x i8> [[WIDE_LOAD]], shufflevector (<vscale x 8 x i8> insertelement (<vscale x 8 x i8> poison, i8 1, i64 0), <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer)
274+
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP6]]
275+
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 0
276+
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
277+
; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 8 x i8> [[TMP9]], [[WIDE_LOAD1]]
278+
; CHECK-NEXT: store <vscale x 8 x i8> [[TMP12]], ptr [[TMP11]], align 1
279+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
271280
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
272281
; CHECK: middle.block:
273-
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
282+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 16, [[N_VEC]]
283+
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
274284
; CHECK: scalar.ph:
275-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
285+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
276286
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
277287
; CHECK: for.body:
278288
; CHECK-NEXT: [[I_08:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
279289
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[I_08]]
280-
; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
281-
; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP7]], 1
290+
; CHECK-NEXT: [[TMP13:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
291+
; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP13]], 1
282292
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[I_08]]
283-
; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
284-
; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP8]]
293+
; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
294+
; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP14]]
285295
; CHECK-NEXT: store i8 [[ADD]], ptr [[ARRAYIDX1]], align 1
286296
; CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_08]], 1
287297
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 16
@@ -313,36 +323,46 @@ for.end: ; preds = %for.body
313323
define void @trip32_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
314324
; CHECK-LABEL: @trip32_i8(
315325
; CHECK-NEXT: entry:
316-
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
326+
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
327+
; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 16
328+
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 32, [[TMP1]]
329+
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
317330
; CHECK: vector.ph:
331+
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
332+
; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 16
333+
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 32, [[TMP3]]
334+
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 32, [[N_MOD_VF]]
335+
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
336+
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 16
318337
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
319338
; CHECK: vector.body:
320339
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
321-
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
322-
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP0]]
323-
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
324-
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP2]], align 1
325-
; CHECK-NEXT: [[TMP3:%.*]] = shl <32 x i8> [[WIDE_LOAD]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
326-
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP0]]
327-
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
328-
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP5]], align 1
329-
; CHECK-NEXT: [[TMP6:%.*]] = add <32 x i8> [[TMP3]], [[WIDE_LOAD1]]
330-
; CHECK-NEXT: store <32 x i8> [[TMP6]], ptr [[TMP5]], align 1
331-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
340+
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0
341+
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP6]]
342+
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0
343+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
344+
; CHECK-NEXT: [[TMP9:%.*]] = shl <vscale x 16 x i8> [[WIDE_LOAD]], shufflevector (<vscale x 16 x i8> insertelement (<vscale x 16 x i8> poison, i8 1, i64 0), <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer)
345+
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP6]]
346+
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 0
347+
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP11]], align 1
348+
; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 16 x i8> [[TMP9]], [[WIDE_LOAD1]]
349+
; CHECK-NEXT: store <vscale x 16 x i8> [[TMP12]], ptr [[TMP11]], align 1
350+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
332351
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
333352
; CHECK: middle.block:
334-
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
353+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 32, [[N_VEC]]
354+
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
335355
; CHECK: scalar.ph:
336-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 32, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
356+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
337357
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
338358
; CHECK: for.body:
339359
; CHECK-NEXT: [[I_08:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
340360
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[I_08]]
341-
; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
342-
; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP7]], 1
361+
; CHECK-NEXT: [[TMP13:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
362+
; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP13]], 1
343363
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[I_08]]
344-
; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
345-
; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP8]]
364+
; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
365+
; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP14]]
346366
; CHECK-NEXT: store i8 [[ADD]], ptr [[ARRAYIDX1]], align 1
347367
; CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_08]], 1
348368
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 32
@@ -373,37 +393,47 @@ for.end: ; preds = %for.body
373393
define void @trip24_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
374394
; CHECK-LABEL: @trip24_i8(
375395
; CHECK-NEXT: entry:
376-
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
396+
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
397+
; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4
398+
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 24, [[TMP1]]
399+
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
377400
; CHECK: vector.ph:
401+
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
402+
; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4
403+
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 24, [[TMP3]]
404+
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 24, [[N_MOD_VF]]
405+
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
406+
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
378407
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
379408
; CHECK: vector.body:
380409
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
381-
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
382-
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP0]]
383-
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
384-
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
385-
; CHECK-NEXT: [[TMP3:%.*]] = shl <8 x i8> [[WIDE_LOAD]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
386-
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP0]]
387-
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
388-
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
389-
; CHECK-NEXT: [[TMP6:%.*]] = add <8 x i8> [[TMP3]], [[WIDE_LOAD1]]
390-
; CHECK-NEXT: store <8 x i8> [[TMP6]], ptr [[TMP5]], align 1
391-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
392-
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
393-
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
410+
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0
411+
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP6]]
412+
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0
413+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP8]], align 1
414+
; CHECK-NEXT: [[TMP9:%.*]] = shl <vscale x 4 x i8> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i8> insertelement (<vscale x 4 x i8> poison, i8 1, i64 0), <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer)
415+
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP6]]
416+
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 0
417+
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
418+
; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i8> [[TMP9]], [[WIDE_LOAD1]]
419+
; CHECK-NEXT: store <vscale x 4 x i8> [[TMP12]], ptr [[TMP11]], align 1
420+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
421+
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
422+
; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
394423
; CHECK: middle.block:
395-
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
424+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 24, [[N_VEC]]
425+
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
396426
; CHECK: scalar.ph:
397-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 24, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
427+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
398428
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
399429
; CHECK: for.body:
400430
; CHECK-NEXT: [[I_08:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
401431
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[I_08]]
402-
; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
403-
; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP8]], 1
432+
; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
433+
; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP14]], 1
404434
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[I_08]]
405-
; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
406-
; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP9]]
435+
; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
436+
; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP15]]
407437
; CHECK-NEXT: store i8 [[ADD]], ptr [[ARRAYIDX1]], align 1
408438
; CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_08]], 1
409439
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 24

0 commit comments

Comments
 (0)