Skip to content

Commit 8a78d14

Browse files
RKSimonRKSimon
committed
[X86] Lower vXi8 multiplies by constant using PMADDUBSW on SSSE3+ targets
As discussed on llvm#90748
1 parent 47f8b85 commit 8a78d14

18 files changed

+700
-858
lines changed

‎llvm/lib/Target/X86/X86ISelLowering.cpp

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28491,6 +28491,8 @@ static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
2849128491
// vector pairs, multiply and truncate.
2849228492
if (VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8) {
2849328493
unsigned NumElts = VT.getVectorNumElements();
28494+
unsigned NumLanes = VT.getSizeInBits() / 128;
28495+
unsigned NumEltsPerLane = NumElts / NumLanes;
2849428496

2849528497
if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
2849628498
(VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
@@ -28504,6 +28506,31 @@ static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
2850428506

2850528507
MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
2850628508

28509+
// For vXi8 mul-by-constant, try PMADDUBSW to avoid the need for extension.
28510+
// Don't do this if we only need to unpack one half.
28511+
if (Subtarget.hasSSSE3() &&
28512+
ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
28513+
bool IsLoLaneZeroOrUndef = true;
28514+
bool IsHiLaneZeroOrUndef = true;
28515+
for (auto [Idx, Val] : enumerate(B->ops())) {
28516+
if ((Idx % NumEltsPerLane) >= (NumEltsPerLane / 2))
28517+
IsHiLaneZeroOrUndef &= isNullConstantOrUndef(Val);
28518+
else
28519+
IsLoLaneZeroOrUndef &= isNullConstantOrUndef(Val);
28520+
}
28521+
if (!(IsLoLaneZeroOrUndef || IsHiLaneZeroOrUndef)) {
28522+
SDValue Mask = DAG.getBitcast(VT, DAG.getConstant(0x00FF, dl, ExVT));
28523+
SDValue BLo = DAG.getNode(ISD::AND, dl, VT, Mask, B);
28524+
SDValue BHi = DAG.getNode(X86ISD::ANDNP, dl, VT, Mask, B);
28525+
SDValue RLo = DAG.getNode(X86ISD::VPMADDUBSW, dl, ExVT, A, BLo);
28526+
SDValue RHi = DAG.getNode(X86ISD::VPMADDUBSW, dl, ExVT, A, BHi);
28527+
RLo = DAG.getNode(ISD::AND, dl, VT, DAG.getBitcast(VT, RLo), Mask);
28528+
RHi = DAG.getNode(X86ISD::VSHLI, dl, ExVT, RHi,
28529+
DAG.getTargetConstant(8, dl, MVT::i8));
28530+
return VT, DAG.getNode(ISD::OR, dl, VT, RLo, DAG.getBitcast(VT, RHi));
28531+
}
28532+
}
28533+
2850728534
// Extract the lo/hi parts to any extend to i16.
2850828535
// We're going to mask off the low byte of each result element of the
2850928536
// pmullw, so it doesn't matter what's in the high byte of each 16-bit

‎llvm/test/CodeGen/X86/combine-mul.ll

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -541,15 +541,12 @@ define i64 @combine_mul_smul_lohi_const_i64(i64 %h) {
541541
define <16 x i8> @PR35579(<16 x i8> %x) {
542542
; SSE-LABEL: PR35579:
543543
; SSE: # %bb.0:
544-
; SSE-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
545-
; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
546-
; SSE-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [8,1,2,1,4,1,2,1]
547-
; SSE-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
548-
; SSE-NEXT: pand %xmm2, %xmm0
549-
; SSE-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [0,1,2,1,4,1,2,1]
550-
; SSE-NEXT: pand %xmm2, %xmm1
551-
; SSE-NEXT: packuswb %xmm0, %xmm1
552-
; SSE-NEXT: movdqa %xmm1, %xmm0
544+
; SSE-NEXT: movdqa %xmm0, %xmm1
545+
; SSE-NEXT: pmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
546+
; SSE-NEXT: psllw $8, %xmm1
547+
; SSE-NEXT: pmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
548+
; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
549+
; SSE-NEXT: por %xmm1, %xmm0
553550
; SSE-NEXT: retq
554551
;
555552
; AVX-LABEL: PR35579:

‎llvm/test/CodeGen/X86/gfni-shifts.ll

Lines changed: 121 additions & 173 deletions
Large diffs are not rendered by default.

‎llvm/test/CodeGen/X86/pmul.ll

Lines changed: 83 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,12 @@ define <16 x i8> @mul_v16i8c(<16 x i8> %i) nounwind {
2222
;
2323
; SSE41-LABEL: mul_v16i8c:
2424
; SSE41: # %bb.0: # %entry
25-
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
26-
; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
27-
; SSE41-NEXT: pmovsxbw {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117]
28-
; SSE41-NEXT: pmullw %xmm2, %xmm0
29-
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
30-
; SSE41-NEXT: pand %xmm3, %xmm0
31-
; SSE41-NEXT: pmullw %xmm2, %xmm1
32-
; SSE41-NEXT: pand %xmm3, %xmm1
33-
; SSE41-NEXT: packuswb %xmm0, %xmm1
34-
; SSE41-NEXT: movdqa %xmm1, %xmm0
25+
; SSE41-NEXT: movdqa %xmm0, %xmm1
26+
; SSE41-NEXT: pmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
27+
; SSE41-NEXT: psllw $8, %xmm1
28+
; SSE41-NEXT: pmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
29+
; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
30+
; SSE41-NEXT: por %xmm1, %xmm0
3531
; SSE41-NEXT: retq
3632
;
3733
; AVX2-LABEL: mul_v16i8c:
@@ -405,50 +401,39 @@ define <32 x i8> @mul_v32i8c(<32 x i8> %i) nounwind {
405401
;
406402
; SSE41-LABEL: mul_v32i8c:
407403
; SSE41: # %bb.0: # %entry
408-
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
409-
; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
410-
; SSE41-NEXT: pmovsxbw {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117]
411-
; SSE41-NEXT: pmullw %xmm4, %xmm0
412-
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
413-
; SSE41-NEXT: pand %xmm5, %xmm0
414-
; SSE41-NEXT: pmullw %xmm4, %xmm2
415-
; SSE41-NEXT: pand %xmm5, %xmm2
416-
; SSE41-NEXT: packuswb %xmm0, %xmm2
417-
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
418-
; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
419-
; SSE41-NEXT: pmullw %xmm4, %xmm1
420-
; SSE41-NEXT: pand %xmm5, %xmm1
421-
; SSE41-NEXT: pmullw %xmm4, %xmm3
422-
; SSE41-NEXT: pand %xmm5, %xmm3
423-
; SSE41-NEXT: packuswb %xmm1, %xmm3
424-
; SSE41-NEXT: movdqa %xmm2, %xmm0
425-
; SSE41-NEXT: movdqa %xmm3, %xmm1
404+
; SSE41-NEXT: pmovsxbw {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117]
405+
; SSE41-NEXT: movdqa %xmm0, %xmm3
406+
; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
407+
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
408+
; SSE41-NEXT: pand %xmm4, %xmm3
409+
; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]
410+
; SSE41-NEXT: pmaddubsw %xmm5, %xmm0
411+
; SSE41-NEXT: psllw $8, %xmm0
412+
; SSE41-NEXT: por %xmm3, %xmm0
413+
; SSE41-NEXT: movdqa %xmm1, %xmm3
414+
; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
415+
; SSE41-NEXT: pand %xmm4, %xmm3
416+
; SSE41-NEXT: pmaddubsw %xmm5, %xmm1
417+
; SSE41-NEXT: psllw $8, %xmm1
418+
; SSE41-NEXT: por %xmm3, %xmm1
426419
; SSE41-NEXT: retq
427420
;
428421
; AVX2-LABEL: mul_v32i8c:
429422
; AVX2: # %bb.0: # %entry
430-
; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
431-
; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
432-
; AVX2-NEXT: vpmullw %ymm2, %ymm1, %ymm1
433-
; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
434-
; AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
435-
; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
436-
; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm0
437-
; AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
438-
; AVX2-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
423+
; AVX2-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
424+
; AVX2-NEXT: vpsllw $8, %ymm1, %ymm1
425+
; AVX2-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
426+
; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
427+
; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0
439428
; AVX2-NEXT: retq
440429
;
441430
; AVX512F-LABEL: mul_v32i8c:
442431
; AVX512F: # %bb.0: # %entry
443-
; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
444-
; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
445-
; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm1
446-
; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
447-
; AVX512F-NEXT: vpand %ymm3, %ymm1, %ymm1
448-
; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
449-
; AVX512F-NEXT: vpmullw %ymm2, %ymm0, %ymm0
450-
; AVX512F-NEXT: vpand %ymm3, %ymm0, %ymm0
451-
; AVX512F-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
432+
; AVX512F-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
433+
; AVX512F-NEXT: vpsllw $8, %ymm1, %ymm1
434+
; AVX512F-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
435+
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
436+
; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0
452437
; AVX512F-NEXT: retq
453438
;
454439
; AVX512BW-LABEL: mul_v32i8c:
@@ -795,95 +780,74 @@ define <64 x i8> @mul_v64i8c(<64 x i8> %i) nounwind {
795780
;
796781
; SSE41-LABEL: mul_v64i8c:
797782
; SSE41: # %bb.0: # %entry
798-
; SSE41-NEXT: movdqa %xmm1, %xmm4
799-
; SSE41-NEXT: movdqa %xmm0, %xmm1
800-
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
801-
; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
802-
; SSE41-NEXT: pmovsxbw {{.*#+}} xmm6 = [117,117,117,117,117,117,117,117]
803-
; SSE41-NEXT: pmullw %xmm6, %xmm1
804-
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
805-
; SSE41-NEXT: pand %xmm7, %xmm1
806-
; SSE41-NEXT: pmullw %xmm6, %xmm0
807-
; SSE41-NEXT: pand %xmm7, %xmm0
808-
; SSE41-NEXT: packuswb %xmm1, %xmm0
809-
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
810-
; SSE41-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
811-
; SSE41-NEXT: pmullw %xmm6, %xmm4
812-
; SSE41-NEXT: pand %xmm7, %xmm4
813-
; SSE41-NEXT: pmullw %xmm6, %xmm1
814-
; SSE41-NEXT: pand %xmm7, %xmm1
815-
; SSE41-NEXT: packuswb %xmm4, %xmm1
816-
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
817-
; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
818-
; SSE41-NEXT: pmullw %xmm6, %xmm2
819-
; SSE41-NEXT: pand %xmm7, %xmm2
820-
; SSE41-NEXT: pmullw %xmm6, %xmm4
821-
; SSE41-NEXT: pand %xmm7, %xmm4
822-
; SSE41-NEXT: packuswb %xmm2, %xmm4
823-
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
824-
; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
825-
; SSE41-NEXT: pmullw %xmm6, %xmm3
826-
; SSE41-NEXT: pand %xmm7, %xmm3
827-
; SSE41-NEXT: pmullw %xmm6, %xmm5
828-
; SSE41-NEXT: pand %xmm7, %xmm5
829-
; SSE41-NEXT: packuswb %xmm3, %xmm5
830-
; SSE41-NEXT: movdqa %xmm4, %xmm2
831-
; SSE41-NEXT: movdqa %xmm5, %xmm3
783+
; SSE41-NEXT: pmovsxbw {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117]
784+
; SSE41-NEXT: movdqa %xmm0, %xmm6
785+
; SSE41-NEXT: pmaddubsw %xmm4, %xmm6
786+
; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
787+
; SSE41-NEXT: pand %xmm5, %xmm6
788+
; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]
789+
; SSE41-NEXT: pmaddubsw %xmm7, %xmm0
790+
; SSE41-NEXT: psllw $8, %xmm0
791+
; SSE41-NEXT: por %xmm6, %xmm0
792+
; SSE41-NEXT: movdqa %xmm1, %xmm6
793+
; SSE41-NEXT: pmaddubsw %xmm4, %xmm6
794+
; SSE41-NEXT: pand %xmm5, %xmm6
795+
; SSE41-NEXT: pmaddubsw %xmm7, %xmm1
796+
; SSE41-NEXT: psllw $8, %xmm1
797+
; SSE41-NEXT: por %xmm6, %xmm1
798+
; SSE41-NEXT: movdqa %xmm2, %xmm6
799+
; SSE41-NEXT: pmaddubsw %xmm4, %xmm6
800+
; SSE41-NEXT: pand %xmm5, %xmm6
801+
; SSE41-NEXT: pmaddubsw %xmm7, %xmm2
802+
; SSE41-NEXT: psllw $8, %xmm2
803+
; SSE41-NEXT: por %xmm6, %xmm2
804+
; SSE41-NEXT: movdqa %xmm3, %xmm6
805+
; SSE41-NEXT: pmaddubsw %xmm4, %xmm6
806+
; SSE41-NEXT: pand %xmm5, %xmm6
807+
; SSE41-NEXT: pmaddubsw %xmm7, %xmm3
808+
; SSE41-NEXT: psllw $8, %xmm3
809+
; SSE41-NEXT: por %xmm6, %xmm3
832810
; SSE41-NEXT: retq
833811
;
834812
; AVX2-LABEL: mul_v64i8c:
835813
; AVX2: # %bb.0: # %entry
836-
; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
837-
; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
838-
; AVX2-NEXT: vpmullw %ymm3, %ymm2, %ymm2
814+
; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0]
815+
; AVX2-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm3
839816
; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
817+
; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
818+
; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm5 = [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]
819+
; AVX2-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm0
820+
; AVX2-NEXT: vpsllw $8, %ymm0, %ymm0
821+
; AVX2-NEXT: vpor %ymm0, %ymm3, %ymm0
822+
; AVX2-NEXT: vpmaddubsw %ymm2, %ymm1, %ymm2
840823
; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2
841-
; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
842-
; AVX2-NEXT: vpmullw %ymm3, %ymm0, %ymm0
843-
; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
844-
; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
845-
; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
846-
; AVX2-NEXT: vpmullw %ymm3, %ymm2, %ymm2
847-
; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2
848-
; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
849-
; AVX2-NEXT: vpmullw %ymm3, %ymm1, %ymm1
850-
; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
851-
; AVX2-NEXT: vpackuswb %ymm2, %ymm1, %ymm1
824+
; AVX2-NEXT: vpmaddubsw %ymm5, %ymm1, %ymm1
825+
; AVX2-NEXT: vpsllw $8, %ymm1, %ymm1
826+
; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1
852827
; AVX2-NEXT: retq
853828
;
854829
; AVX512F-LABEL: mul_v64i8c:
855830
; AVX512F: # %bb.0: # %entry
856831
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
857-
; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
858-
; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm3 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
859-
; AVX512F-NEXT: vpmullw %ymm3, %ymm2, %ymm2
860-
; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
861-
; AVX512F-NEXT: vpand %ymm4, %ymm2, %ymm2
862-
; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
863-
; AVX512F-NEXT: vpmullw %ymm3, %ymm1, %ymm1
864-
; AVX512F-NEXT: vpand %ymm4, %ymm1, %ymm1
865-
; AVX512F-NEXT: vpackuswb %ymm2, %ymm1, %ymm1
866-
; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
867-
; AVX512F-NEXT: vpmullw %ymm3, %ymm2, %ymm2
868-
; AVX512F-NEXT: vpand %ymm4, %ymm2, %ymm2
869-
; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
870-
; AVX512F-NEXT: vpmullw %ymm3, %ymm0, %ymm0
871-
; AVX512F-NEXT: vpand %ymm4, %ymm0, %ymm0
872-
; AVX512F-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
832+
; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm2 = [117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0]
833+
; AVX512F-NEXT: vpmaddubsw %ymm2, %ymm1, %ymm3
834+
; AVX512F-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm2
835+
; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
836+
; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm3 = [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]
837+
; AVX512F-NEXT: vpmaddubsw %ymm3, %ymm0, %ymm0
838+
; AVX512F-NEXT: vpsllw $8, %ymm0, %ymm0
839+
; AVX512F-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1
840+
; AVX512F-NEXT: vpsllw $8, %ymm1, %ymm1
873841
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
842+
; AVX512F-NEXT: vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0
874843
; AVX512F-NEXT: retq
875844
;
876845
; AVX512BW-LABEL: mul_v64i8c:
877846
; AVX512BW: # %bb.0: # %entry
878-
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
879-
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]
880-
; AVX512BW-NEXT: vpmullw %zmm2, %zmm1, %zmm1
881-
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
882-
; AVX512BW-NEXT: vpandq %zmm3, %zmm1, %zmm1
883-
; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
884-
; AVX512BW-NEXT: vpmullw %zmm2, %zmm0, %zmm0
885-
; AVX512BW-NEXT: vpandq %zmm3, %zmm0, %zmm0
886-
; AVX512BW-NEXT: vpackuswb %zmm1, %zmm0, %zmm0
847+
; AVX512BW-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
848+
; AVX512BW-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
849+
; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm0
850+
; AVX512BW-NEXT: vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
887851
; AVX512BW-NEXT: retq
888852
entry:
889853
%A = mul <64 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >

0 commit comments

Comments
 (0)