@@ -10733,6 +10733,7 @@ SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
10733
10733
SDValue BasePtr = MemSD->getBasePtr();
10734
10734
10735
10735
SDValue Mask, PassThru, VL;
10736
+ bool IsExpandingLoad = false;
10736
10737
if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Op)) {
10737
10738
Mask = VPLoad->getMask();
10738
10739
PassThru = DAG.getUNDEF(VT);
@@ -10741,6 +10742,7 @@ SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
10741
10742
const auto *MLoad = cast<MaskedLoadSDNode>(Op);
10742
10743
Mask = MLoad->getMask();
10743
10744
PassThru = MLoad->getPassThru();
10745
+ IsExpandingLoad = MLoad->isExpandingLoad();
10744
10746
}
10745
10747
10746
10748
bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
@@ -10760,16 +10762,38 @@ SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
10760
10762
if (!VL)
10761
10763
VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
10762
10764
10763
- unsigned IntID =
10764
- IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
10765
+ SDValue Index;
10766
+ if (!IsUnmasked && IsExpandingLoad) {
10767
+ MVT IndexVT = ContainerVT;
10768
+ if (ContainerVT.isFloatingPoint())
10769
+ IndexVT = IndexVT.changeVectorElementTypeToInteger();
10770
+
10771
+ if (Subtarget.isRV32() && IndexVT.getVectorElementType().bitsGT(XLenVT))
10772
+ IndexVT = IndexVT.changeVectorElementType(XLenVT);
10773
+
10774
+ Index = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
10775
+ DAG.getConstant(Intrinsic::riscv_viota, DL, XLenVT),
10776
+ DAG.getUNDEF(IndexVT), Mask, VL);
10777
+ if (uint64_t EltSize = ContainerVT.getScalarSizeInBits(); EltSize > 8)
10778
+ Index = DAG.getNode(RISCVISD::SHL_VL, DL, IndexVT, Index,
10779
+ DAG.getConstant(Log2_64(EltSize / 8), DL, IndexVT),
10780
+ DAG.getUNDEF(IndexVT), Mask, VL);
10781
+ }
10782
+
10783
+ unsigned IntID = IsUnmasked ? Intrinsic::riscv_vle
10784
+ : IsExpandingLoad ? Intrinsic::riscv_vluxei_mask
10785
+ : Intrinsic::riscv_vle_mask;
10765
10786
SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
10766
10787
if (IsUnmasked)
10767
10788
Ops.push_back(DAG.getUNDEF(ContainerVT));
10768
10789
else
10769
10790
Ops.push_back(PassThru);
10770
10791
Ops.push_back(BasePtr);
10771
- if (!IsUnmasked)
10792
+ if (!IsUnmasked) {
10793
+ if (IsExpandingLoad)
10794
+ Ops.push_back(Index);
10772
10795
Ops.push_back(Mask);
10796
+ }
10773
10797
Ops.push_back(VL);
10774
10798
if (!IsUnmasked)
10775
10799
Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
0 commit comments