diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h index 2f4b569041a6f..e1e50ac2078a7 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h @@ -65,13 +65,9 @@ enum { VLMulShift = ConstraintShift + 3, VLMulMask = 0b111 << VLMulShift, - // Force a tail agnostic policy even this instruction has a tied destination. - ForceTailAgnosticShift = VLMulShift + 3, - ForceTailAgnosticMask = 1 << ForceTailAgnosticShift, - // Is this a _TIED vector pseudo instruction. For these instructions we // shouldn't skip the tied operand when converting to MC instructions. - IsTiedPseudoShift = ForceTailAgnosticShift + 1, + IsTiedPseudoShift = VLMulShift + 3, IsTiedPseudoMask = 1 << IsTiedPseudoShift, // Does this instruction have a SEW operand. It will be the last explicit @@ -148,10 +144,6 @@ static inline unsigned getFormat(uint64_t TSFlags) { static inline VLMUL getLMul(uint64_t TSFlags) { return static_cast((TSFlags & VLMulMask) >> VLMulShift); } -/// \returns true if tail agnostic is enforced for the instruction. -static inline bool doesForceTailAgnostic(uint64_t TSFlags) { - return TSFlags & ForceTailAgnosticMask; -} /// \returns true if this a _TIED pseudo. static inline bool isTiedPseudo(uint64_t TSFlags) { return TSFlags & IsTiedPseudoMask; diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index fb2c5c62ef871..72b60439ca840 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -1838,13 +1838,16 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { return; } + SDValue PolicyOp = + CurDAG->getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT); + if (IsCmpConstant) { SDValue Imm = selectImm(CurDAG, SDLoc(Src2), XLenVT, CVal - 1, *Subtarget); ReplaceNode(Node, CurDAG->getMachineNode( VMSGTMaskOpcode, DL, VT, - {MaskedOff, Src1, Imm, Mask, VL, SEW})); + {MaskedOff, Src1, Imm, Mask, VL, SEW, PolicyOp})); return; } @@ -1853,10 +1856,10 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { // The result is mask undisturbed. // We use the same instructions to emulate mask agnostic behavior, because // the agnostic result can be either undisturbed or all 1. - SDValue Cmp = SDValue( - CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT, - {MaskedOff, Src1, Src2, Mask, VL, SEW}), - 0); + SDValue Cmp = SDValue(CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT, + {MaskedOff, Src1, Src2, Mask, + VL, SEW, PolicyOp}), + 0); // vmxor.mm vd, vd, v0 is used to update active value. ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT, {Cmp, Mask, VL, MaskSEW})); @@ -3792,9 +3795,9 @@ bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) { const MCInstrDesc &MaskedMCID = TII->get(N->getMachineOpcode()); const bool MaskedHasPassthru = RISCVII::isFirstDefTiedToFirstUse(MaskedMCID); - assert(RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) == - RISCVII::hasVecPolicyOp(MCID.TSFlags) && - "Masked and unmasked pseudos are inconsistent"); + assert((RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) || + !RISCVII::hasVecPolicyOp(MCID.TSFlags)) && + "Unmasked pseudo has policy but masked pseudo doesn't?"); assert(RISCVII::hasVecPolicyOp(MCID.TSFlags) == HasPassthru && "Unexpected pseudo structure"); assert(!(HasPassthru && !MaskedHasPassthru) && @@ -3803,11 +3806,18 @@ bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) { SmallVector Ops; // Skip the passthru operand at index 0 if the unmasked don't have one. bool ShouldSkip = !HasPassthru && MaskedHasPassthru; + bool DropPolicy = !RISCVII::hasVecPolicyOp(MCID.TSFlags) && + RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags); + bool HasChainOp = + N->getOperand(N->getNumOperands() - 1).getValueType() == MVT::Other; + unsigned LastOpNum = N->getNumOperands() - 1 - HasChainOp; for (unsigned I = ShouldSkip, E = N->getNumOperands(); I != E; I++) { // Skip the mask SDValue Op = N->getOperand(I); if (I == MaskOpIdx) continue; + if (DropPolicy && I == LastOpNum) + continue; Ops.push_back(Op); } diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index 4a74906ed3cc3..ffc7e09368824 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -1049,11 +1049,6 @@ RISCVInsertVSETVLI::computeInfoForInstr(const MachineInstr &MI) const { MaskAgnostic = Policy & RISCVII::MASK_AGNOSTIC; } - // Some pseudo instructions force a tail agnostic policy despite having a - // tied def. - if (RISCVII::doesForceTailAgnostic(TSFlags)) - TailAgnostic = true; - if (!RISCVII::usesMaskPolicy(TSFlags)) MaskAgnostic = true; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td index cea28bdce284c..47fe51bafd17c 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td +++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td @@ -193,36 +193,33 @@ class RVInstCommon VLMul = 0; let TSFlags{10-8} = VLMul; - bit ForceTailAgnostic = false; - let TSFlags{11} = ForceTailAgnostic; - bit IsTiedPseudo = 0; - let TSFlags{12} = IsTiedPseudo; + let TSFlags{11} = IsTiedPseudo; bit HasSEWOp = 0; - let TSFlags{13} = HasSEWOp; + let TSFlags{12} = HasSEWOp; bit HasVLOp = 0; - let TSFlags{14} = HasVLOp; + let TSFlags{13} = HasVLOp; bit HasVecPolicyOp = 0; - let TSFlags{15} = HasVecPolicyOp; + let TSFlags{14} = HasVecPolicyOp; bit IsRVVWideningReduction = 0; - let TSFlags{16} = IsRVVWideningReduction; + let TSFlags{15} = IsRVVWideningReduction; bit UsesMaskPolicy = 0; - let TSFlags{17} = UsesMaskPolicy; + let TSFlags{16} = UsesMaskPolicy; // Indicates that the result can be considered sign extended from bit 31. Some // instructions with this flag aren't W instructions, but are either sign // extended from a smaller size, always outputs a small integer, or put zeros // in bits 63:31. Used by the SExtWRemoval pass. bit IsSignExtendingOpW = 0; - let TSFlags{18} = IsSignExtendingOpW; + let TSFlags{17} = IsSignExtendingOpW; bit HasRoundModeOp = 0; - let TSFlags{19} = HasRoundModeOp; + let TSFlags{18} = HasRoundModeOp; // This is only valid when HasRoundModeOp is set to 1. HasRoundModeOp is set // to 1 for vector fixed-point or floating-point intrinsics. This bit is @@ -230,7 +227,7 @@ class RVInstCommon narrowing case // 3 -> widening case bits<2> TargetOverlapConstraintType = 0; - let TSFlags{22-21} = TargetOverlapConstraintType; + let TSFlags{21-20} = TargetOverlapConstraintType; // Most vector instructions are elementwise, but some may depend on the value // of VL (e.g. vslide1down.vx), and others may depend on the VL and mask // (e.g. vredsum.vs, viota.m). Mark these instructions so that peepholes avoid // changing their VL and/or mask. EltDeps ElementsDependOn = EltDepsNone; - let TSFlags{23} = ElementsDependOn.VL; - let TSFlags{24} = ElementsDependOn.Mask; + let TSFlags{22} = ElementsDependOn.VL; + let TSFlags{23} = ElementsDependOn.Mask; // Indicates the EEW of a vector instruction's destination operand. EEW DestEEW = EEWSEWx1; - let TSFlags{26-25} = DestEEW.Value; + let TSFlags{25-24} = DestEEW.Value; } class RVInst, + VMaskOp:$vm, AVL:$vl, sew:$sew, vec_policy:$policy), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; @@ -1427,6 +1427,7 @@ class VPseudoBinaryMOutMask { VPseudoBinaryNoMask; - let ForceTailAgnostic = true in def "_" # m.MX # "_MASK" : VPseudoBinaryMOutMask; + (mask_type VMV0:$vm), GPR:$vl, sew, TA_MU)>; class VPatBinaryMaskPolicy; + (mask_type VMV0:$vm), GPR:$vl, sew, TA_MU)>; class VPatTiedBinaryNoMask; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MU)>; } } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td index ffa3d3982647d..43cfc9d1e77ca 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td @@ -1020,7 +1020,7 @@ multiclass VPatIntegerSetCCVL_VV; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MU)>; } // Inherits from VPatIntegerSetCCVL_VV and adds a pattern with operands swapped. @@ -1034,7 +1034,8 @@ multiclass VPatIntegerSetCCVL_VV_Swappable(instruction_name#"_VV_"#vti.LMul.MX#"_MASK") VR:$passthru, vti.RegClass:$rs1, - vti.RegClass:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; + vti.RegClass:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, + vti.Log2SEW, TA_MU)>; } multiclass VPatIntegerSetCCVL_VX_Swappable; + GPR:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, + vti.Log2SEW, TA_MU)>; def : Pat<(vti.Mask (riscv_setcc_vl (SplatPat (XLenVT GPR:$rs2)), (vti.Vector vti.RegClass:$rs1), invcc, VR:$passthru, (vti.Mask VMV0:$vm), VLOpFrag)), (instruction_masked VR:$passthru, vti.RegClass:$rs1, - GPR:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; + GPR:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, + vti.Log2SEW, TA_MU)>; } multiclass VPatIntegerSetCCVL_VI_Swappable; + vti.Log2SEW, TA_MU)>; // FIXME: Can do some canonicalization to remove these patterns. def : Pat<(vti.Mask (riscv_setcc_vl (splatpat_kind simm5:$rs2), @@ -1077,7 +1080,7 @@ multiclass VPatIntegerSetCCVL_VI_Swappable; + vti.Log2SEW, TA_MU)>; } multiclass VPatFPSetCCVL_VV_VF_FV(inst_name#"_VV_"#fvti.LMul.MX#"_MASK") VR:$passthru, fvti.RegClass:$rs1, fvti.RegClass:$rs2, (fvti.Mask VMV0:$vm), - GPR:$vl, fvti.Log2SEW)>; + GPR:$vl, fvti.Log2SEW, TA_MU)>; def : Pat<(fvti.Mask (vop (fvti.Vector fvti.RegClass:$rs1), (SplatFPOp fvti.ScalarRegClass:$rs2), cc, @@ -1104,7 +1107,7 @@ multiclass VPatFPSetCCVL_VV_VF_FV(inst_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_MASK") VR:$passthru, fvti.RegClass:$rs1, fvti.ScalarRegClass:$rs2, (fvti.Mask VMV0:$vm), - GPR:$vl, fvti.Log2SEW)>; + GPR:$vl, fvti.Log2SEW, TA_MU)>; def : Pat<(fvti.Mask (vop (SplatFPOp fvti.ScalarRegClass:$rs2), (fvti.Vector fvti.RegClass:$rs1), cc, @@ -1114,7 +1117,7 @@ multiclass VPatFPSetCCVL_VV_VF_FV(swapped_op_inst_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_MASK") VR:$passthru, fvti.RegClass:$rs1, fvti.ScalarRegClass:$rs2, (fvti.Mask VMV0:$vm), - GPR:$vl, fvti.Log2SEW)>; + GPR:$vl, fvti.Log2SEW, TA_MU)>; } } } diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp index a4e7219c39f37..5ef1c9444f59a 100644 --- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp +++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp @@ -466,9 +466,9 @@ bool RISCVVectorPeephole::convertToUnmasked(MachineInstr &MI) const { RISCVII::hasVecPolicyOp(MCID.TSFlags); const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(MCID); const MCInstrDesc &MaskedMCID = TII->get(MI.getOpcode()); - assert(RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) == - RISCVII::hasVecPolicyOp(MCID.TSFlags) && - "Masked and unmasked pseudos are inconsistent"); + assert((RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) || + !RISCVII::hasVecPolicyOp(MCID.TSFlags)) && + "Unmasked pseudo has policy but masked pseudo doesn't?"); assert(HasPolicyOp == HasPassthru && "Unexpected pseudo structure"); assert(!(HasPassthru && !RISCVII::isFirstDefTiedToFirstUse(MaskedMCID)) && "Unmasked with passthru but masked with no passthru?"); @@ -476,6 +476,11 @@ bool RISCVVectorPeephole::convertToUnmasked(MachineInstr &MI) const { MI.setDesc(MCID); + // Drop the policy operand if unmasked doesn't need it. + if (RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) && + !RISCVII::hasVecPolicyOp(MCID.TSFlags)) + MI.removeOperand(RISCVII::getVecPolicyOpNum(MaskedMCID)); + // TODO: Increment all MaskOpIdxs in tablegen by num of explicit defs? unsigned MaskOpIdx = I->MaskOpIdx + MI.getNumExplicitDefs(); MI.removeOperand(MaskOpIdx); diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir b/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir index d2906c4613295..c84f7735b66d4 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir +++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.mir @@ -1116,10 +1116,10 @@ body: | bb.0: ; CHECK-LABEL: name: vmop_vv_passthru_use ; CHECK: %x:vrnov0 = PseudoVMAND_MM_B8 $noreg, $noreg, 1, 0 /* e8 */ - ; CHECK-NEXT: %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */ + ; CHECK-NEXT: %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */, 1 ; CHECK-NEXT: %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e8 */ %x:vrnov0 = PseudoVMAND_MM_B8 $noreg, $noreg, -1, 0 /* e1 */ - %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */ + %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */, 1 %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e1 */ ... --- @@ -1128,10 +1128,10 @@ body: | bb.0: ; CHECK-LABEL: name: vmop_vv_passthru_use_incompatible_eew ; CHECK: %x:vrnov0 = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 /* tu, mu */ - ; CHECK-NEXT: %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */ + ; CHECK-NEXT: %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */, 1 ; CHECK-NEXT: %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e8 */ %x:vrnov0 = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, -1, 3 /* e8 */, 0 - %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */ + %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */, 1 %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e1 */ ... --- @@ -1140,10 +1140,10 @@ body: | bb.0: ; CHECK-LABEL: name: vmop_vv_passthru_use_incompatible_emul ; CHECK: %x:vrnov0 = PseudoVMAND_MM_B16 $noreg, $noreg, -1, 0 /* e8 */ - ; CHECK-NEXT: %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */ + ; CHECK-NEXT: %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */, 1 ; CHECK-NEXT: %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e8 */ %x:vrnov0 = PseudoVMAND_MM_B16 $noreg, $noreg, -1, 0 /* e1 */ - %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */ + %y:vrnov0 = PseudoVMSEQ_VV_M1_MASK %x, $noreg, $noreg, $noreg, 1, 3 /* e8 */, 1 %z:vr = PseudoVMAND_MM_B8 %y, $noreg, 1, 0 /* e1 */ ... ---