Skip to content

Commit 45a334d

Browse files
committed
[RISCV] Generaize reduction tree matching to all integer reductions (#68014) (reapply)
This was reverted in 824251c exposed by this change in a previous patch. Fixed in 199cbec. Original commit message follows. This builds on the transform introduced in #67821, and generalizes it for all integer reduction types. A couple of notes: * This will only form smax/smin/umax/umin reductions when zbb is enabled. Otherwise, we lower the min/max expressions early. I don't care about this case, and don't plan to address this further. * This excludes floating point. Floating point introduces concerns about associativity. I may or may not do a follow up patch for that case. * The explodevector test change is mildly undesirable from a clarity perspective. If anyone sees a good way to rewrite that to stablize the test, please suggest.
1 parent 7a73a2c commit 45a334d

File tree

3 files changed

+867
-646
lines changed

3 files changed

+867
-646
lines changed

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11113,6 +11113,31 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
1111311113
}
1111411114
}
1111511115

11116+
/// Given an integer binary operator, return the generic ISD::VECREDUCE_OP
11117+
/// which corresponds to it.
11118+
static unsigned getVecReduceOpcode(unsigned Opc) {
11119+
switch (Opc) {
11120+
default:
11121+
llvm_unreachable("Unhandled binary to transfrom reduction");
11122+
case ISD::ADD:
11123+
return ISD::VECREDUCE_ADD;
11124+
case ISD::UMAX:
11125+
return ISD::VECREDUCE_UMAX;
11126+
case ISD::SMAX:
11127+
return ISD::VECREDUCE_SMAX;
11128+
case ISD::UMIN:
11129+
return ISD::VECREDUCE_UMIN;
11130+
case ISD::SMIN:
11131+
return ISD::VECREDUCE_SMIN;
11132+
case ISD::AND:
11133+
return ISD::VECREDUCE_AND;
11134+
case ISD::OR:
11135+
return ISD::VECREDUCE_OR;
11136+
case ISD::XOR:
11137+
return ISD::VECREDUCE_XOR;
11138+
}
11139+
}
11140+
1111611141
/// Perform two related transforms whose purpose is to incrementally recognize
1111711142
/// an explode_vector followed by scalar reduction as a vector reduction node.
1111811143
/// This exists to recover from a deficiency in SLP which can't handle
@@ -11136,8 +11161,15 @@ combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG,
1113611161

1113711162
const SDLoc DL(N);
1113811163
const EVT VT = N->getValueType(0);
11139-
[[maybe_unused]] const unsigned Opc = N->getOpcode();
11140-
assert(Opc == ISD::ADD && "extend this to other reduction types");
11164+
11165+
// TODO: Handle floating point here.
11166+
if (!VT.isInteger())
11167+
return SDValue();
11168+
11169+
const unsigned Opc = N->getOpcode();
11170+
const unsigned ReduceOpc = getVecReduceOpcode(Opc);
11171+
assert(Opc == ISD::getVecReduceBaseOpcode(ReduceOpc) &&
11172+
"Inconsistent mappings");
1114111173
const SDValue LHS = N->getOperand(0);
1114211174
const SDValue RHS = N->getOperand(1);
1114311175

@@ -11167,13 +11199,13 @@ combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG,
1116711199
EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, 2);
1116811200
SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec,
1116911201
DAG.getVectorIdxConstant(0, DL));
11170-
return DAG.getNode(ISD::VECREDUCE_ADD, DL, VT, Vec);
11202+
return DAG.getNode(ReduceOpc, DL, VT, Vec);
1117111203
}
1117211204

1117311205
// Match (binop (reduce (extract_subvector V, 0),
1117411206
// (extract_vector_elt V, sizeof(SubVec))))
1117511207
// into a reduction of one more element from the original vector V.
11176-
if (LHS.getOpcode() != ISD::VECREDUCE_ADD)
11208+
if (LHS.getOpcode() != ReduceOpc)
1117711209
return SDValue();
1117811210

1117911211
SDValue ReduceVec = LHS.getOperand(0);
@@ -11189,7 +11221,7 @@ combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG,
1118911221
EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, Idx + 1);
1119011222
SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec,
1119111223
DAG.getVectorIdxConstant(0, DL));
11192-
return DAG.getNode(ISD::VECREDUCE_ADD, DL, VT, Vec);
11224+
return DAG.getNode(ReduceOpc, DL, VT, Vec);
1119311225
}
1119411226
}
1119511227

@@ -11697,6 +11729,8 @@ static SDValue performANDCombine(SDNode *N,
1169711729

1169811730
if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
1169911731
return V;
11732+
if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
11733+
return V;
1170011734

1170111735
if (DCI.isAfterLegalizeDAG())
1170211736
if (SDValue V = combineDeMorganOfBoolean(N, DAG))
@@ -11749,6 +11783,8 @@ static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
1174911783

1175011784
if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
1175111785
return V;
11786+
if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
11787+
return V;
1175211788

1175311789
if (DCI.isAfterLegalizeDAG())
1175411790
if (SDValue V = combineDeMorganOfBoolean(N, DAG))
@@ -11800,6 +11836,9 @@ static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
1180011836

1180111837
if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
1180211838
return V;
11839+
if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
11840+
return V;
11841+
1180311842
// fold (xor (select cond, 0, y), x) ->
1180411843
// (select cond, x, (xor x, y))
1180511844
return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
@@ -14005,8 +14044,13 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
1400514044
case ISD::SMAX:
1400614045
case ISD::SMIN:
1400714046
case ISD::FMAXNUM:
14008-
case ISD::FMINNUM:
14009-
return combineBinOpToReduce(N, DAG, Subtarget);
14047+
case ISD::FMINNUM: {
14048+
if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
14049+
return V;
14050+
if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
14051+
return V;
14052+
return SDValue();
14053+
}
1401014054
case ISD::SETCC:
1401114055
return performSETCCCombine(N, DAG, Subtarget);
1401214056
case ISD::SIGN_EXTEND_INREG:

0 commit comments

Comments
 (0)