Skip to content

Commit a12d7e4

Browse files
authored
[SLP] getVectorCallCosts - don't provide scalar argument data for vector IntrinsicCostAttributes (#124254)
getVectorCallCosts determines the cost of a vector intrinsic, based off an existing scalar intrinsic call - but we were including the scalar argument data to the IntrinsicCostAttributes, which meant that not only was the cost calculation not type-only based, it was making incorrect assumptions about constant values etc. This also exposed an issue that x86 relied on fallback calculations for funnel shift costs - this is great when we have the argument data as that improves the accuracy of uniform shift amounts etc., but meant that type-only costs would default to Cost=2 for all custom lowered funnel shifts, which was far too cheap. This is the reverse of #124129 where we weren't including argument data when we could. Fixes #63980
1 parent c546b53 commit a12d7e4

File tree

4 files changed

+409
-503
lines changed

4 files changed

+409
-503
lines changed

llvm/lib/Target/X86/X86TargetTransformInfo.cpp

+18
Original file line numberDiff line numberDiff line change
@@ -4719,6 +4719,24 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
47194719
if (const auto *Entry = CostTableLookup(X86CostTbl, ISD, MTy))
47204720
if (auto KindCost = Entry->Cost[CostKind])
47214721
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
4722+
4723+
// Without arg data, we need to compute the expanded costs of custom lowered
4724+
// intrinsics to prevent use of the (very low) default costs.
4725+
if (ICA.isTypeBasedOnly() &&
4726+
(IID == Intrinsic::fshl || IID == Intrinsic::fshr)) {
4727+
Type *CondTy = RetTy->getWithNewBitWidth(1);
4728+
InstructionCost Cost = 0;
4729+
Cost += getArithmeticInstrCost(BinaryOperator::Or, RetTy, CostKind);
4730+
Cost += getArithmeticInstrCost(BinaryOperator::Sub, RetTy, CostKind);
4731+
Cost += getArithmeticInstrCost(BinaryOperator::Shl, RetTy, CostKind);
4732+
Cost += getArithmeticInstrCost(BinaryOperator::LShr, RetTy, CostKind);
4733+
Cost += getArithmeticInstrCost(BinaryOperator::And, RetTy, CostKind);
4734+
Cost += getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
4735+
CmpInst::ICMP_EQ, CostKind);
4736+
Cost += getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
4737+
CmpInst::ICMP_EQ, CostKind);
4738+
return Cost;
4739+
}
47224740
}
47234741

47244742
return BaseT::getIntrinsicInstrCost(ICA, CostKind);

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

+1-3
Original file line numberDiff line numberDiff line change
@@ -9031,9 +9031,7 @@ getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
90319031
FastMathFlags FMF;
90329032
if (auto *FPCI = dyn_cast<FPMathOperator>(CI))
90339033
FMF = FPCI->getFastMathFlags();
9034-
SmallVector<const Value *> Arguments(CI->args());
9035-
IntrinsicCostAttributes CostAttrs(ID, VecTy, Arguments, ArgTys, FMF,
9036-
dyn_cast<IntrinsicInst>(CI));
9034+
IntrinsicCostAttributes CostAttrs(ID, VecTy, ArgTys, FMF);
90379035
auto IntrinsicCost =
90389036
TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
90399037

0 commit comments

Comments
 (0)