[VP] TTI and lowering to non-VP ops

The ExpandVectorPredication pass lowers VP intrinsics into non-VP
operations. Also, TTI queries for VP intrinsic support in a target.

This patch is part of the integer patch set of the Vector Predication
extension (D57504).

VP / integer slice / patch #2

Author: Simon Moll

llvm/include/llvm/Analysis/TargetTransformInfo.h

@@ -45,6 +45,7 @@ class BranchInst;
 class Function;
 class GlobalValue;
 class IntrinsicInst;
+class PredicatedInstruction;
 class LoadInst;
 class LoopAccessInfo;
 class Loop;
@@ -1148,6 +1149,15 @@ class TargetTransformInfo {
   bool useReductionIntrinsic(unsigned Opcode, Type *Ty,
                              ReductionFlags Flags) const;
 
+  /// \returns True if the vector length parameter should be folded into the
+  /// vector mask.
+  bool
+  shouldFoldVectorLengthIntoMask(const PredicatedInstruction &PredInst) const;
+
+  /// \returns False if this VP op should be replaced by a non-VP op or an
+  /// unpredicated op plus a select.
+  bool supportsVPOperation(const PredicatedInstruction &PredInst) const;
+
   /// \returns True if the target wants to expand the given reduction intrinsic
   /// into a shuffle sequence.
   bool shouldExpandReduction(const IntrinsicInst *II) const;
@@ -1400,6 +1410,10 @@ class TargetTransformInfo::Concept {
   virtual unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
                                         unsigned ChainSizeInBytes,
                                         VectorType *VecTy) const = 0;
+  virtual bool shouldFoldVectorLengthIntoMask(
+      const PredicatedInstruction &PredInst) const = 0;
+  virtual bool
+  supportsVPOperation(const PredicatedInstruction &PredInst) const = 0;
   virtual bool useReductionIntrinsic(unsigned Opcode, Type *Ty,
                                      ReductionFlags) const = 0;
   virtual bool shouldExpandReduction(const IntrinsicInst *II) const = 0;
@@ -1879,6 +1893,13 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
                                 VectorType *VecTy) const override {
     return Impl.getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
   }
+  bool shouldFoldVectorLengthIntoMask(
+      const PredicatedInstruction &PredInst) const override {
+    return Impl.shouldFoldVectorLengthIntoMask(PredInst);
+  }
+  bool supportsVPOperation(const PredicatedInstruction &PredInst) const {
+    return Impl.supportsVPOperation(PredInst);
+  }
   bool useReductionIntrinsic(unsigned Opcode, Type *Ty,
                              ReductionFlags Flags) const override {
     return Impl.useReductionIntrinsic(Opcode, Ty, Flags);

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

@@ -607,6 +607,15 @@ class TargetTransformInfoImplBase {
     return VF;
   }
 
+  bool
+  shouldFoldVectorLengthIntoMask(const PredicatedInstruction &PredInst) const {
+    return true;
+  }
+
+  bool supportsVPOperation(const PredicatedInstruction &PredInst) const {
+    return false;
+  }
+
   bool useReductionIntrinsic(unsigned Opcode, Type *Ty,
                              TTI::ReductionFlags Flags) const {
     return false;

llvm/include/llvm/CodeGen/ExpandVectorPredication.h

@@ -0,0 +1,23 @@
+//===----- ExpandVectorPredication.h - Expand vector predication --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_EXPANDVECTORPREDICATION_H
+#define LLVM_CODEGEN_EXPANDVECTORPREDICATION_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class ExpandVectorPredicationPass
+    : public PassInfoMixin<ExpandVectorPredicationPass> {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_EXPANDVECTORPREDICATION_H

llvm/include/llvm/CodeGen/Passes.h

@@ -444,6 +444,11 @@ namespace llvm {
   /// shuffles.
   FunctionPass *createExpandReductionsPass();
 
+  /// This pass expands the vector predication intrinsics into unpredicated
+  /// instructions with selects or just the explicit vector length into the
+  /// predicate mask.
+  FunctionPass *createExpandVectorPredicationPass();
+
   // This pass expands memcmp() to load/stores.
   FunctionPass *createExpandMemCmpPass();
 

llvm/include/llvm/IR/IntrinsicInst.h

@@ -218,6 +218,15 @@ namespace llvm {
     // Whether \p ID is a VP intrinsic ID.
     static bool IsVPIntrinsic(Intrinsic::ID);
 
+    /// set the mask parameter.
+    /// this asserts if the underlying intrinsic has no mask parameter.
+    void setMaskParam(Value *);
+
+    /// set the vector length parameter.
+    /// this asserts if the underlying intrinsic has no vector length
+    /// parameter.
+    void setVectorLengthParam(Value *);
+
     /// \return the mask parameter or nullptr.
     Value *getMaskParam() const;
 
@@ -231,6 +240,9 @@ namespace llvm {
     /// length parameter applies to.
     ElementCount getVectorLength() const;
 
+    bool isBinaryOp() const;
+    static bool IsBinaryVPOp(Intrinsic::ID);
+
     // Methods for support type inquiry through isa, cast, and dyn_cast:
     static bool classof(const IntrinsicInst *I) {
       return IsVPIntrinsic(I->getIntrinsicID());

llvm/include/llvm/IR/PredicatedInst.h

@@ -0,0 +1,90 @@
+//===-- llvm/PredicatedInst.h - Predication utility subclass --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines various classes for working with predicated instructions.
+// Predicated instructions are either regular instructions or calls to
+// Vector Predication (VP) intrinsics that have a mask and an explicit
+// vector length argument.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_IR_PREDICATEDINST_H
+#define LLVM_IR_PREDICATEDINST_H
+
+#include "llvm/ADT/None.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Support/Casting.h"
+
+#include <cstddef>
+
+namespace llvm {
+
+class BasicBlock;
+
+class PredicatedInstruction : public User {
+public:
+  // The PredicatedInstruction class is intended to be used as a utility, and is
+  // never itself instantiated.
+  PredicatedInstruction() = delete;
+  ~PredicatedInstruction() = delete;
+
+  void *operator new(size_t s) = delete;
+
+  Value *getMaskParam() const {
+    auto thisVP = dyn_cast<VPIntrinsic>(this);
+    if (!thisVP)
+      return nullptr;
+    return thisVP->getMaskParam();
+  }
+
+  Value *getVectorLengthParam() const {
+    auto thisVP = dyn_cast<VPIntrinsic>(this);
+    if (!thisVP)
+      return nullptr;
+    return thisVP->getVectorLengthParam();
+  }
+
+  /// \returns True if the passed vector length value has no predicating effect
+  /// on the op.
+  bool canIgnoreVectorLengthParam() const;
+
+  /// \return True if the static operator of this instruction has a mask or
+  /// vector length parameter.
+  bool isVectorPredicatedOp() const { return isa<VPIntrinsic>(this); }
+
+  /// \returns the effective Opcode of this operation (ignoring the mask and
+  /// vector length param).
+  unsigned getOpcode() const {
+    auto *VPInst = dyn_cast<VPIntrinsic>(this);
+
+    if (!VPInst) {
+      return cast<Instruction>(this)->getOpcode();
+    }
+
+    return VPInst->getFunctionalOpcode();
+  }
+
+  static bool classof(const Instruction *I) { return isa<Instruction>(I); }
+  static bool classof(const ConstantExpr *CE) { return false; }
+  static bool classof(const Value *V) { return isa<Instruction>(V); }
+
+  /// Convenience function for getting all the fast-math flags, which must be an
+  /// operator which supports these flags. See LangRef.html for the meaning of
+  /// these flags.
+  FastMathFlags getFastMathFlags() const;
+};
+
+} // namespace llvm
+
+#endif // LLVM_IR_PREDICATEDINST_H

llvm/include/llvm/IR/VPIntrinsics.def

@@ -26,59 +26,79 @@
 #define HANDLE_VP_TO_OC(VPID, OC)
 #endif
 
+// This VP Intrinsic is a binary operator
+// (only count data params)
+#ifndef HANDLE_VP_IS_BINARY
+#define HANDLE_VP_IS_BINARY(VPID)
+#endif
+
 ///// Integer Arithmetic /////
 
 // llvm.vp.add(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_add, 2, 3)
 HANDLE_VP_TO_OC(vp_add, Add)
+HANDLE_VP_IS_BINARY(vp_add)
 
 // llvm.vp.and(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_and, 2, 3)
 HANDLE_VP_TO_OC(vp_and, And)
+HANDLE_VP_IS_BINARY(vp_and)
 
 // llvm.vp.ashr(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_ashr, 2, 3)
 HANDLE_VP_TO_OC(vp_ashr, AShr)
+HANDLE_VP_IS_BINARY(vp_ashr)
 
 // llvm.vp.lshr(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_lshr, 2, 3)
 HANDLE_VP_TO_OC(vp_lshr, LShr)
+HANDLE_VP_IS_BINARY(vp_lshr)
 
 // llvm.vp.mul(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_mul, 2, 3)
 HANDLE_VP_TO_OC(vp_mul, Mul)
+HANDLE_VP_IS_BINARY(vp_mul)
 
 // llvm.vp.or(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_or, 2, 3)
 HANDLE_VP_TO_OC(vp_or, Or)
+HANDLE_VP_IS_BINARY(vp_or)
 
 // llvm.vp.sdiv(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_sdiv, 2, 3)
 HANDLE_VP_TO_OC(vp_sdiv, SDiv)
+HANDLE_VP_IS_BINARY(vp_sdiv)
 
 // llvm.vp.shl(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_shl, 2, 3)
 HANDLE_VP_TO_OC(vp_shl, Shl)
+HANDLE_VP_IS_BINARY(vp_shl)
 
 // llvm.vp.srem(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_srem, 2, 3)
 HANDLE_VP_TO_OC(vp_srem, SRem)
+HANDLE_VP_IS_BINARY(vp_srem)
 
 // llvm.vp.sub(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_sub, 2, 3)
 HANDLE_VP_TO_OC(vp_sub, Sub)
+HANDLE_VP_IS_BINARY(vp_sub)
 
 // llvm.vp.udiv(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_udiv, 2, 3)
 HANDLE_VP_TO_OC(vp_udiv, UDiv)
+HANDLE_VP_IS_BINARY(vp_udiv)
 
 // llvm.vp.urem(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_urem, 2, 3)
 HANDLE_VP_TO_OC(vp_urem, URem)
+HANDLE_VP_IS_BINARY(vp_urem)
 
 // llvm.vp.xor(x,y,mask,vlen)
 REGISTER_VP_INTRINSIC(vp_xor, 2, 3)
 HANDLE_VP_TO_OC(vp_xor, Xor)
+HANDLE_VP_IS_BINARY(vp_xor)
 
 #undef REGISTER_VP_INTRINSIC
 #undef HANDLE_VP_TO_OC
+#undef HANDLE_VP_IS_BINARY

llvm/include/llvm/InitializePasses.h

@@ -145,6 +145,7 @@ void initializeEntryExitInstrumenterPass(PassRegistry&);
 void initializeExpandMemCmpPassPass(PassRegistry&);
 void initializeExpandPostRAPass(PassRegistry&);
 void initializeExpandReductionsPass(PassRegistry&);
+void initializeExpandVectorPredicationPass(PassRegistry&);
 void initializeMakeGuardsExplicitLegacyPassPass(PassRegistry&);
 void initializeExternalAAWrapperPassPass(PassRegistry&);
 void initializeFEntryInserterPass(PassRegistry&);

llvm/lib/Analysis/TargetTransformInfo.cpp

@@ -847,6 +847,16 @@ unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
   return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
 }
 
+bool TargetTransformInfo::shouldFoldVectorLengthIntoMask(
+    const PredicatedInstruction &PI) const {
+  return TTIImpl->shouldFoldVectorLengthIntoMask(PI);
+}
+
+bool TargetTransformInfo::supportsVPOperation(
+    const PredicatedInstruction &PI) const {
+  return TTIImpl->supportsVPOperation(PI);
+}
+
 bool TargetTransformInfo::useReductionIntrinsic(unsigned Opcode,
                                                 Type *Ty, ReductionFlags Flags) const {
   return TTIImpl->useReductionIntrinsic(Opcode, Ty, Flags);

llvm/lib/CodeGen/CMakeLists.txt

@@ -25,6 +25,7 @@ add_llvm_component_library(LLVMCodeGen
   ExpandMemCmp.cpp
   ExpandPostRAPseudos.cpp
   ExpandReductions.cpp
+  ExpandVectorPredication.cpp
   FaultMaps.cpp
   FEntryInserter.cpp
   FinalizeISel.cpp