[IR] Initial introduction of memset_pattern

llvm/docs/LangRef.rst

@@ -15234,6 +15234,59 @@ The behavior of '``llvm.memset.inline.*``' is equivalent to the behavior of
 '``llvm.memset.*``', but the generated code is guaranteed not to call any
 external functions.
 
+.. _int_memset_pattern_inline:
+
+'``llvm.memset_pattern.inline``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use ``llvm.memset_pattern.inline`` on
+any integer bit width and for different address spaces. Not all targets
+support all bit widths however.
+
+::
+
+      declare void @llvm.memset_pattern.inline.p0.i64.i128(ptr <dest>, i128 <val>,
+                                                   i64 <len>, i1 <isvolatile>)
+
+Overview:
+"""""""""
+
+The '``llvm.memset_pattern.inline.*``' intrinsics fill a block of memory with
+a particular value and guarantees that no external functions are called.
+
+Arguments:
+""""""""""
+
+The first argument is a pointer to the destination to fill, the second
+is the value with which to fill it, the third argument is a constant integer
+argument specifying the number of bytes to fill, and the fourth is a boolean
+indicating a volatile access.
+
+The :ref:`align <attr_align>` parameter attribute can be provided
+for the first argument.
+
+If the ``isvolatile`` parameter is ``true``, the
+``llvm.memset_pattern.inline`` call is a :ref:`volatile operation <volatile>`.
+The detailed access behavior is not very cleanly specified and it is unwise to
+depend on it.
+
+Semantics:
+""""""""""
+
+The '``llvm.memset_pattern.inline.*``' intrinsics fill "len" bytes of memory
+starting at the destination location. If the argument is known to be aligned
+to some boundary, this can be specified as an attribute on the argument.
+
+``len`` must be a constant expression.
+If ``<len>`` is 0, it is no-op modulo the behavior of attributes attached to
+the arguments.
+If ``<len>`` is not a well-defined value, the behavior is undefined.
+If ``<len>`` is not zero, ``<dest>`` should be well-defined, otherwise the
+behavior is undefined.
+
 .. _int_sqrt:
 
 '``llvm.sqrt.*``' Intrinsic

llvm/include/llvm/CodeGen/SelectionDAG.h

@@ -1205,6 +1205,12 @@ class SelectionDAG {
                     MachinePointerInfo DstPtrInfo,
                     const AAMDNodes &AAInfo = AAMDNodes());
 
+  SDValue getMemsetPatternInline(SDValue Chain, const SDLoc &dl, SDValue Dst,
+                                 SDValue Src, SDValue Size, Align Alignment,
+                                 bool isVol, bool isTailCall,
+                                 MachinePointerInfo DstPtrInfo,
+                                 const AAMDNodes &AAInfo = AAMDNodes());
+
   SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
                           SDValue Src, SDValue Size, Type *SizeTy,
                           unsigned ElemSz, bool isTailCall,

llvm/include/llvm/IR/InstVisitor.h

@@ -208,6 +208,7 @@ class InstVisitor {
   RetTy visitDbgInfoIntrinsic(DbgInfoIntrinsic &I){ DELEGATE(IntrinsicInst); }
   RetTy visitMemSetInst(MemSetInst &I)            { DELEGATE(MemIntrinsic); }
   RetTy visitMemSetInlineInst(MemSetInlineInst &I){ DELEGATE(MemSetInst); }
+  RetTy visitMemSetPatternInlineInst(MemSetPatternInlineInst &I){ DELEGATE(MemSetInst); }
   RetTy visitMemCpyInst(MemCpyInst &I)            { DELEGATE(MemTransferInst); }
   RetTy visitMemCpyInlineInst(MemCpyInlineInst &I){ DELEGATE(MemCpyInst); }
   RetTy visitMemMoveInst(MemMoveInst &I)          { DELEGATE(MemTransferInst); }
@@ -295,6 +296,8 @@ class InstVisitor {
       case Intrinsic::memset:      DELEGATE(MemSetInst);
       case Intrinsic::memset_inline:
         DELEGATE(MemSetInlineInst);
+      case Intrinsic::memset_pattern_inline:
+        DELEGATE(MemSetPatternInlineInst);
       case Intrinsic::vastart:     DELEGATE(VAStartInst);
       case Intrinsic::vaend:       DELEGATE(VAEndInst);
       case Intrinsic::vacopy:      DELEGATE(VACopyInst);

llvm/include/llvm/IR/IntrinsicInst.h

@@ -1170,6 +1170,7 @@ class MemIntrinsic : public MemIntrinsicBase<MemIntrinsic> {
     case Intrinsic::memmove:
     case Intrinsic::memset:
     case Intrinsic::memset_inline:
+    case Intrinsic::memset_pattern_inline:
     case Intrinsic::memcpy_inline:
       return true;
     default:
@@ -1181,14 +1182,16 @@ class MemIntrinsic : public MemIntrinsicBase<MemIntrinsic> {
   }
 };
 
-/// This class wraps the llvm.memset and llvm.memset.inline intrinsics.
+/// This class wraps the llvm.memset, llvm.memset.inline, and
+/// llvm.memset_pattern.inline intrinsics.
 class MemSetInst : public MemSetBase<MemIntrinsic> {
 public:
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const IntrinsicInst *I) {
     switch (I->getIntrinsicID()) {
     case Intrinsic::memset:
     case Intrinsic::memset_inline:
+    case Intrinsic::memset_pattern_inline:
       return true;
     default:
       return false;
@@ -1214,6 +1217,21 @@ class MemSetInlineInst : public MemSetInst {
   }
 };
 
+/// This class wraps the llvm.memset.inline intrinsic.
+class MemSetPatternInlineInst : public MemSetInst {
+public:
+  ConstantInt *getLength() const {
+    return cast<ConstantInt>(MemSetInst::getLength());
+  }
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static bool classof(const IntrinsicInst *I) {
+    return I->getIntrinsicID() == Intrinsic::memset_pattern_inline;
+  }
+  static bool classof(const Value *V) {
+    return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
+  }
+};
+
 /// This class wraps the llvm.memcpy/memmove intrinsics.
 class MemTransferInst : public MemTransferBase<MemIntrinsic> {
 public:
@@ -1293,6 +1311,7 @@ class AnyMemIntrinsic : public MemIntrinsicBase<AnyMemIntrinsic> {
     case Intrinsic::memmove:
     case Intrinsic::memset:
     case Intrinsic::memset_inline:
+    case Intrinsic::memset_pattern_inline:
     case Intrinsic::memcpy_element_unordered_atomic:
     case Intrinsic::memmove_element_unordered_atomic:
     case Intrinsic::memset_element_unordered_atomic:
@@ -1315,6 +1334,7 @@ class AnyMemSetInst : public MemSetBase<AnyMemIntrinsic> {
     switch (I->getIntrinsicID()) {
     case Intrinsic::memset:
     case Intrinsic::memset_inline:
+    case Intrinsic::memset_pattern_inline:
     case Intrinsic::memset_element_unordered_atomic:
       return true;
     default:

llvm/include/llvm/IR/Intrinsics.td

@@ -1008,6 +1008,15 @@ def int_memset_inline
        NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>,
        ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
 
+// Memset variant that writes a given pattern. Like memset.inline, this is
+// guaranteed not to call any external function.
+def int_memset_pattern_inline
+    : Intrinsic<[],
+      [llvm_anyptr_ty, llvm_anyint_ty, llvm_anyint_ty, llvm_i1_ty],
+      [IntrWriteMem, IntrArgMemOnly, IntrWillReturn, IntrNoFree, IntrNoCallback,
+       NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>,
+       ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>], "llvm.memset_pattern.inline">;
+
 // FIXME: Add version of these floating point intrinsics which allow non-default
 // rounding modes and FP exception handling.
 

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

@@ -8484,6 +8484,47 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
   return CallResult.second;
 }
 
+SDValue SelectionDAG::getMemsetPatternInline(SDValue Chain, const SDLoc &dl,
+                                             SDValue Dst, SDValue Src,
+                                             SDValue Size, Align Alignment,
+                                             bool isVol, bool isTailCall,
+                                             MachinePointerInfo DstPtrInfo,
+                                             const AAMDNodes &AAInfo) {
+  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
+  if (ConstantSize->isZero())
+    return Chain;
+
+  uint64_t SrcWidth = Src.getScalarValueSizeInBits() / 8;
+  unsigned NumFullWidthStores = ConstantSize->getZExtValue() / SrcWidth;
+  unsigned RemainingBytes = ConstantSize->getZExtValue() % SrcWidth;
+  SmallVector<SDValue, 8> OutChains;
+  uint64_t DstOff = 0;
+
+  for (unsigned i = 0; i < ConstantSize->getZExtValue() / SrcWidth; i++) {
+    SDValue Store = getStore(
+        Chain, dl, Src,
+        getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
+        DstPtrInfo.getWithOffset(DstOff), Alignment,
+        isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
+        AAInfo);
+    OutChains.push_back(Store);
+    DstOff += Src.getValueType().getSizeInBits() / 8;
+  }
+
+  if (RemainingBytes) {
+    EVT IntVT = EVT::getIntegerVT(*getContext(), RemainingBytes * 8);
+    SDValue Store = getTruncStore(
+        Chain, dl, Src,
+        getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
+        DstPtrInfo.getWithOffset(DstOff), IntVT, Alignment,
+        isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
+        AAInfo);
+    OutChains.push_back(Store);
+  }
+
+  return getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
+}
+
 SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
                                       SDValue Dst, SDValue Value, SDValue Size,
                                       Type *SizeTy, unsigned ElemSz,

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

@@ -6485,6 +6485,24 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     updateDAGForMaybeTailCall(MC);
     return;
   }
+  case Intrinsic::memset_pattern_inline: {
+    const auto &MSPII = cast<MemSetPatternInlineInst>(I);
+    SDValue Dst = getValue(I.getArgOperand(0));
+    SDValue Value = getValue(I.getArgOperand(1));
+    SDValue Size = getValue(I.getArgOperand(2));
+    assert(isa<ConstantSDNode>(Size) &&
+           "memset_pattern_inline needs constant size");
+    // @llvm.memset defines 0 and 1 to both mean no alignment.
+    Align DstAlign = MSPII.getDestAlign().valueOrOne();
+    bool isVol = MSPII.isVolatile();
+    bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
+    SDValue Root = isVol ? getRoot() : getMemoryRoot();
+    SDValue MC = DAG.getMemsetPatternInline(
+        Root, sdl, Dst, Value, Size, DstAlign, isVol, isTC,
+        MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
+    updateDAGForMaybeTailCall(MC);
+    return;
+  }
   case Intrinsic::memmove: {
     const auto &MMI = cast<MemMoveInst>(I);
     SDValue Op1 = getValue(I.getArgOperand(0));

llvm/lib/IR/Verifier.cpp

@@ -5434,7 +5434,8 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
   case Intrinsic::memcpy_inline:
   case Intrinsic::memmove:
   case Intrinsic::memset:
-  case Intrinsic::memset_inline: {
+  case Intrinsic::memset_inline:
+  case Intrinsic::memset_pattern_inline: {
     break;
   }
   case Intrinsic::memcpy_element_unordered_atomic: