[X86][MOVRS] Support MOVRS

clang/docs/ReleaseNotes.rst

@@ -752,6 +752,7 @@ X86 Support
 - Support ISA of ``AMX-MOVRS``.
 - Support ISA of ``AMX-AVX512``.
 - Support ISA of ``AMX-TF32``.
+- Support ISA of ``MOVRS``.
 
 Arm and AArch64 Support
 ^^^^^^^^^^^^^^^^^^^^^^^

clang/include/clang/Basic/BuiltinsX86.def

@@ -660,6 +660,9 @@ TARGET_BUILTIN(__builtin_ia32_vpdpbuud256, "V8iV8iV8iV8i", "ncV:256:", "avxvnnii
 TARGET_BUILTIN(__builtin_ia32_vpdpbuuds128, "V4iV4iV4iV4i", "ncV:128:", "avxvnniint8|avx10.2-256")
 TARGET_BUILTIN(__builtin_ia32_vpdpbuuds256, "V8iV8iV8iV8i", "ncV:256:", "avxvnniint8|avx10.2-256")
 
+// MOVRS
+TARGET_BUILTIN(__builtin_ia32_prefetchrs, "vvC*", "nc", "movrs")
+
 TARGET_BUILTIN(__builtin_ia32_gather3div2df, "V2dV2dvC*V2OiUcIi", "nV:128:", "avx512vl")
 TARGET_BUILTIN(__builtin_ia32_gather3div2di, "V2OiV2OivC*V2OiUcIi", "nV:128:", "avx512vl")
 TARGET_BUILTIN(__builtin_ia32_gather3div4df, "V4dV4dvC*V4OiUcIi", "nV:256:", "avx512vl")

clang/include/clang/Basic/BuiltinsX86_64.def

@@ -216,6 +216,12 @@ TARGET_BUILTIN(__builtin_ia32_aand64, "vv*SOi", "n", "raoint")
 TARGET_BUILTIN(__builtin_ia32_aor64, "vv*SOi", "n", "raoint")
 TARGET_BUILTIN(__builtin_ia32_axor64, "vv*SOi", "n", "raoint")
 
+// MOVRS
+TARGET_BUILTIN(__builtin_ia32_movrsqi, "ScvC*", "n", "movrs")
+TARGET_BUILTIN(__builtin_ia32_movrshi, "SsvC*", "n", "movrs")
+TARGET_BUILTIN(__builtin_ia32_movrssi, "SivC*", "n", "movrs")
+TARGET_BUILTIN(__builtin_ia32_movrsdi, "SLLivC*", "n", "movrs")
+
 // MOVRS and AVX10.2
 TARGET_BUILTIN(__builtin_ia32_vmovrsb128, "V16cV16cC*", "nV:128:", "movrs,avx10.2-256")
 TARGET_BUILTIN(__builtin_ia32_vmovrsb256, "V32cV32cC*", "nV:256:", "movrs,avx10.2-256")

clang/lib/Headers/CMakeLists.txt

@@ -230,6 +230,7 @@ set(x86_files
   movdirintrin.h
   movrs_avx10_2_512intrin.h
   movrs_avx10_2intrin.h
+  movrsintrin.h
   mwaitxintrin.h
   nmmintrin.h
   pconfigintrin.h

clang/lib/Headers/immintrin.h

@@ -605,6 +605,10 @@ _storebe_i64(void * __P, long long __D) {
 #include <movdirintrin.h>
 #endif
 
+#if !defined(__SCE__) || __has_feature(modules) || defined(__MOVRS__)
+#include <movrsintrin.h>
+#endif
+
 #if !defined(__SCE__) || __has_feature(modules) ||                             \
     (defined(__AVX10_2__) && defined(__MOVRS__))
 #include <movrs_avx10_2intrin.h>

clang/lib/Headers/movrsintrin.h

@@ -0,0 +1,59 @@
+/*===---------------- movrsintrin.h - MOVRS intrinsics ----------------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===----------------------------------------------------------------------===*/
+
+#ifndef __IMMINTRIN_H
+#error "Never use <movrsintrin.h> directly; include <immintrin.h> instead."
+#endif // __IMMINTRIN_H
+
+#ifndef __MOVRSINTRIN_H
+#define __MOVRSINTRIN_H
+
+#define __DEFAULT_FN_ATTRS                                                     \
+  __attribute__((__always_inline__, __nodebug__, __target__("movrs")))
+
+#ifdef __x86_64__
+static __inline__ char __DEFAULT_FN_ATTRS _movrs_i8(const void *__A) {
+  return (char)__builtin_ia32_movrsqi((const void *)__A);
+}
+
+static __inline__ short __DEFAULT_FN_ATTRS _movrs_i16(const void *__A) {
+  return (short)__builtin_ia32_movrshi((const void *)__A);
+}
+
+static __inline__ int __DEFAULT_FN_ATTRS _movrs_i32(const void *__A) {
+  return (int)__builtin_ia32_movrssi((const void *)__A);
+}
+
+static __inline__ long long __DEFAULT_FN_ATTRS _movrs_i64(const void *__A) {
+  return (long long)__builtin_ia32_movrsdi((const void *)__A);
+}
+#endif // __x86_64__
+
+// Loads a memory sequence containing the specified memory address into
+/// the L3 data cache. Data will be shared (read/written) to by requesting
+/// core and other cores.
+///
+/// Note that the effect of this intrinsic is dependent on the processor
+/// implementation.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the \c PREFETCHRS instruction.
+///
+/// \param __P
+///    A pointer specifying the memory address to be prefetched.
+static __inline__ void __DEFAULT_FN_ATTRS
+_m_prefetchrs(volatile const void *__P) {
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcast-qual"
+  __builtin_ia32_prefetchrs((const void *)__P);
+#pragma clang diagnostic pop
+}
+
+#undef __DEFAULT_FN_ATTRS
+#endif // __MOVRSINTRIN_H

clang/test/CodeGen/X86/movrs-builtins.c

@@ -0,0 +1,35 @@
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown -target-feature +movrs \
+// RUN: -emit-llvm -o - -Wall -Werror -pedantic -Wno-gnu-statement-expression | FileCheck %s
+
+#include <immintrin.h>
+#include <stddef.h>
+
+char test_movrs_si8(const char * __A) {
+  // CHECK-LABEL: @test_movrs_si8(
+  // CHECK: call i8 @llvm.x86.movrsqi(
+  return _movrs_i8(__A);
+}
+
+short test_movrs_si16(const short * __A) {
+  // CHECK-LABEL: @test_movrs_si16(
+  // CHECK: call i16 @llvm.x86.movrshi(
+  return _movrs_i16(__A);
+}
+
+int test_movrs_si32(const int * __A) {
+  // CHECK-LABEL: @test_movrs_si32(
+  // CHECK: call i32 @llvm.x86.movrssi(
+  return _movrs_i32(__A);
+}
+
+long long test_movrs_si64(const long long * __A) {
+  // CHECK-LABEL: @test_movrs_si64(
+  // CHECK: call i64 @llvm.x86.movrsdi(
+  return _movrs_i64(__A);
+}
+
+void test_m_prefetch_rs(void *p) {
+  _m_prefetchrs(p);
+  // CHECK-LABEL: define{{.*}} void @test_m_prefetch_rs
+  // CHECK: call void @llvm.x86.prefetchrs({{.*}})
+}

llvm/include/llvm/IR/IntrinsicsX86.td

@@ -7820,3 +7820,20 @@ def int_x86_avx10_vmovrsw512 : ClangBuiltin<"__builtin_ia32_vmovrsw512">,
         DefaultAttrsIntrinsic<[llvm_v32i16_ty], [llvm_ptr_ty],
                   [IntrReadMem]>;
 }
+
+let TargetPrefix = "x86" in {
+def int_x86_movrsqi : ClangBuiltin<"__builtin_ia32_movrsqi">,
+        Intrinsic<[llvm_i8_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_movrshi : ClangBuiltin<"__builtin_ia32_movrshi">,
+        Intrinsic<[llvm_i16_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_movrssi : ClangBuiltin<"__builtin_ia32_movrssi">,
+        Intrinsic<[llvm_i32_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_movrsdi : ClangBuiltin<"__builtin_ia32_movrsdi">,
+        Intrinsic<[llvm_i64_ty], [llvm_ptr_ty],
+                  [IntrReadMem]>;
+def int_x86_prefetchrs : ClangBuiltin<"__builtin_ia32_prefetchrs">,
+        Intrinsic<[], [llvm_ptr_ty], []>;
+}

llvm/lib/Target/X86/X86DiscriminateMemOps.cpp

@@ -71,7 +71,8 @@ class X86DiscriminateMemOps : public MachineFunctionPass {
 bool IsPrefetchOpcode(unsigned Opcode) {
   return Opcode == X86::PREFETCHNTA || Opcode == X86::PREFETCHT0 ||
          Opcode == X86::PREFETCHT1 || Opcode == X86::PREFETCHT2 ||
-         Opcode == X86::PREFETCHIT0 || Opcode == X86::PREFETCHIT1;
+         Opcode == X86::PREFETCHIT0 || Opcode == X86::PREFETCHIT1 ||
+         Opcode == X86::PREFETCHRST2;
 }
 } // end anonymous namespace
 

llvm/lib/Target/X86/X86InstrMisc.td

@@ -1727,3 +1727,27 @@ def CLWB       : I<0xAE, MRM6m, (outs), (ins i8mem:$src), "clwb\t$src",
 let Predicates = [HasCLDEMOTE], SchedRW = [WriteLoad] in
 def CLDEMOTE : I<0x1C, MRM0m, (outs), (ins i8mem:$src), "cldemote\t$src",
                    [(int_x86_cldemote addr:$src)]>, TB;
+
+//===----------------------------------------------------------------------===//
+// MOVRS Instructions
+//
+
+let SchedRW = [WriteLoad] in {
+let Predicates = [HasMOVRS, NoEGPR] in {
+def MOVRS8rm     : I<0x8A, MRMSrcMem, (outs GR8 :$dst), (ins i8mem :$src),
+                   "movrs{b}\t{$src, $dst|$dst, $src}",
+                   [(set GR8:$dst, (int_x86_movrsqi addr:$src))]>, T8;
+def MOVRS16rm    : I<0x8B, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src),
+                   "movrs{w}\t{$src, $dst|$dst, $src}",
+                   [(set GR16:$dst, (int_x86_movrshi addr:$src))]>, OpSize16, T8;
+def MOVRS32rm    : I<0x8B, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src),
+                   "movrs{l}\t{$src, $dst|$dst, $src}",
+                   [(set GR32:$dst, (int_x86_movrssi addr:$src))]>, OpSize32, T8;
+def MOVRS64rm    : RI<0x8B, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src),
+                   "movrs{q}\t{$src, $dst|$dst, $src}",
+                   [(set GR64:$dst, (int_x86_movrsdi addr:$src))]>, T8;
+def PREFETCHRST2 : I<0x18, MRM4m, (outs), (ins i8mem:$src),
+                   "prefetchrst2\t$src",
+                   [(int_x86_prefetchrs addr:$src)]>, TB;
+}
+}

llvm/test/CodeGen/X86/movrs-builtins.ll

@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown --show-mc-encoding -mattr=+movrs | FileCheck %s
+
+define i8 @test_movrs_si8(ptr %__A) {
+; CHECK-LABEL: test_movrs_si8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movrsb (%rdi), %al # encoding: [0x0f,0x38,0x8a,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = call i8 @llvm.x86.movrsqi(ptr %__A)
+  ret i8 %0
+}
+declare i8 @llvm.x86.movrsqi(ptr)
+
+define i16 @test_movrs_si16(ptr %__A) {
+; CHECK-LABEL: test_movrs_si16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movrsw (%rdi), %ax # encoding: [0x66,0x0f,0x38,0x8b,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = call i16 @llvm.x86.movrshi(ptr %__A)
+  ret i16 %0
+}
+declare i16 @llvm.x86.movrshi(ptr)
+
+define i32 @test_movrs_si32(ptr %__A) {
+; CHECK-LABEL: test_movrs_si32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movrsl (%rdi), %eax # encoding: [0x0f,0x38,0x8b,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = call i32 @llvm.x86.movrssi(ptr %__A)
+  ret i32 %0
+}
+declare i32 @llvm.x86.movrssi(ptr)
+
+define i64 @test_movrs_si64(ptr %__A) {
+; CHECK-LABEL: test_movrs_si64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movrsq (%rdi), %rax # encoding: [0x48,0x0f,0x38,0x8b,0x07]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+entry:
+  %0 = call i64 @llvm.x86.movrsdi(ptr %__A)
+  ret i64 %0
+}
+declare i64 @llvm.x86.movrsdi(ptr)

llvm/test/CodeGen/X86/movrs-prefetch-builtins.ll

@@ -0,0 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse,+prfchw,+movrs | FileCheck %s
+; RUN: llc < %s -mtriple=i686-- -mattr=-sse,+prfchw,+movrs | FileCheck %s
+
+define void @t(ptr %ptr) nounwind  {
+; CHECK-LABEL: t:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    prefetchrst2 (%eax)
+; CHECK-NEXT:    retl
+entry:
+  tail call void @llvm.x86.prefetchrs( ptr %ptr )
+  ret void
+}

llvm/test/MC/Disassembler/X86/movrs.txt

@@ -0,0 +1,98 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=x86_64 --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+# ATT:   movrsb  268435456(%rbp,%r14,8), %bl
+# INTEL: movrs bl, byte ptr [rbp + 8*r14 + 268435456]
+0x42,0x0f,0x38,0x8a,0x9c,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   movrsb  291(%r8,%rax,4), %bl
+# INTEL: movrs bl, byte ptr [r8 + 4*rax + 291]
+0x41,0x0f,0x38,0x8a,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   movrsb  (%rip), %bl
+# INTEL: movrs bl, byte ptr [rip]
+0x0f,0x38,0x8a,0x1d,0x00,0x00,0x00,0x00
+
+# ATT:   movrsb  -32(,%rbp,2), %bl
+# INTEL: movrs bl, byte ptr [2*rbp - 32]
+0x0f,0x38,0x8a,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# ATT:   movrsb  127(%rcx), %bl
+# INTEL: movrs bl, byte ptr [rcx + 127]
+0x0f,0x38,0x8a,0x59,0x7f
+
+# ATT:   movrsb  -128(%rdx), %bl
+# INTEL: movrs bl, byte ptr [rdx - 128]
+0x0f,0x38,0x8a,0x5a,0x80
+
+# ATT:   movrsw  268435456(%rbp,%r14,8), %bx
+# INTEL: movrs bx, word ptr [rbp + 8*r14 + 268435456]
+0x66,0x42,0x0f,0x38,0x8b,0x9c,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   movrsw  291(%r8,%rax,4), %bx
+# INTEL: movrs bx, word ptr [r8 + 4*rax + 291]
+0x66,0x41,0x0f,0x38,0x8b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   movrsw  (%rip), %bx
+# INTEL: movrs bx, word ptr [rip]
+0x66,0x0f,0x38,0x8b,0x1d,0x00,0x00,0x00,0x00
+
+# ATT:   movrsw  -32(,%rbp,2), %bx
+# INTEL: movrs bx, word ptr [2*rbp - 32]
+0x66,0x0f,0x38,0x8b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# ATT:   movrsw  127(%rcx), %bx
+# INTEL: movrs bx, word ptr [rcx + 127]
+0x66,0x0f,0x38,0x8b,0x59,0x7f
+
+# ATT:   movrsw  -128(%rdx), %bx
+# INTEL: movrs bx, word ptr [rdx - 128]
+0x66,0x0f,0x38,0x8b,0x5a,0x80
+
+# ATT:   movrsl  268435456(%rbp,%r14,8), %ebx
+# INTEL: movrs ebx, dword ptr [rbp + 8*r14 + 268435456]
+0x42,0x0f,0x38,0x8b,0x9c,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   movrsl  291(%r8,%rax,4), %ebx
+# INTEL: movrs ebx, dword ptr [r8 + 4*rax + 291]
+0x41,0x0f,0x38,0x8b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   movrsl  (%rip), %ebx
+# INTEL: movrs ebx, dword ptr [rip]
+0x0f,0x38,0x8b,0x1d,0x00,0x00,0x00,0x00
+
+# ATT:   movrsl  -32(,%rbp,2), %ebx
+# INTEL: movrs ebx, dword ptr [2*rbp - 32]
+0x0f,0x38,0x8b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# ATT:   movrsl  127(%rcx), %ebx
+# INTEL: movrs ebx, dword ptr [rcx + 127]
+0x0f,0x38,0x8b,0x59,0x7f
+
+# ATT:   movrsl  -128(%rdx), %ebx
+# INTEL: movrs ebx, dword ptr [rdx - 128]
+0x0f,0x38,0x8b,0x5a,0x80
+
+# ATT:   movrsq  268435456(%rbp,%r14,8), %rbx
+# INTEL: movrs rbx, qword ptr [rbp + 8*r14 + 268435456]
+0x4a,0x0f,0x38,0x8b,0x9c,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   movrsq  291(%r8,%rax,4), %rbx
+# INTEL: movrs rbx, qword ptr [r8 + 4*rax + 291]
+0x49,0x0f,0x38,0x8b,0x9c,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   movrsq  (%rip), %rbx
+# INTEL: movrs rbx, qword ptr [rip]
+0x48,0x0f,0x38,0x8b,0x1d,0x00,0x00,0x00,0x00
+
+# ATT:   movrsq  -32(,%rbp,2), %rbx
+# INTEL: movrs rbx, qword ptr [2*rbp - 32]
+0x48,0x0f,0x38,0x8b,0x1c,0x6d,0xe0,0xff,0xff,0xff
+
+# ATT:   movrsq  127(%rcx), %rbx
+# INTEL: movrs rbx, qword ptr [rcx + 127]
+0x48,0x0f,0x38,0x8b,0x59,0x7f
+
+# ATT:   movrsq  -128(%rdx), %rbx
+# INTEL: movrs rbx, qword ptr [rdx - 128]
+0x48,0x0f,0x38,0x8b,0x5a,0x80

llvm/test/MC/Disassembler/X86/prefetchrst2-32.txt

@@ -0,0 +1,26 @@
+# RUN: llvm-mc --disassemble %s -triple=i386 | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=i386 --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+# ATT:   prefetchrst2  268435456(%esp,%esi,8)
+# INTEL: prefetchrst2 byte ptr [esp + 8*esi + 268435456]
+0x0f,0x18,0xa4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   prefetchrst2  291(%edi,%eax,4)
+# INTEL: prefetchrst2 byte ptr [edi + 4*eax + 291]
+0x0f,0x18,0xa4,0x87,0x23,0x01,0x00,0x00
+
+# ATT:   prefetchrst2  (%eax)
+# INTEL: prefetchrst2 byte ptr [eax]
+0x0f,0x18,0x20
+
+# ATT:   prefetchrst2  -32(,%ebp,2)
+# INTEL: prefetchrst2 byte ptr [2*ebp - 32]
+0x0f,0x18,0x24,0x6d,0xe0,0xff,0xff,0xff
+
+# ATT:   prefetchrst2  127(%ecx)
+# INTEL: prefetchrst2 byte ptr [ecx + 127]
+0x0f,0x18,0x61,0x7f
+
+# ATT:   prefetchrst2  -128(%edx)
+# INTEL: prefetchrst2 byte ptr [edx - 128]
+0x0f,0x18,0x62,0x80