Fix the isEmbeddedMaskingCompatible checks (#117607)

* Fix the isEmbeddedMaskingCompatible checks

* Ensure the regression test is setup to use Xunit

* Ensure the fuzzlyn test can actually run, not always throw

Author: tannergooding

src/coreclr/jit/gentree.cpp

@@ -20737,7 +20737,7 @@ bool GenTree::isEmbeddedMaskingCompatible(Compiler* comp, unsigned tgtMaskSize,
         // Some intrinsics are effectively bitwise operations and so we
         // can freely update them to match the size of the actual mask
 
-        bool supportsMaskBaseSize4Or8 = false;
+        bool supportsMaskBaseSize2Or4 = false;
 
         switch (ins)
         {
@@ -20762,27 +20762,27 @@ bool GenTree::isEmbeddedMaskingCompatible(Compiler* comp, unsigned tgtMaskSize,
             case INS_xorpd:
             case INS_xorps:
             {
-                // These intrinsics support embedded broadcast and have masking support for 4 or 8
-                assert((maskBaseSize == 4) || (maskBaseSize == 8));
+                // These intrinsics support embedded broadcast and have masking support for 2 or 4
+                assert((maskBaseSize == 2) || (maskBaseSize == 4));
 
                 if (!comp->codeGen->IsEmbeddedBroadcastEnabled(ins, node->Op(2)))
                 {
                     // We cannot change the base type if we've already contained a broadcast
-                    supportsMaskBaseSize4Or8 = true;
+                    supportsMaskBaseSize2Or4 = true;
                 }
                 break;
             }
 
             case INS_vpternlogd:
             case INS_vpternlogq:
             {
-                // These intrinsics support embedded broadcast and have masking support for 4 or 8
-                assert((maskBaseSize == 4) || (maskBaseSize == 8));
+                // These intrinsics support embedded broadcast and have masking support for 2 or 4
+                assert((maskBaseSize == 2) || (maskBaseSize == 4));
 
                 if (!comp->codeGen->IsEmbeddedBroadcastEnabled(ins, node->Op(3)))
                 {
                     // We cannot change the base type if we've already contained a broadcast
-                    supportsMaskBaseSize4Or8 = true;
+                    supportsMaskBaseSize2Or4 = true;
                 }
                 break;
             }
@@ -20812,9 +20812,9 @@ bool GenTree::isEmbeddedMaskingCompatible(Compiler* comp, unsigned tgtMaskSize,
             case INS_vinserti64x2:
             case INS_vinserti64x4:
             {
-                // These intrinsics don't support embedded broadcast and have masking support for 4 or 8
-                assert((maskBaseSize == 4) || (maskBaseSize == 8));
-                supportsMaskBaseSize4Or8 = true;
+                // These intrinsics don't support embedded broadcast and have masking support for 2 or 4
+                assert((maskBaseSize == 2) || (maskBaseSize == 4));
+                supportsMaskBaseSize2Or4 = true;
                 break;
             }
 
@@ -20824,9 +20824,9 @@ bool GenTree::isEmbeddedMaskingCompatible(Compiler* comp, unsigned tgtMaskSize,
             }
         }
 
-        if (supportsMaskBaseSize4Or8)
+        if (supportsMaskBaseSize2Or4)
         {
-            if (tgtMaskBaseSize == 8)
+            if (tgtMaskBaseSize == 2)
             {
                 if (varTypeIsFloating(simdBaseType))
                 {

src/tests/JIT/Regression/JitBlue/Runtime_117605/Runtime_117605.cs

@@ -0,0 +1,52 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// Generated by Fuzzlyn v3.3 on 2025-07-14 11:26:52
+// Run on X64 Windows
+// Seed: 5654087083843205658-vectort,vector128,vector256,x86aes,x86avx,x86avx2,x86avx512bw,x86avx512bwvl,x86avx512cd,x86avx512cdvl,x86avx512dq,x86avx512dqvl,x86avx512f,x86avx512fvl,x86avx512fx64,x86bmi1,x86bmi1x64,x86bmi2,x86bmi2x64,x86fma,x86lzcnt,x86lzcntx64,x86pclmulqdq,x86popcnt,x86popcntx64,x86sse,x86ssex64,x86sse2,x86sse2x64,x86sse3,x86sse41,x86sse41x64,x86sse42,x86sse42x64,x86ssse3,x86x86base
+// Reduced from 33.4 KiB to 0.9 KiB in 00:01:30
+// Hits JIT assert for Release:
+// Assertion failed '(maskBaseSize == 4) || (maskBaseSize == 8)' in 'Program:M0()' during 'Rationalize IR' (IL size 87; hash 0xaf50ff37; FullOpts)
+//
+//     File: D:\a\_work\1\s\src\coreclr\jit\gentree.cpp Line: 20819
+//
+
+using System.Numerics;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using Xunit;
+
+public class Runtime_117605
+{
+    static Vector<uint>[] s_2;
+
+    [Fact]
+    public static void TestEntryPoint()
+    {
+        if (Avx2.IsSupported)
+        {
+            M0();
+        }
+    }
+
+    private static void M0()
+    {
+        var vr3 = Vector256.Create(0, 7424648407429701945UL, 0, 0);
+        var vr6 = Vector256.Create<ulong>(0);
+        var vr7 = Vector128.CreateScalar(9831122154695836571UL);
+        var vr4 = Avx2.InsertVector128(vr6, vr7, 0);
+        var vr8 = Vector256.CreateScalar(1497050855019840058UL);
+        var vr2 = Avx2.BlendVariable(vr3, vr4, vr8);
+        C0 vr9 = new C0();
+        var vr1 = vr9.M3(ref s_2, vr2);
+    }
+
+    class C0
+    {
+        public short M3(ref Vector<uint>[] arg0, Vector256<ulong> arg1)
+        {
+            var vr5 = Vector128.Create(0, -1, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0);
+            return (short)Sse2.MoveMask(vr5);
+        }
+    }
+}

src/tests/JIT/Regression/JitBlue/Runtime_117605/Runtime_117605.csproj

@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <Optimize>True</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+</Project>