a fix that allows for only one particular edge case scenario

Author: adamsitnik

src/libraries/System.Formats.Nrbf/src/System/Formats/Nrbf/MemberTypeInfo.cs

@@ -91,9 +91,7 @@ internal static MemberTypeInfo Decode(BinaryReader reader, int count, PayloadOpt
         const AllowedRecordTypes SystemClass = Classes | AllowedRecordTypes.SystemClassWithMembersAndTypes
             // All primitive types can be stored by using one of the interfaces they implement.
             // Example: `new IEnumerable[1] { "hello" }` or `new IComparable[1] { int.MaxValue }`.
-            | AllowedRecordTypes.BinaryObjectString | AllowedRecordTypes.MemberPrimitiveTyped
-            // System.Nullable<UserStruct> is a special case of SystemClassWithMembersAndTypes
-            | AllowedRecordTypes.ClassWithMembersAndTypes;
+            | AllowedRecordTypes.BinaryObjectString | AllowedRecordTypes.MemberPrimitiveTyped;
         const AllowedRecordTypes NonSystemClass = Classes | AllowedRecordTypes.ClassWithMembersAndTypes;
 
         return binaryType switch
@@ -104,10 +102,29 @@ internal static MemberTypeInfo Decode(BinaryReader reader, int count, PayloadOpt
             BinaryType.StringArray => (StringArray, default),
             BinaryType.PrimitiveArray => (PrimitiveArray, default),
             BinaryType.Class => (NonSystemClass, default),
+            BinaryType.SystemClass when IsNullableUserTypeRepresentedAsSystemType((TypeName)additionalInfo!) => (SystemClass | AllowedRecordTypes.ClassWithMembersAndTypes, default),
             BinaryType.SystemClass => (SystemClass, default),
             BinaryType.ObjectArray => (ObjectArray, default),
             _ => throw new InvalidOperationException()
         };
+
+        static bool IsNullableUserTypeRepresentedAsSystemType(TypeName typeName)
+        {
+            if (!typeName.IsConstructedGenericType || typeName.Name != typeof(Nullable<>).Name)
+            {
+                return false;
+            }
+
+            var genericArgs = typeName.GetGenericArguments();
+            if (genericArgs.Length != 1)
+            {
+                return false;
+            }
+
+            // If the generic argument is not from CoreLib, then it is a user type.
+            var assemblyName = genericArgs[0].AssemblyName;
+            return assemblyName is not null && !assemblyName.FullName.Equals(TypeNameHelpers.CoreLibAssemblyName.FullName, StringComparison.Ordinal);
+        }
     }
 
     internal TypeName GetArrayTypeName(ArrayInfo arrayInfo)

src/libraries/System.Formats.Nrbf/src/System/Formats/Nrbf/Utils/TypeNameHelpers.cs

@@ -21,6 +21,9 @@ internal static class TypeNameHelpers
     private static readonly TypeName?[] s_primitiveSZArrayTypeNames = new TypeName?[(int)UIntPtrPrimitiveType + 1];
     private static AssemblyNameInfo? s_coreLibAssemblyName;
 
+    internal static AssemblyNameInfo CoreLibAssemblyName
+        => s_coreLibAssemblyName ??= AssemblyNameInfo.Parse("mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089".AsSpan());
+
     internal static TypeName GetPrimitiveTypeName(PrimitiveType primitiveType)
     {
         TypeName? typeName = s_primitiveTypeNames[(int)primitiveType];
@@ -155,7 +158,7 @@ internal static TypeName ParseSystemRecordTypeName(this string rawName, PayloadO
                 .WithCoreLibAssemblyName(); // We know it's a System Record, so we set the LibraryName to CoreLib
 
     internal static TypeName WithCoreLibAssemblyName(this TypeName systemType)
-        => systemType.With(s_coreLibAssemblyName ??= AssemblyNameInfo.Parse("mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089".AsSpan()));
+        => systemType.With(CoreLibAssemblyName);
 
     private static TypeName With(this TypeName typeName, AssemblyNameInfo assemblyName)
     {