`DisableRuntimeMarshalling` block Vector types (#74612)

* DisableRuntimeMarshalling block Vector types

When DisableRuntimeMarshalling is set, the runtime should block
a bad-list for blittable types.

This doesn't address the case where the P/Invoke is inlined.

src/coreclr/tools/Common/TypeSystem/Interop/IL/MarshalHelpers.cs

@@ -411,26 +411,9 @@ public static partial class MarshalHelpers
                     return MarshallerKind.Invalid;
                 }
 
-                bool isBlittable = MarshalUtils.IsBlittableType(type);
-
-                // Blittable generics are allowed to be marshalled with the following exceptions:
-                // * Nullable<T>: We don't want to be locked into the default behavior as we may want special handling later
-                // * Vector64<T>: Represents the __m64 ABI primitive which requires currently unimplemented handling
-                // * Vector128<T>: Represents the __m128 ABI primitive which requires currently unimplemented handling
-                // * Vector256<T>: Represents the __m256 ABI primitive which requires currently unimplemented handling
-                // * Vector<T>: Has a variable size (either __m128 or __m256) and isn't readily usable for interop scenarios
-                // We can't block these types for field scenarios for back-compat reasons.
-
-                if (type.HasInstantiation && !isField && (!isBlittable
-                    || InteropTypes.IsSystemSpan(context, type)
-                    || InteropTypes.IsSystemReadOnlySpan(context, type)
-                    || InteropTypes.IsSystemNullable(context, type)
-                    || InteropTypes.IsSystemRuntimeIntrinsicsVector64T(context, type)
-                    || InteropTypes.IsSystemRuntimeIntrinsicsVector128T(context, type)
-                    || InteropTypes.IsSystemRuntimeIntrinsicsVector256T(context, type)
-                    || InteropTypes.IsSystemNumericsVectorT(context, type)))
+                if (!IsValidForGenericMarshalling(type, isField))
                 {
-                    // Generic types cannot be marshaled.
+                    // Generic types cannot be marshalled.
                     return MarshallerKind.Invalid;
                 }
 
@@ -440,7 +423,7 @@ public static partial class MarshalHelpers
                     return MarshallerKind.Invalid;
                 }
 
-                if (isBlittable)
+                if (MarshalUtils.IsBlittableType(type))
                 {
                     if (nativeType != NativeTypeKind.Default && nativeType != NativeTypeKind.Struct)
                         return MarshallerKind.Invalid;
@@ -868,8 +851,43 @@ public static partial class MarshalHelpers
             }
         }
 
+        private static bool IsValidForGenericMarshalling(
+            TypeDesc type,
+            bool isFieldScenario,
+            bool builtInMarshallingEnabled = true)
+        {
+            // Not generic, so passes "generic" test
+            if (!type.HasInstantiation)
+                return true;
+
+            // We can't block generic types for field scenarios for back-compat reasons.
+            if (isFieldScenario)
+                return true;
+
+            // Built-in marshalling considers the blittability for a generic type.
+            if (builtInMarshallingEnabled && !MarshalUtils.IsBlittableType(type))
+                return false;
+
+            // Generics (blittable when built-in is enabled) are allowed to be marshalled with the following exceptions:
+            // * Nullable<T>: We don't want to be locked into the default behavior as we may want special handling later
+            // * Span<T>: Not supported by built-in marshalling
+            // * ReadOnlySpan<T>: Not supported by built-in marshalling
+            // * Vector64<T>: Represents the __m64 ABI primitive which requires currently unimplemented handling
+            // * Vector128<T>: Represents the __m128 ABI primitive which requires currently unimplemented handling
+            // * Vector256<T>: Represents the __m256 ABI primitive which requires currently unimplemented handling
+            // * Vector<T>: Has a variable size (either __m128 or __m256) and isn't readily usable for interop scenarios
+            return !InteropTypes.IsSystemNullable(type.Context, type)
+                && !InteropTypes.IsSystemSpan(type.Context, type)
+                && !InteropTypes.IsSystemReadOnlySpan(type.Context, type)
+                && !InteropTypes.IsSystemRuntimeIntrinsicsVector64T(type.Context, type)
+                && !InteropTypes.IsSystemRuntimeIntrinsicsVector128T(type.Context, type)
+                && !InteropTypes.IsSystemRuntimeIntrinsicsVector256T(type.Context, type)
+                && !InteropTypes.IsSystemNumericsVectorT(type.Context, type);
+        }
+
         internal static MarshallerKind GetDisabledMarshallerKind(
-            TypeDesc type)
+            TypeDesc type,
+            bool isFieldScenario)
         {
             // Get the underlying type for enum types.
             TypeDesc underlyingType = type.UnderlyingType;
@@ -893,7 +911,10 @@ public static partial class MarshalHelpers
             else if (underlyingType.IsValueType)
             {
                 var defType = (DefType)underlyingType;
-                if (!defType.ContainsGCPointers && !defType.IsAutoLayoutOrHasAutoLayoutFields && !defType.IsInt128OrHasInt128Fields)
+                if (!defType.ContainsGCPointers
+                    && !defType.IsAutoLayoutOrHasAutoLayoutFields
+                    && !defType.IsInt128OrHasInt128Fields
+                    && IsValidForGenericMarshalling(defType, isFieldScenario, builtInMarshallingEnabled: false))
                 {
                     return MarshallerKind.BlittableValue;
                 }

src/coreclr/tools/Common/TypeSystem/Interop/IL/Marshaller.cs

@@ -344,7 +344,7 @@ protected Marshaller()
             PInvokeFlags flags,
             bool isReturn)
         {
-            MarshallerKind marshallerKind = MarshalHelpers.GetDisabledMarshallerKind(parameterType);
+            MarshallerKind marshallerKind = MarshalHelpers.GetDisabledMarshallerKind(parameterType, marshallerType is MarshallerType.Field);
 
             TypeSystemContext context = parameterType.Context;
             // Create the marshaller based on MarshallerKind

src/coreclr/vm/mlinfo.cpp

@@ -1058,10 +1058,44 @@ OleColorMarshalingInfo *EEMarshalingData::GetOleColorMarshalingInfo()
 
 namespace
 {
+    bool IsValidForGenericMarshalling(MethodTable* pMT, bool isFieldScenario, bool builtInMarshallingEnabled = true)
+    {
+        _ASSERTE(pMT != NULL);
+
+        // Not generic, so passes "generic" test
+        if (!pMT->HasInstantiation())
+            return true;
+
+        // We can't block generic types for field scenarios for back-compat reasons.
+        if (isFieldScenario)
+            return true;
+
+        // Built-in marshalling considers the blittability for a generic type.
+        if (builtInMarshallingEnabled && !pMT->IsBlittable())
+            return false;
+
+        // Generics (blittable when built-in is enabled) are allowed to be marshalled with the following exceptions:
+        // * Nullable<T>: We don't want to be locked into the default behavior as we may want special handling later
+        // * Span<T>: Not supported by built-in marshalling
+        // * ReadOnlySpan<T>: Not supported by built-in marshalling
+        // * Vector64<T>: Represents the __m64 ABI primitive which requires currently unimplemented handling
+        // * Vector128<T>: Represents the __m128 ABI primitive which requires currently unimplemented handling
+        // * Vector256<T>: Represents the __m256 ABI primitive which requires currently unimplemented handling
+        // * Vector<T>: Has a variable size (either __m128 or __m256) and isn't readily usable for interop scenarios
+        return !pMT->HasSameTypeDefAs(g_pNullableClass)
+            && !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__SPAN))
+            && !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__READONLY_SPAN))
+            && !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR64T))
+            && !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR128T))
+            && !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR256T))
+            && !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTORT));
+    }
+
     MarshalInfo::MarshalType GetDisabledMarshallerType(
         Module* pModule,
         SigPointer sig,
-        const SigTypeContext * pTypeContext,
+        const SigTypeContext* pTypeContext,
+        bool isFieldScenario,
         MethodTable** pMTOut,
         UINT* errorResIDOut)
     {
@@ -1131,6 +1165,11 @@ namespace
                     *errorResIDOut = IDS_EE_BADMARSHAL_AUTOLAYOUT;
                     return MarshalInfo::MARSHAL_TYPE_UNKNOWN;
                 }
+                if (!IsValidForGenericMarshalling(pMT, isFieldScenario, false /* builtInMarshallingEnabled */))
+                {
+                    *errorResIDOut = IDS_EE_BADMARSHAL_GENERICS_RESTRICTION;
+                    return MarshalInfo::MARSHAL_TYPE_UNKNOWN;
+                }
                 if (pMT->IsInt128OrHasInt128Fields())
                 {
                     *errorResIDOut = IDS_EE_BADMARSHAL_INT128_RESTRICTION;
@@ -1273,6 +1312,7 @@ MarshalInfo::MarshalInfo(Module* pModule,
             pModule,
             sig,
             pTypeContext,
+            IsFieldScenario(),
             &m_pMT,
             &m_resID);
         m_args.m_pMT = m_pMT;
@@ -2266,23 +2306,7 @@ MarshalInfo::MarshalInfo(Module* pModule,
                 if (m_pMT == NULL)
                     break;
 
-                // Blittable generics are allowed to be marshalled with the following exceptions:
-                // * Nullable<T>: We don't want to be locked into the default behavior as we may want special handling later
-                // * Vector64<T>: Represents the __m64 ABI primitive which requires currently unimplemented handling
-                // * Vector128<T>: Represents the __m128 ABI primitive which requires currently unimplemented handling
-                // * Vector256<T>: Represents the __m256 ABI primitive which requires currently unimplemented handling
-                // * Vector<T>: Has a variable size (either __m128 or __m256) and isn't readily usable for interop scenarios
-                // We can't block these types for field scenarios for back-compat reasons.
-                if (m_pMT->HasInstantiation() && !IsFieldScenario()
-                    && (!m_pMT->IsBlittable()
-                        || (m_pMT->HasSameTypeDefAs(g_pNullableClass)
-                        || m_pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__SPAN))
-                        || m_pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__READONLY_SPAN))
-                        || m_pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR64T))
-                        || m_pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR128T))
-                        || m_pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR256T))
-                        || m_pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTORT))
-                    )))
+                if (!IsValidForGenericMarshalling(m_pMT, IsFieldScenario()))
                 {
                     m_resID = IDS_EE_BADMARSHAL_GENERICS_RESTRICTION;
                     IfFailGoto(E_FAIL, lFail);

src/tests/Interop/DisabledRuntimeMarshalling/Native_DisabledMarshalling/DisabledRuntimeMarshallingNative.cs

@@ -2,7 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Numerics;
 using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics;
 using Xunit;
 
 public unsafe class DisabledRuntimeMarshallingNative
@@ -178,6 +180,21 @@ public enum ByteEnum : byte
     [return:MarshalAs(UnmanagedType.U1)]
     public static extern bool CheckStructWithShortAndBoolWithVariantBool_FailureExpected(StructWithShortAndBool str, short s, [MarshalAs(UnmanagedType.VariantBool)] bool b);
 
+    [DllImport(nameof(DisabledRuntimeMarshallingNative), EntryPoint = "Invalid")]
+    public static extern void CallWith(Nullable<int> s);
+    [DllImport(nameof(DisabledRuntimeMarshallingNative), EntryPoint = "Invalid")]
+    public static extern void CallWith(Span<int> s);
+    [DllImport(nameof(DisabledRuntimeMarshallingNative), EntryPoint = "Invalid")]
+    public static extern void CallWith(ReadOnlySpan<int> ros);
+    [DllImport(nameof(DisabledRuntimeMarshallingNative), EntryPoint = "Invalid")]
+    public static extern void CallWith(Vector64<int> v);
+    [DllImport(nameof(DisabledRuntimeMarshallingNative), EntryPoint = "Invalid")]
+    public static extern void CallWith(Vector128<int> v);
+    [DllImport(nameof(DisabledRuntimeMarshallingNative), EntryPoint = "Invalid")]
+    public static extern void CallWith(Vector256<int> v);
+    [DllImport(nameof(DisabledRuntimeMarshallingNative), EntryPoint = "Invalid")]
+    public static extern void CallWith(Vector<int> v);
+
     // Apply the UnmanagedFunctionPointer attributes with the default calling conventions so that Mono's AOT compiler
     // recognizes that these delegate types are used in interop and should have managed->native thunks generated for them.
     [UnmanagedFunctionPointer(CallingConvention.Winapi)]

src/tests/Interop/DisabledRuntimeMarshalling/PInvokeAssemblyMarshallingDisabled/Generics.cs

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Numerics;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics;
+using Xunit;
+using static DisabledRuntimeMarshallingNative;
+
+namespace DisabledRuntimeMarshalling;
+
+public unsafe class Generics
+{
+    [Fact]
+    public static void BlittableGeneric_NotSupported()
+    {
+        Assert.Throws<MarshalDirectiveException>(() =>
+        {
+            Nullable<int> s = default;
+            DisabledRuntimeMarshallingNative.CallWith(s);
+        });
+        Assert.Throws<MarshalDirectiveException>(() =>
+        {
+            Span<int> s = default;
+            DisabledRuntimeMarshallingNative.CallWith(s);
+        });
+        Assert.Throws<MarshalDirectiveException>(() =>
+        {
+            ReadOnlySpan<int> ros = default;
+            DisabledRuntimeMarshallingNative.CallWith(ros);
+        });
+        Assert.Throws<MarshalDirectiveException>(() =>
+        {
+            Vector64<int> v = default;
+            DisabledRuntimeMarshallingNative.CallWith(v);
+        });
+        Assert.Throws<MarshalDirectiveException>(() =>
+        {
+            Vector128<int> v = default;
+            DisabledRuntimeMarshallingNative.CallWith(v);
+        });
+        Assert.Throws<MarshalDirectiveException>(() =>
+        {
+            Vector256<int> v = default;
+            DisabledRuntimeMarshallingNative.CallWith(v);
+        });
+        Assert.Throws<MarshalDirectiveException>(() =>
+        {
+            Vector<int> v = default;
+            DisabledRuntimeMarshallingNative.CallWith(v);
+        });
+    }
+}