Remove HashSet allocations in IsMutableValueType

This method is only recursive in one limited scenario, which is now
specially handled to avoid the need to track a visited set.

src/EditorFeatures/CSharpTest/UseAutoProperty/UseAutoPropertyTests.cs

@@ -131,6 +131,28 @@ public async Task TestNullable4()
 }");
         }
 
+        [Fact, Trait(Traits.Feature, Traits.Features.CodeActionsUseAutoProperty)]
+        [WorkItem(28511, "https://github.com/dotnet/roslyn/issues/28511")]
+        public async Task TestNullable5()
+        {
+            // Recursive type check
+            await TestMissingInRegularAndScriptAsync(
+@"using System;
+class Class
+{
+    [|Nullable<MutableInt?> i|];
+
+    Nullable<MutableInt?> P
+    {
+        get
+        {
+            return i;
+        }
+    }
+}
+struct MutableInt { public int Value; }");
+        }
+
         [Fact, Trait(Traits.Feature, Traits.Features.CodeActionsUseAutoProperty)]
         [WorkItem(28511, "https://github.com/dotnet/roslyn/issues/28511")]
         public async Task TestMutableValueType1()

src/EditorFeatures/VisualBasicTest/UseAutoProperty/UseAutoPropertyTests.vb

@@ -137,6 +137,25 @@ end class",
 end class")
         End Function
 
+        <Fact, Trait(Traits.Feature, Traits.Features.CodeActionsUseAutoProperty)>
+        <WorkItem(28511, "https://github.com/dotnet/roslyn/issues/28511")>
+        Public Async Function TestNullable5() As Task
+            ' Recursive type check
+            Await TestMissingInRegularAndScriptAsync(
+"Imports System
+class Class1
+    [|dim i as Nullable(Of Nullable(Of MutableInt))|]
+    readonly property P as Nullable(Of Nullable(Of MutableInt))
+        get
+            return i
+        end get
+    end property
+end class
+Structure MutableInt
+    Public Value As Integer
+End Structure")
+        End Function
+
         <Fact, Trait(Traits.Feature, Traits.Features.CodeActionsUseAutoProperty)>
         <WorkItem(28511, "https://github.com/dotnet/roslyn/issues/28511")>
         Public Async Function TestMutableValueType1() As Task

src/Workspaces/Core/Portable/Shared/Extensions/ITypeSymbolExtensions.cs

@@ -856,16 +856,7 @@ public static bool IsEnumType(this ITypeSymbol type)
         }
 
         public static bool? IsMutableValueType(this ITypeSymbol type)
-            => IsMutableValueType(type, visitedOpt: null);
-
-        private static bool? IsMutableValueType(ITypeSymbol type, HashSet<ITypeSymbol> visitedOpt)
         {
-            if (visitedOpt != null && !visitedOpt.Add(type))
-            {
-                // Recursive type, unable to determine
-                return null;
-            }
-
             if (type.IsErrorType())
             {
                 return null;
@@ -880,7 +871,20 @@ public static bool IsEnumType(this ITypeSymbol type)
             {
                 // Nullable<T> can only be mutable if T is mutable. This case ensures types like 'int?' are treated as
                 // immutable.
-                return IsMutableValueType(type.GetTypeArguments()[0], visitedOpt ?? new HashSet<ITypeSymbol> { type });
+                var typeArguments = type.GetTypeArguments();
+                if (typeArguments.Length != 1)
+                {
+                    return null;
+                }
+
+                if (typeArguments[0].IsNullable())
+                {
+                    // Recursion prevention. This is not a valid type anyway since the T in Nullable<T> cannot itself be
+                    // nullable.
+                    return null;
+                }
+
+                return typeArguments[0].IsMutableValueType();
             }
 
             foreach (var member in type.GetMembers())