Add cycle detection for more scenarios involving looking in base interfaces. (#38738)

Fixes #38735.

src/Compilers/CSharp/Portable/Binder/Binder_Lookup.cs

@@ -951,6 +951,8 @@ private static ImmutableArray<NamedTypeSymbol> GetBaseInterfaces(NamedTypeSymbol
                 return ImmutableArray<NamedTypeSymbol>.Empty;
             }
 
+            var cycleGuard = ConsList<NamedTypeSymbol>.Empty.Prepend(type.OriginalDefinition);
+
             // Consumers of the result depend on the sorting performed by AllInterfacesWithDefinitionUseSiteDiagnostics.
             // Let's use similar sort algorithm.
             var result = ArrayBuilder<NamedTypeSymbol>.GetInstance();
@@ -958,7 +960,7 @@ private static ImmutableArray<NamedTypeSymbol> GetBaseInterfaces(NamedTypeSymbol
 
             for (int i = interfaces.Length - 1; i >= 0; i--)
             {
-                addAllInterfaces(interfaces[i], visited, result, basesBeingResolved);
+                addAllInterfaces(interfaces[i], visited, result, basesBeingResolved, cycleGuard);
             }
 
             result.ReverseContents();
@@ -970,17 +972,24 @@ private static ImmutableArray<NamedTypeSymbol> GetBaseInterfaces(NamedTypeSymbol
 
             return result.ToImmutableAndFree();
 
-            static void addAllInterfaces(NamedTypeSymbol @interface, HashSet<NamedTypeSymbol> visited, ArrayBuilder<NamedTypeSymbol> result, ConsList<TypeSymbol> basesBeingResolved)
+            static void addAllInterfaces(NamedTypeSymbol @interface, HashSet<NamedTypeSymbol> visited, ArrayBuilder<NamedTypeSymbol> result, ConsList<TypeSymbol> basesBeingResolved, ConsList<NamedTypeSymbol> cycleGuard)
             {
-                if (@interface.IsInterface && visited.Add(@interface))
+                NamedTypeSymbol originalDefinition;
+
+                if (@interface.IsInterface && !cycleGuard.ContainsReference(originalDefinition = @interface.OriginalDefinition) && visited.Add(@interface))
                 {
-                    if (!basesBeingResolved.ContainsReference(@interface.OriginalDefinition))
+                    if (!basesBeingResolved.ContainsReference(originalDefinition))
                     {
                         ImmutableArray<NamedTypeSymbol> baseInterfaces = @interface.GetDeclaredInterfaces(basesBeingResolved);
-                        for (int i = baseInterfaces.Length - 1; i >= 0; i--)
+
+                        if (!baseInterfaces.IsEmpty)
                         {
-                            var baseInterface = baseInterfaces[i];
-                            addAllInterfaces(baseInterface, visited, result, basesBeingResolved);
+                            cycleGuard = cycleGuard.Prepend(originalDefinition);
+                            for (int i = baseInterfaces.Length - 1; i >= 0; i--)
+                            {
+                                var baseInterface = baseInterfaces[i];
+                                addAllInterfaces(baseInterface, visited, result, basesBeingResolved, cycleGuard);
+                            }
                         }
                     }
 

src/Compilers/CSharp/Test/Symbol/Symbols/DefaultInterfaceImplementationTests.cs

@@ -29512,6 +29512,37 @@ public interface I14 : I11.I13
                 );
         }
 
+        [Fact]
+        [WorkItem(38735, "https://github.com/dotnet/roslyn/issues/38735")]
+        public void NestedTypes_52()
+        {
+            var source1 =
+@"
+interface I1 : IA<int>.NF { }
+
+interface IQ<T> { }
+
+interface IA<T> : IB<IQ<T>> { }
+
+interface IB<T> : IA<IQ<T>> { }
+";
+
+            var compilation1 = CreateCompilation(source1, options: TestOptions.DebugDll,
+                                                 parseOptions: TestOptions.Regular);
+
+            compilation1.VerifyDiagnostics(
+                // (2,24): error CS0426: The type name 'NF' does not exist in the type 'IA<int>'
+                // interface I1 : IA<int>.NF { }
+                Diagnostic(ErrorCode.ERR_DottedTypeNameNotFoundInAgg, "NF").WithArguments("NF", "IA<int>").WithLocation(2, 24),
+                // (6,11): error CS0529: Inherited interface 'IB<IQ<T>>' causes a cycle in the interface hierarchy of 'IA<T>'
+                // interface IA<T> : IB<IQ<T>> { }
+                Diagnostic(ErrorCode.ERR_CycleInInterfaceInheritance, "IA").WithArguments("IA<T>", "IB<IQ<T>>").WithLocation(6, 11),
+                // (8,11): error CS0529: Inherited interface 'IA<IQ<T>>' causes a cycle in the interface hierarchy of 'IB<T>'
+                // interface IB<T> : IA<IQ<T>> { }
+                Diagnostic(ErrorCode.ERR_CycleInInterfaceInheritance, "IB").WithArguments("IB<T>", "IA<IQ<T>>").WithLocation(8, 11)
+                );
+        }
+
         [Fact]
         [WorkItem(32540, "https://github.com/dotnet/roslyn/issues/32540")]
         public void MethodImplementationInDerived_01()