Prevent trying to convert metadata references into circular project references

If you have two projects that reference each other with metadata
references, we should avoid converting both of them into project
references as that causes a circularity in the Workspace layer.
The workspace API correctly throws exceptions if you try to do this,
but the Visual Studio project system layer wasn't checking, and so it'd
call into the Workspace APIs and throw exceptions. Unfortunately this
often resulted in us getting into corrupted states where the data
structures got out of sync, which might later result in other exceptions
too.

This fix isn't entirely correct in that we're not going to re-convert
the metadata reference to the project reference if something else breaks
the circularity; that's better than crashing so this is an improvement
either way.

Fixes https://github.com/dotnet/roslyn/issues/39904
Fixes https://github.com/dotnet/roslyn/issues/43633

src/VisualStudio/Core/Def/Implementation/ProjectSystem/VisualStudioWorkspaceImpl.cs

@@ -1661,6 +1661,12 @@ private bool CanConvertMetadataReferenceToProjectReference_NoLock(ProjectId proj
                 }
             }
 
+            // If this is going to cause a circular reference, also disallow it
+            if (CurrentSolution.GetProjectDependencyGraph().GetProjectsThatThisProjectTransitivelyDependsOn(referencedProjectId).Contains(projectIdWithMetadataReference))
+            {
+                return false;
+            }
+
             return true;
         }
 

src/VisualStudio/Core/Test/ProjectSystemShim/VisualStudioProjectTests/MetadataToProjectReferenceConversionTests.vb

@@ -262,5 +262,49 @@ Namespace Microsoft.VisualStudio.LanguageServices.UnitTests.ProjectSystemShim
                 Assert.Empty(getReferencingProject().MetadataReferences)
             End Using
         End Sub
+
+        <WpfFact>
+        <WorkItem(39904, "https://github.com/dotnet/roslyn/issues/39904")>
+        Public Sub MetadataReferenceCycleDoesNotCreateProjectReferenceCycleWhenAddingReferencesFirst()
+            Using environment = New TestEnvironment()
+                Dim project1 = environment.ProjectFactory.CreateAndAddToWorkspace("project1", LanguageNames.CSharp)
+                Dim project2 = environment.ProjectFactory.CreateAndAddToWorkspace("project2", LanguageNames.CSharp)
+
+                Const ReferencePath1 = "C:\project1.dll"
+                Const ReferencePath2 = "C:\project2.dll"
+
+                project1.AddMetadataReference(ReferencePath2, MetadataReferenceProperties.Assembly)
+                project2.AddMetadataReference(ReferencePath1, MetadataReferenceProperties.Assembly)
+
+                project1.OutputFilePath = ReferencePath1
+                project2.OutputFilePath = ReferencePath2
+
+                ' Remove both from the workspace to ensure we aren't in a corrupted state somehow where removal will break further
+                project1.RemoveFromWorkspace()
+                project2.RemoveFromWorkspace()
+            End Using
+        End Sub
+
+        <WpfFact>
+        <WorkItem(39904, "https://github.com/dotnet/roslyn/issues/39904")>
+        Public Sub MetadataReferenceCycleDoesNotCreateProjectReferenceCycleWhenSettingOutputPathsFirst()
+            Using environment = New TestEnvironment()
+                Dim project1 = environment.ProjectFactory.CreateAndAddToWorkspace("project1", LanguageNames.CSharp)
+                Dim project2 = environment.ProjectFactory.CreateAndAddToWorkspace("project2", LanguageNames.CSharp)
+
+                Const ReferencePath1 = "C:\project1.dll"
+                Const ReferencePath2 = "C:\project2.dll"
+
+                project1.OutputFilePath = ReferencePath1
+                project2.OutputFilePath = ReferencePath2
+
+                project1.AddMetadataReference(ReferencePath2, MetadataReferenceProperties.Assembly)
+                project2.AddMetadataReference(ReferencePath1, MetadataReferenceProperties.Assembly)
+
+                ' Remove both from the workspace to ensure we aren't in a corrupted state somehow where removal will break further
+                project1.RemoveFromWorkspace()
+                project2.RemoveFromWorkspace()
+            End Using
+        End Sub
     End Class
 End Namespace