Make sure we convert project references back on project removal

If we don't do this, projects will end up with dangling project
references, which isn't something we want. Even if the project later
reloads, the fact there's still old ProjectIds floating around causes
us to fail to convert to P2P references again later.

Fixes https://devdiv.visualstudio.com/DevDiv/_workitems/edit/714766.

src/VisualStudio/CSharp/Test/ProjectSystemShim/CPS/CSharpReferencesTests.cs

@@ -85,6 +85,29 @@ IEnumerable<PortableExecutableReference> GetProject3MetadataReferences()
             }
         }
 
+
+        [WpfFact]
+        [Trait(Traits.Feature, Traits.Features.ProjectSystemShims)]
+        public void RemoveProjectConvertsProjectReferencesBack()
+        {
+            using (var environment = new TestEnvironment())
+            {
+                var project1 = CreateCSharpCPSProject(environment, "project1", commandLineArguments: @"/out:c:\project1.dll");
+                var project2 = CreateCSharpCPSProject(environment, "project2");
+
+                // Add project reference as metadata reference: since this is known to be the output path of project1, the metadata reference is converted to a project reference
+                project2.AddMetadataReference(@"c:\project1.dll", new MetadataReferenceProperties());
+                Assert.Single(environment.Workspace.CurrentSolution.GetProject(project2.Id).AllProjectReferences);
+
+                // Remove project1. project2's reference should have been converted back
+                project1.Dispose();
+                Assert.Empty(environment.Workspace.CurrentSolution.GetProject(project2.Id).AllProjectReferences);
+                Assert.Single(environment.Workspace.CurrentSolution.GetProject(project2.Id).MetadataReferences);
+
+                project2.Dispose();
+            }
+        }
+
         [WpfFact]
         [Trait(Traits.Feature, Traits.Features.ProjectSystemShims)]
         public void AddRemoveAnalyzerReference_CPS()

src/VisualStudio/Core/Def/Implementation/ProjectSystem/VisualStudioWorkspaceImpl.cs

@@ -1288,18 +1288,29 @@ public void ApplyBatchChangeToProject(ProjectId projectId, Func<CodeAnalysis.Sol
             }
         }
 
-        private Dictionary<ProjectId, ProjectReferenceInformation> _projectReferenceInfo = new Dictionary<ProjectId, ProjectReferenceInformation>();
+        private Dictionary<ProjectId, ProjectReferenceInformation> _projectReferenceInfoMap = new Dictionary<ProjectId, ProjectReferenceInformation>();
 
         private ProjectReferenceInformation GetReferenceInfo_NoLock(ProjectId projectId)
         {
-            return _projectReferenceInfo.GetOrAdd(projectId, _ => new ProjectReferenceInformation());
+            return _projectReferenceInfoMap.GetOrAdd(projectId, _ => new ProjectReferenceInformation());
         }
 
         protected internal override void OnProjectRemoved(ProjectId projectId)
         {
             lock (_gate)
             {
-                _projectReferenceInfo.Remove(projectId);
+                if (_projectReferenceInfoMap.TryGetValue(projectId, out var projectReferenceInfo))
+                {
+                    // If we still had any output paths, we'll want to remove them to cause conversion back to metadata references.
+                    // The call below implicitly is modifying the collection we've fetched, so we'll make a copy.
+                    foreach (var outputPath in projectReferenceInfo.OutputPaths.ToList())
+                    {
+                        RemoveProjectOutputPath(projectId, outputPath);
+                    }
+
+                    _projectReferenceInfoMap.Remove(projectId);
+                }
+
                 _projectToGuidMap = _projectToGuidMap.Remove(projectId);
                 _projectToHierarchyMap = _projectToHierarchyMap.Remove(projectId);