Avoid leaking documents if an exception is thrown in AddDocument

Fixes #22723

src/VisualStudio/Core/Def/Implementation/ProjectSystem/AbstractProject.cs

@@ -1027,7 +1027,13 @@ internal void AddDocument(IVisualStudioHostDocument document, bool isCurrentCont
             {
                 lock (_gate)
                 {
-                    _documents.Add(document.Id, document);
+                    // This condition ensures that if we throw an exception for either Add operation, the document will
+                    // not be added to either collection.
+                    if (!_documentMonikers.ContainsKey(document.Key.Moniker))
+                    {
+                        _documents.Add(document.Id, document);
+                    }
+
                     _documentMonikers.Add(document.Key.Moniker, document);
                 }