AnalyzerAssemblyLoader optimization to search the assemblies loaded by identity cache (#13176)

Adrian pointed out that AnalyzerAssemblyLoader.LoadFromPathUnchecked was missing cache hits on _loadedAssembliesByIdentity for same assemblies (analyzer dependencies) loaded from different paths. This was causing large number of redundant ModLoads, which should now be fixed.
Fixes VSO [249557](https://devdiv.visualstudio.com/DevDiv/_workitems?id=249557&fullScreen=false&_a=edit)

src/Compilers/Core/Portable/DiagnosticAnalyzer/AnalyzerAssemblyLoader.cs

@@ -15,6 +15,7 @@ internal abstract class AnalyzerAssemblyLoader : IAnalyzerAssemblyLoader
 
         // lock _guard to read/write
         private readonly Dictionary<string, Assembly> _loadedAssembliesByPath = new Dictionary<string, Assembly>();
+        private readonly Dictionary<string, AssemblyIdentity> _loadedAssemblyIdentitiesByPath = new Dictionary<string, AssemblyIdentity>();
         private readonly Dictionary<AssemblyIdentity, Assembly> _loadedAssembliesByIdentity = new Dictionary<AssemblyIdentity, Assembly>();
 
         // maps file name to a full path (lock _guard to read/write):
@@ -52,28 +53,50 @@ public Assembly LoadFromPath(string fullPath)
         #endregion
 
         private Assembly LoadFromPathUnchecked(string fullPath)
+        {
+            return LoadFromPathUncheckedCore(fullPath);
+        }
+
+        private Assembly LoadFromPathUncheckedCore(string fullPath, AssemblyIdentity identity = null)
         {
             Debug.Assert(PathUtilities.IsAbsolute(fullPath));
 
+            // Check if we have already loaded an assembly with the same identity or from the given path.
+            Assembly loadedAssembly = null;
             lock (_guard)
             {
                 Assembly existingAssembly;
                 if (_loadedAssembliesByPath.TryGetValue(fullPath, out existingAssembly))
                 {
-                    return existingAssembly;
+                    loadedAssembly = existingAssembly;
                 }
+                else
+                {
+                    identity = identity ?? GetOrAddAssemblyIdentity(fullPath);
+                    if (identity != null && _loadedAssembliesByIdentity.TryGetValue(identity, out existingAssembly))
+                    {
+                        loadedAssembly = existingAssembly;
+                    }
+                }
+            }
+
+            // Otherwise, load the assembly.
+            if (loadedAssembly == null)
+            {
+                loadedAssembly = LoadFromPathImpl(fullPath);
             }
 
-            Assembly assembly = LoadFromPathImpl(fullPath);
-            return AddToCache(assembly, fullPath);
+            // Add the loaded assembly to both path and identity cache.
+            return AddToCache(loadedAssembly, fullPath, identity);
         }
 
-        private Assembly AddToCache(Assembly assembly, string fullPath)
+        private Assembly AddToCache(Assembly assembly, string fullPath, AssemblyIdentity identity)
         {
             Debug.Assert(PathUtilities.IsAbsolute(fullPath));
             Debug.Assert(assembly != null);
 
-            var identity = AssemblyIdentity.FromAssemblyDefinition(assembly);
+            identity = AddToCache(fullPath, identity ?? AssemblyIdentity.FromAssemblyDefinition(assembly));
+            Debug.Assert(identity != null);
 
             lock (_guard)
             {
@@ -82,10 +105,12 @@ private Assembly AddToCache(Assembly assembly, string fullPath)
                 Assembly existingAssembly;
                 if (_loadedAssembliesByIdentity.TryGetValue(identity, out existingAssembly))
                 {
-                    return existingAssembly;
+                    assembly = existingAssembly;
+                }
+                else
+                {
+                    _loadedAssembliesByIdentity.Add(identity, assembly);
                 }
-
-                _loadedAssembliesByIdentity.Add(identity, assembly);
 
                 // An assembly file might be replaced by another file with a different identity.
                 // Last one wins.
@@ -95,6 +120,41 @@ private Assembly AddToCache(Assembly assembly, string fullPath)
             }
         }
 
+        private AssemblyIdentity GetOrAddAssemblyIdentity(string fullPath)
+        {
+            Debug.Assert(PathUtilities.IsAbsolute(fullPath));
+
+            lock (_guard)
+            {
+                AssemblyIdentity existingIdentity;
+                if (_loadedAssemblyIdentitiesByPath.TryGetValue(fullPath, out existingIdentity))
+                {
+                    return existingIdentity;
+                }
+            }
+
+            var identity = AssemblyIdentityUtils.TryGetAssemblyIdentity(fullPath);
+            return AddToCache(fullPath, identity);
+        }
+
+        private AssemblyIdentity AddToCache(string fullPath, AssemblyIdentity identity)
+        {
+            lock (_guard)
+            {
+                AssemblyIdentity existingIdentity;
+                if (_loadedAssemblyIdentitiesByPath.TryGetValue(fullPath, out existingIdentity) && existingIdentity != null)
+                {
+                    identity = existingIdentity;
+                }
+                else
+                {
+                    _loadedAssemblyIdentitiesByPath[fullPath] = identity;
+                }
+            }
+
+            return identity;
+        }
+
         public Assembly Load(string displayName)
         {
             AssemblyIdentity requestedIdentity;
@@ -129,11 +189,11 @@ public Assembly Load(string displayName)
             // Load the one that matches the requested identity (if any).
             foreach (var candidatePath in candidatePaths)
             {
-                var candidateIdentity = AssemblyIdentityUtils.TryGetAssemblyIdentity(candidatePath);
+                var candidateIdentity = GetOrAddAssemblyIdentity(candidatePath);
 
                 if (requestedIdentity.Equals(candidateIdentity))
                 {
-                    return LoadFromPathUnchecked(candidatePath);
+                    return LoadFromPathUncheckedCore(candidatePath, candidateIdentity);
                 }
             }