Merge pull request #5507 from tmat/EagerMissingAssemblyResolution

Missing assembly resolution

src/Compilers/Core/CodeAnalysisTest/Collections/ArrayBuilderTests.cs

@@ -41,5 +41,29 @@ public void SelectDistinct1()
             builder = new ArrayBuilder<int> { 1, 2, 3, 2, 4, 5, 1 };
             AssertEx.Equal(new byte[] { 1, 2, 3, 4, 5 }, builder.SelectDistinct(n => (byte)n));
         }
+
+        [Fact]
+        public void AddRange()
+        {
+            var builder = new ArrayBuilder<int>();
+
+            builder.AddRange(new int[0], 0, 0);
+            AssertEx.Equal(new int[0], builder.ToArray());
+
+            builder.AddRange(new[] { 1, 2, 3 }, 0, 3);
+            AssertEx.Equal(new[] { 1, 2, 3 }, builder.ToArray());
+
+            builder.AddRange(new[] { 1, 2, 3 }, 2, 0);
+            AssertEx.Equal(new[] { 1, 2, 3 }, builder.ToArray());
+
+            builder.AddRange(new[] { 1, 2, 3 }, 1, 1);
+            AssertEx.Equal(new[] { 1, 2, 3, 2 }, builder.ToArray());
+
+            builder.AddRange(new[] { 1, 2, 3 }, 1, 2);
+            AssertEx.Equal(new[] { 1, 2, 3, 2, 2, 3 }, builder.ToArray());
+
+            builder.AddRange(new[] { 1, 2, 3 }, 2, 1);
+            AssertEx.Equal(new[] { 1, 2, 3, 2, 2, 3, 3 }, builder.ToArray());
+        }
     }
 }

src/Compilers/Core/Portable/Collections/ArrayBuilderExtensions.cs

@@ -139,5 +139,10 @@ public static T Peek<T>(this ArrayBuilder<T> builder)
         {
             return builder[builder.Count - 1];
         }
+
+        public static ImmutableArray<T> ToImmutableOrEmptyAndFree<T>(this ArrayBuilder<T> builderOpt)
+        {
+            return builderOpt?.ToImmutableAndFree() ?? ImmutableArray<T>.Empty;
+        }
     }
 }

src/Compilers/Core/Portable/MetadataReference/MetadataReferenceResolver.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
+using System;
 using System.Collections.Immutable;
 
 namespace Microsoft.CodeAnalysis
@@ -12,5 +13,18 @@ public abstract class MetadataReferenceResolver
         public abstract override bool Equals(object other);
         public abstract override int GetHashCode();
         public abstract ImmutableArray<PortableExecutableReference> ResolveReference(string reference, string baseFilePath, MetadataReferenceProperties properties);
+
+        /// <summary>
+        /// True to instruct the compiler to invoke <see cref="ResolveMissingAssembly(AssemblyIdentity)"/> for each assembly reference that
+        /// doesn't match any of the assemblies explicitly referenced by the <see cref="Compilation"/> (via <see cref="Compilation.ExternalReferences"/>, or #r directives.
+        /// </summary>
+        public virtual bool ResolveMissingAssemblies => false;
+
+        /// <summary>
+        /// Resolves a missing assembly reference.
+        /// </summary>
+        /// <param name="identity">Identity of the assembly reference.</param>
+        /// <returns>Resolved reference or null if the identity can't be resolved.</returns>
+        public virtual PortableExecutableReference ResolveMissingAssembly(AssemblyIdentity identity) => null;
     }
 }

src/Compilers/Core/Portable/PublicAPI.Unshipped.txt

@@ -60,4 +60,6 @@ static Microsoft.CodeAnalysis.Diagnostics.Telemetry.AnalyzerTelemetry.GetAnalyze
 static Microsoft.CodeAnalysis.Diagnostics.Telemetry.AnalyzerTelemetry.GetAnalyzerExecutionTime(this Microsoft.CodeAnalysis.Diagnostics.CompilationWithAnalyzers compilationWithAnalyzers, Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer analyzer) -> System.TimeSpan
 static Microsoft.CodeAnalysis.SyntaxNodeExtensions.NormalizeWhitespace<TNode>(this TNode node, string indentation = "    ", string eol = "\r\n", bool elasticTrivia = false) -> TNode
 static Microsoft.CodeAnalysis.SyntaxNodeExtensions.NormalizeWhitespace<TNode>(this TNode node, string indentation, bool elasticTrivia) -> TNode
-virtual Microsoft.CodeAnalysis.SourceReferenceResolver.ReadText(string resolvedPath) -> Microsoft.CodeAnalysis.Text.SourceText
+virtual Microsoft.CodeAnalysis.MetadataReferenceResolver.ResolveMissingAssemblies.get -> bool
+virtual Microsoft.CodeAnalysis.MetadataReferenceResolver.ResolveMissingAssembly(Microsoft.CodeAnalysis.AssemblyIdentity identity) -> Microsoft.CodeAnalysis.PortableExecutableReference
+virtual Microsoft.CodeAnalysis.SourceReferenceResolver.ReadText(string resolvedPath) -> Microsoft.CodeAnalysis.Text.SourceText
\ No newline at end of file

src/Compilers/Core/Portable/ReferenceManager/AssemblyData.cs

@@ -2,6 +2,7 @@
 
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Diagnostics;
 
 namespace Microsoft.CodeAnalysis
 {
@@ -10,6 +11,7 @@ internal partial class CommonReferenceManager<TCompilation, TAssemblySymbol>
         /// <summary>
         /// Information about an assembly, used as an input for the Binder class.
         /// </summary>
+        [DebuggerDisplay("{GetDebuggerDisplay(), nq}")]
         internal abstract class AssemblyData
         {
             /// <summary>
@@ -63,6 +65,8 @@ internal abstract class AssemblyData
             /// Returns null otherwise.
             /// </summary>
             public abstract Compilation SourceCompilation { get; }
+
+            private string GetDebuggerDisplay() => $"{GetType().Name}: [{Identity.GetDisplayName()}]";
         }
     }
 }

src/Compilers/Core/Portable/ReferenceManager/AssemblyDataForAssemblyBeingBuilt.cs

@@ -20,10 +20,6 @@ protected sealed class AssemblyDataForAssemblyBeingBuilt : AssemblyData
             // all referenced assembly names including assemblies referenced by modules:
             private readonly ImmutableArray<AssemblyIdentity> _referencedAssemblies;
 
-            // [0] is the number of assembly names in referencedAssemblies that are direct references specified in referencedAssemblyData.
-            // [i] is the number of references coming from module moduleInfo[i-1]. These names are also added to referencedAssemblies.
-            private readonly int[] _moduleReferenceCounts;
-
             public AssemblyDataForAssemblyBeingBuilt(
                 AssemblyIdentity identity,
                 ImmutableArray<AssemblyData> referencedAssemblyData,
@@ -36,37 +32,21 @@ protected sealed class AssemblyDataForAssemblyBeingBuilt : AssemblyData
 
                 _referencedAssemblyData = referencedAssemblyData;
 
-                int count = modules.Length;
-
-                var refs = ArrayBuilder<AssemblyIdentity>.GetInstance(referencedAssemblyData.Length + count); //approximate size
+                var refs = ArrayBuilder<AssemblyIdentity>.GetInstance(referencedAssemblyData.Length + modules.Length); //approximate size
                 foreach (AssemblyData data in referencedAssemblyData)
                 {
                     refs.Add(data.Identity);
                 }
 
-                _moduleReferenceCounts = new int[count + 1]; // Plus one for the source module.
-                _moduleReferenceCounts[0] = refs.Count;
-
                 // add assembly names from modules:
-                for (int i = 1; i <= count; i++)
+                for (int i = 1; i <= modules.Length; i++)
                 {
-                    var module = modules[i - 1];
-
-                    _moduleReferenceCounts[i] = module.ReferencedAssemblies.Length;
-                    refs.AddRange(module.ReferencedAssemblies);
+                    refs.AddRange(modules[i - 1].ReferencedAssemblies);
                 }
 
                 _referencedAssemblies = refs.ToImmutableAndFree();
             }
 
-            public int[] ReferencesCountForModule
-            {
-                get
-                {
-                    return _moduleReferenceCounts;
-                }
-            }
-
             public override AssemblyIdentity Identity
             {
                 get
@@ -103,14 +83,17 @@ public override IEnumerable<TAssemblySymbol> AvailableSymbols
                     boundReferences[i] = new AssemblyReferenceBinding(assemblies[i + 1].Identity, i + 1);
                 }
 
+                // references from added modules shouldn't resolve against the assembly being built (definition #0)
+                const int definitionStartIndex = 1;
+
                 // resolve references coming from linked modules:
                 for (int i = _referencedAssemblyData.Length; i < _referencedAssemblies.Length; i++)
                 {
                     boundReferences[i] = ResolveReferencedAssembly(
                         _referencedAssemblies[i],
                         assemblies,
-                        assemblyIdentityComparer,
-                        okToResolveAgainstCompilationBeingCreated: false); // references from added modules shouldn't resolve against the assembly we are building.
+                        definitionStartIndex,
+                        assemblyIdentityComparer); 
                 }
 
                 return boundReferences;

src/Compilers/Core/Portable/ReferenceManager/AssemblyReferenceBinding.cs

@@ -95,7 +95,6 @@ internal AssemblyIdentity ReferenceIdentity
             {
                 get
                 {
-                    Debug.Assert(IsBound);
                     return _referenceIdentity;
                 }
             }

src/Compilers/Core/Portable/ReferenceManager/CommonReferenceManager.Resolution.cs

@@ -106,7 +106,7 @@ public int Index
 
             private string GetDebuggerDisplay()
             {
-                return IsSkipped ? "<skipped>" : (_kind == MetadataImageKind.Assembly ? "A[" : "M[") + Index + "]: aliases=" + _aliases.ToString();
+                return IsSkipped ? "<skipped>" : $"{(_kind == MetadataImageKind.Assembly ? "A" : "M")}[{Index}]: aliases='{string.Join("','", _aliases)}'";
             }
         }
 
@@ -344,7 +344,7 @@ public ReferencedAssemblyIdentity(AssemblyIdentity identity, MetadataReference m
                         : ((object)modulesBuilder == null ? 0 : modulesBuilder.Count);
 
                     int reversedIndex = count - 1 - referenceMap[i].Index;
-                    referenceMap[i] = new ResolvedReference(reversedIndex, referenceMap[i].Kind, GetAliases(references[i], aliasMap));
+                    referenceMap[i] = new ResolvedReference(reversedIndex, referenceMap[i].Kind, GetAndFreeAliases(references[i], aliasMap));
                 }
             }
 
@@ -371,10 +371,10 @@ public ReferencedAssemblyIdentity(AssemblyIdentity identity, MetadataReference m
             return ImmutableArray.CreateRange(referenceMap);
         }
 
-        private static ImmutableArray<string> GetAliases(MetadataReference reference, Dictionary<MetadataReference, ArrayBuilder<string>> aliasMap)
+        private static ImmutableArray<string> GetAndFreeAliases(MetadataReference reference, Dictionary<MetadataReference, ArrayBuilder<string>> aliasMapOpt)
         {
             ArrayBuilder<string> aliases;
-            if (aliasMap != null && aliasMap.TryGetValue(reference, out aliases))
+            if (aliasMapOpt != null && aliasMapOpt.TryGetValue(reference, out aliases))
             {
                 return aliases.ToImmutableAndFree();
             }
@@ -573,10 +573,17 @@ private static void AddModule(PEModule module, int referenceIndex, ResolvedRefer
             modules.Add(module);
         }
 
-        // Returns null if an assembly of an equivalent identity has not been added previously, otherwise returns the reference that added it.
-        // - Both assembly names are strong (have keys) and are either equal or FX unified 
-        // - Both assembly names are weak (no keys) and have the same simple name.
-        private MetadataReference TryAddAssembly(AssemblyIdentity identity, MetadataReference boundReference, DiagnosticBag diagnostics, Location location,
+        /// <summary>
+        /// Returns null if an assembly of an equivalent identity has not been added previously, otherwise returns the reference that added it.
+        /// Two identities are considered equivalent if
+        /// - both assembly names are strong (have keys) and are either equal or FX unified 
+        /// - both assembly names are weak (no keys) and have the same simple name.
+        /// </summary>
+        private MetadataReference TryAddAssembly(
+            AssemblyIdentity identity,
+            MetadataReference boundReference,
+            DiagnosticBag diagnostics, 
+            Location location,
             ref Dictionary<string, List<ReferencedAssemblyIdentity>> referencesBySimpleName)
         {
             if (referencesBySimpleName == null)
@@ -767,13 +774,13 @@ private static PortableExecutableReference ResolveReferenceDirective(string refe
         internal static AssemblyReferenceBinding[] ResolveReferencedAssemblies(
             ImmutableArray<AssemblyIdentity> references,
             ImmutableArray<AssemblyData> definitions,
-            AssemblyIdentityComparer assemblyIdentityComparer,
-            bool okToResolveAgainstCompilationBeingCreated)
+            int definitionStartIndex,
+            AssemblyIdentityComparer assemblyIdentityComparer)
         {
             var boundReferences = new AssemblyReferenceBinding[references.Length];
             for (int j = 0; j < references.Length; j++)
             {
-                boundReferences[j] = ResolveReferencedAssembly(references[j], definitions, assemblyIdentityComparer, okToResolveAgainstCompilationBeingCreated);
+                boundReferences[j] = ResolveReferencedAssembly(references[j], definitions, definitionStartIndex, assemblyIdentityComparer);
             }
 
             return boundReferences;
@@ -783,17 +790,17 @@ private static PortableExecutableReference ResolveReferenceDirective(string refe
         /// Used to match AssemblyRef with AssemblyDef.
         /// </summary>
         /// <param name="definitions">Array of definition identities to match against.</param>
+        /// <param name="definitionStartIndex">An index of the first definition to consider, <paramref name="definitions"/> preceding this index are ignored.</param>
         /// <param name="reference">Reference identity to resolve.</param>
         /// <param name="assemblyIdentityComparer">Assembly identity comparer.</param>
-        /// <param name="okToResolveAgainstCompilationBeingCreated"> Is it Ok to resolve reference against the compilation we are creating?</param>
         /// <returns>
         /// Returns an index the reference is bound.
         /// </returns>
         internal static AssemblyReferenceBinding ResolveReferencedAssembly(
             AssemblyIdentity reference,
             ImmutableArray<AssemblyData> definitions,
-            AssemblyIdentityComparer assemblyIdentityComparer,
-            bool okToResolveAgainstCompilationBeingCreated)
+            int definitionStartIndex,
+            AssemblyIdentityComparer assemblyIdentityComparer)
         {
             // Dev11 C# compiler allows the versions to not match exactly, assuming that a newer library may be used instead of an older version.
             // For a given reference it finds a definition with the lowest version that is higher then or equal to the reference version.
@@ -803,9 +810,11 @@ private static PortableExecutableReference ResolveReferenceDirective(string refe
             int minHigherVersionDefinition = -1;
             int maxLowerVersionDefinition = -1;
 
-            // NB: Start at 1, since we checked 0 above.
-            const int definitionOffset = 1;
-            for (int i = definitionOffset; i < definitions.Length; i++)
+            // Skip assembly being built for now; it will be considered at the very end:
+            bool resolveAgainstAssemblyBeingBuilt = definitionStartIndex == 0;
+            definitionStartIndex = Math.Max(definitionStartIndex, 1);
+
+            for (int i = definitionStartIndex; i < definitions.Length; i++)
             {
                 AssemblyIdentity definition = definitions[i].Identity;
 
@@ -838,6 +847,9 @@ private static PortableExecutableReference ResolveReferenceDirective(string refe
                         }
 
                         continue;
+
+                    default:
+                        throw ExceptionUtilities.Unreachable;
                 }
             }
 
@@ -860,7 +872,7 @@ private static PortableExecutableReference ResolveReferenceDirective(string refe
             // substitute for a collection of Windows.*.winmd compile-time references.
             if (reference.IsWindowsComponent())
             {
-                for (int i = definitionOffset; i < definitions.Length; i++)
+                for (int i = definitionStartIndex; i < definitions.Length; i++)
                 {
                     if (definitions[i].Identity.IsWindowsRuntime())
                     {
@@ -878,7 +890,7 @@ private static PortableExecutableReference ResolveReferenceDirective(string refe
             // allow the compilation to match the reference.
             if (reference.ContentType == AssemblyContentType.WindowsRuntime)
             {
-                for (int i = definitionOffset; i < definitions.Length; i++)
+                for (int i = definitionStartIndex; i < definitions.Length; i++)
                 {
                     var definition = definitions[i].Identity;
                     var sourceCompilation = definitions[i].SourceCompilation;
@@ -898,7 +910,7 @@ private static PortableExecutableReference ResolveReferenceDirective(string refe
             // As in the native compiler (see IMPORTER::MapAssemblyRefToAid), we compare against the
             // compilation (i.e. source) assembly as a last resort.  We follow the native approach of
             // skipping the public key comparison since we have yet to compute it.
-            if (okToResolveAgainstCompilationBeingCreated &&
+            if (resolveAgainstAssemblyBeingBuilt &&
                 AssemblyIdentityComparer.SimpleNameComparer.Equals(reference.Name, definitions[0].Identity.Name))
             {
                 Debug.Assert(definitions[0].Identity.PublicKeyToken.IsEmpty);

src/Compilers/Core/Portable/ReferenceManager/CommonReferenceManager.State.cs

@@ -323,8 +323,6 @@ internal void AssertBound()
 
             // lazyCorLibrary is null if the compilation is corlib
             Debug.Assert(_lazyReferencedAssemblies.Length == 0 || _lazyCorLibraryOpt != null);
-
-            Debug.Assert(_lazyReferencedAssemblies.Length == _lazyAliasesOfReferencedAssemblies.Length);
         }
 
         [Conditional("DEBUG")]
@@ -362,6 +360,7 @@ internal bool IsBound
 
             Debug.Assert(referencedModules.Length == referencedModulesReferences.Length);
             Debug.Assert(referencedModules.Length == referencedModulesMap.Count);
+            Debug.Assert(referencedAssemblies.Length == aliasesOfReferencedAssemblies.Length);
 
             _lazyReferencedAssembliesMap = referencedAssembliesMap;
             _lazyReferencedModuleIndexMap = referencedModulesMap;
@@ -381,6 +380,45 @@ internal bool IsBound
             Interlocked.Exchange(ref _isBound, 1);
         }
 
+        protected static void BuildReferencedAssembliesAndModulesMaps(
+            ImmutableArray<MetadataReference> references,
+            ImmutableArray<ResolvedReference> referenceMap,
+            int referencedModuleCount,
+            out Dictionary<MetadataReference, int> referencedAssembliesMap,
+            out Dictionary<MetadataReference, int> referencedModulesMap,
+            out ImmutableArray<ImmutableArray<string>> aliasesOfReferencedAssemblies)
+        {
+            referencedAssembliesMap = new Dictionary<MetadataReference, int>(referenceMap.Length);
+            referencedModulesMap = new Dictionary<MetadataReference, int>(referencedModuleCount);
+            var aliasesOfReferencedAssembliesBuilder = ArrayBuilder<ImmutableArray<string>>.GetInstance(referenceMap.Length - referencedModuleCount);
+
+            for (int i = 0; i < referenceMap.Length; i++)
+            {
+                if (referenceMap[i].IsSkipped)
+                {
+                    continue;
+                }
+
+                if (referenceMap[i].Kind == MetadataImageKind.Module)
+                {
+                    // add 1 for the manifest module:
+                    int moduleIndex = 1 + referenceMap[i].Index;
+                    referencedModulesMap.Add(references[i], moduleIndex);
+                }
+                else
+                {
+                    // index into assembly data array
+                    int assemblyIndex = referenceMap[i].Index;
+                    Debug.Assert(aliasesOfReferencedAssembliesBuilder.Count == assemblyIndex);
+
+                    referencedAssembliesMap.Add(references[i], assemblyIndex);
+                    aliasesOfReferencedAssembliesBuilder.Add(referenceMap[i].Aliases);
+                }
+            }
+
+            aliasesOfReferencedAssemblies = aliasesOfReferencedAssembliesBuilder.ToImmutableAndFree();
+        }
+
         #region Compilation APIs Implementation
 
         // for testing purposes

src/Compilers/Core/SharedCollections/ArrayBuilder.cs

@@ -383,6 +383,14 @@ public void AddRange(ImmutableArray<T> items, int length)
             _builder.AddRange(items, length);
         }
 
+        public void AddRange(T[] items, int start, int length)
+        {
+            for (int i = start, end = start + length; i < end; i++)
+            {
+                Add(items[i]);
+            }
+        }
+
         public void AddRange(IEnumerable<T> items)
         {
             _builder.AddRange(items);

src/Test/Utilities/Shared/Mocks/TestMissingMetadataReferenceResolver.cs

+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using Microsoft.CodeAnalysis;
+
+namespace Roslyn.Test.Utilities
+{
+    internal class TestMissingMetadataReferenceResolver : MetadataReferenceResolver
+    {
+        private readonly Dictionary<string, MetadataReference> _map;
+        public readonly List<AssemblyIdentity> ResolutionAttempts = new List<AssemblyIdentity>();
+
+        public TestMissingMetadataReferenceResolver(Dictionary<string, MetadataReference> map)
+        {
+            _map = map;
+        }
+
+        public override PortableExecutableReference ResolveMissingAssembly(AssemblyIdentity identity)
+        {
+            ResolutionAttempts.Add(identity);
+
+            MetadataReference reference;
+            string nameAndVersion = identity.Name + (identity.Version != AssemblyIdentity.NullVersion ? $", {identity.Version}" : "");
+            return _map.TryGetValue(nameAndVersion, out reference) ? (PortableExecutableReference)reference : null;
+        }
+
+        public override bool ResolveMissingAssemblies => true;
+        public override bool Equals(object other) => true;
+        public override int GetHashCode() => 1;
+        public override ImmutableArray<PortableExecutableReference> ResolveReference(string reference, string baseFilePath, MetadataReferenceProperties properties) => default(ImmutableArray<PortableExecutableReference>);
+    }
+}

src/Test/Utilities/Shared/TestUtilities.projitems

@@ -52,6 +52,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)Metadata\TypeAttributesMissing.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Metadata\PEModuleTestHelpers.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Mocks\TestDocumentationCommentProvider.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Mocks\TestMissingMetadataReferenceResolver.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Mocks\VirtualizedRelativePathResolver.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Pdb\MockSymWriter.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Pdb\PdbTestUtilities.cs" />