Verification for VS.ExternalAPIs.Roslyn package

A recent refactoring caused a number of insertion failures as we weren't
properly updating the contents of VS.ExternalAPIs.Roslyn.nupkg. This
adds basic verification that the contents are correct based on our build
output.

src/Tools/BuildBoss/BuildBoss.csproj

@@ -16,6 +16,7 @@
     <Reference Include="WindowsBase" />
     <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
     <PackageReference Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableVersion)" />
+    <PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
     <PackageReference Include="MSBuild.StructuredLogger" Version="$(MSBuildStructuredLoggerVersion)" />
     <PackageReference Include="Mono.Options" Version="$(MonoOptionsVersion)" />
   </ItemGroup>

src/Tools/BuildBoss/CompilerNuGetCheckerUtil.cs

@@ -4,6 +4,8 @@
 using System.IO;
 using System.IO.Packaging;
 using System.Linq;
+using System.Reflection.Metadata;
+using System.Reflection.PortableExecutable;
 using System.Security.Cryptography;
 using System.Text.RegularExpressions;
 
@@ -74,6 +76,7 @@ public bool Check(TextWriter textWriter)
                 allGood &= CheckDesktop(textWriter, filter(isDesktop: true));
                 allGood &= CheckCoreClr(textWriter, filter(isDesktop: false));
                 allGood &= CheckCombined(textWriter, packageAssets);
+                allGood &= CheckExternalApis(textWriter);
                 return allGood;
 
                 IEnumerable<string> filter(bool isDesktop) => packageAssets.Where(x => x.IsDesktop == isDesktop).Select(x => x.FileRelativeName);
@@ -147,6 +150,74 @@ private bool CheckCombined(TextWriter textWriter, IEnumerable<PackageAsset> pack
                     list);
         }
 
+        /// <summary>
+        /// Verifies the VS.ExternalAPIs.Roslyn package is self consistent. Need to ensure that we insert all of the project dependencies
+        /// that we build into the package. If we miss a dependency then the VS insertion will fail. Big refactorings can often forget to
+        /// properly update this package.
+        /// </summary>
+        /// <param name="textWriter"></param>
+        /// <returns></returns>
+        private bool CheckExternalApis(TextWriter textWriter)
+        {
+            var packageFilePath = FindNuGetPackage(Path.Combine(ArtifactsDirectory, "VSSetup", Configuration, "DevDivPackages"), "VS.ExternalAPIs.Roslyn");
+            var allGood = true;
+
+            textWriter.WriteLine("Verifying contents of VS.ExternalAPIs.Roslyn");
+            textWriter.WriteLine("\tRoot Folder");
+            verifyFolder("");
+            textWriter.WriteLine("\tRemote Debugger net20");
+            verifyFolder(@"RemoteDebugger\net20");
+            textWriter.WriteLine("\tRemote Debugger net50");
+            verifyFolder(@"RemoteDebugger\net50");
+            return allGood;
+
+            void verifyFolder(string folderRelativeName)
+            {
+                var foundDllNameSet = new HashSet<string>(PathComparer);
+                var neededDllNameSet = new HashSet<string>(PathComparer);
+                foreach (var part in GetPartsInFolder(packageFilePath, ""))
+                {
+                    var name = part.GetName();
+                    if (Path.GetExtension(name) != ".dll")
+                    {
+                        continue;
+                    }
+
+                    foundDllNameSet.Add(Path.GetFileNameWithoutExtension(name));
+                    using var peReader = new PEReader(part.GetStream(FileMode.Open, FileAccess.Read));
+                    var metadataReader = peReader.GetMetadataReader();
+                    foreach (var handle in metadataReader.AssemblyReferences)
+                    {
+                        var assemblyReference = metadataReader.GetAssemblyReference(handle);
+                        var assemblyName = metadataReader.GetString(assemblyReference.Name);
+                        neededDllNameSet.Add(assemblyName);
+                    }
+                }
+
+                if (foundDllNameSet.Count == 0)
+                {
+                    allGood = false;
+                    textWriter.WriteLine($"\t\tFound zero DLLs in {folderRelativeName}");
+                    return;
+                }
+
+                // As a simplification we only validate the assembly names that begin with Microsoft.CodeAnalysis. This is a good 
+                // hueristic for finding assemblies that we build. Can be expanded in the future if we find more assemblies that
+                // are worth validating here.
+                var neededDllNames = neededDllNameSet
+                    .Where(x => x.StartsWith("Microsoft.CodeAnalysis"))
+                    .OrderBy(x => x, PathComparer);
+                foreach (var name in neededDllNames)
+                {
+                    if (!foundDllNameSet.Contains(name))
+                    {
+                        textWriter.WriteLine($"\t\tMissing dependency {name}");
+                        allGood = false;
+                    }
+                }
+            }
+        }
+
         private bool GetPackageAssets(TextWriter textWriter, List<PackageAsset> packageAssets)
         {
             var allGood = true;
@@ -300,40 +371,6 @@ IEnumerable<string> enumerateFiles()
             string folderRelativePath,
             IEnumerable<string> dllFileNames)
         {
-            Debug.Assert(string.IsNullOrEmpty(folderRelativePath) || folderRelativePath[0] != '\\');
-
-            // Get all of the assets parts that are in the specified folder. Will exclude items that
-            // are in any child folder
-            IEnumerable<string> getPartsInFolder()
-            {
-                using (var package = Package.Open(packageFilePath, FileMode.Open, FileAccess.Read))
-                {
-                    foreach (var part in package.GetParts())
-                    {
-                        var relativeName = part.Uri.ToString().Replace('/', '\\');
-                        if (string.IsNullOrEmpty(relativeName))
-                        {
-                            continue;
-                        }
-
-                        if (relativeName[0] == '\\')
-                        {
-                            relativeName = relativeName.Substring(1);
-                        }
-
-                        if (!relativeName.StartsWith(folderRelativePath, PathComparison))
-                        {
-                            continue;
-                        }
-
-                        if (IsTrackedAsset(relativeName))
-                        {
-                            yield return relativeName;
-                        }
-                    }
-                }
-            }
-
             var map = dllFileNames
                 .ToDictionary(
                     keySelector: x => Path.Combine(folderRelativePath, x),
@@ -343,8 +380,14 @@ IEnumerable<string> getPartsInFolder()
             var packageFileName = Path.GetFileName(packageFilePath);
 
             textWriter.WriteLine($"Verifying {packageFileName}");
-            foreach (var relativeName in getPartsInFolder())
+            foreach (var part in GetPartsInFolder(packageFilePath, folderRelativePath))
             {
+                var relativeName = part.GetRelativeName();
+                if (!IsTrackedAsset(relativeName))
+                {
+                    continue;
+                }
+
                 var name = Path.GetFileName(relativeName);
                 if (map.TryGetValue(relativeName, out var isFound))
                 {
@@ -374,6 +417,33 @@ IEnumerable<string> getPartsInFolder()
             return allGood;
         }
 
+        /// <summary>
+        /// Get all of the parts in the specified folder. Will exclude all items in child folders.
+        /// </summary>
+        private static IEnumerable<PackagePart> GetPartsInFolder(string packageFilePath, string folderRelativePath)
+        {
+            Debug.Assert(string.IsNullOrEmpty(folderRelativePath) || folderRelativePath[0] != '\\');
+
+            using (var package = Package.Open(packageFilePath, FileMode.Open, FileAccess.Read))
+            {
+                foreach (var part in package.GetParts())
+                {
+                    var relativeName = part.GetRelativeName();
+                    if (string.IsNullOrEmpty(relativeName))
+                    {
+                        continue;
+                    }
+
+                    if (!relativeName.StartsWith(folderRelativePath, PathComparison))
+                    {
+                        continue;
+                    }
+
+                    yield return part;
+                }
+            }
+        }
+
         private string FindNuGetPackage(string directory, string partialName)
         {
             var regex = $@"{partialName}.\d.*\.nupkg";

src/Tools/BuildBoss/Extensions.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Packaging;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BuildBoss
+{
+    internal static class Extensions
+    {
+        internal static string GetRelativeName(this PackagePart part)
+        {
+            var relativeName = part.Uri.ToString().Replace('/', '\\');
+            if (!string.IsNullOrEmpty(relativeName) && relativeName[0] == '\\')
+            {
+                relativeName = relativeName.Substring(1);
+            }
+
+            return relativeName;
+        }
+
+        internal static string GetName(this PackagePart part) => Path.GetFileName(GetRelativeName(part));
+    }
+}

src/Tools/BuildBoss/Program.cs

@@ -96,9 +96,9 @@ private static bool Go(string repositoryDirectory, string configuration, List<st
 
             var artifactsDirectory = Path.Combine(repositoryDirectory, "artifacts");
 
-            allGood &= ProcessStructuredLog(artifactsDirectory, configuration);
             allGood &= ProcessTargets(repositoryDirectory);
             allGood &= ProcessPackages(repositoryDirectory, artifactsDirectory, configuration);
+            allGood &= ProcessStructuredLog(artifactsDirectory, configuration);
 
             if (!allGood)
             {