SignFast: verify is all checksum based now

src/Tools/SignRoslyn/BinaryName.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace SignRoslyn
+{
+    internal struct BinaryName : IEquatable<BinaryName>
+    {
+        internal string Name { get; }
+        internal string FullPath { get; }
+        internal string RelativePath { get; }
+
+        internal bool IsAssembly => PathUtil.IsAssembly(Name);
+        internal bool IsVsix => PathUtil.IsVsix(Name);
+
+        internal BinaryName(string rootBinaryPath, string relativePath)
+        {
+            Name = Path.GetFileName(relativePath);
+            FullPath = Path.Combine(rootBinaryPath, relativePath);
+            RelativePath = relativePath;
+        }
+
+        public static bool operator ==(BinaryName left, BinaryName right) => left.FullPath == right.FullPath;
+        public static bool operator !=(BinaryName left, BinaryName right) => !(left == right);
+        public bool Equals(BinaryName other) => this == other;
+        public override int GetHashCode() => FullPath.GetHashCode();
+        public override string ToString() => RelativePath;
+        public override bool Equals(object obj) => obj is BinaryName && Equals((BinaryName)obj);
+    }
+}

src/Tools/SignRoslyn/ContentUtil.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace SignRoslyn
+{
+    internal sealed class ContentUtil
+    {
+        private readonly Dictionary<string, string> _filePathCache = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        private readonly MD5 _md5 = MD5.Create();
+
+        internal string GetChecksum(Stream stream)
+        {
+            var hash = _md5.ComputeHash(stream);
+            return HashBytesToString(hash);
+        }
+
+        internal string GetChecksum(string filePath)
+        {
+            string checksum;
+            if (!_filePathCache.TryGetValue(filePath, out checksum))
+            {
+                using (var stream = File.OpenRead(filePath))
+                {
+                    checksum = GetChecksum(stream);
+                }
+                _filePathCache[filePath] = checksum;
+            }
+
+            return checksum;
+        }
+
+        private string HashBytesToString(byte[] hash)
+        {
+            var data = BitConverter.ToString(hash);
+            return data.Replace("-", "");
+        }
+    }
+}

src/Tools/SignRoslyn/PathUtil.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
 
@@ -19,6 +20,5 @@ internal static bool IsAssembly(string fileName)
             var ext = Path.GetExtension(fileName);
             return ext == ".exe" || ext == ".dll";
         }
-
     }
 }

src/Tools/SignRoslyn/RunSignUtil.cs

@@ -23,8 +23,11 @@ internal sealed class RunSignUtil
              + sizeof(Int16)  // minor version
              + sizeof(Int64); // metadata directory
 
+        internal static readonly StringComparer FilePathComparer = StringComparer.OrdinalIgnoreCase;
+
         private readonly SignData _signData;
         private readonly ISignTool _signTool;
+        private readonly ContentUtil _contentUtil = new ContentUtil();
 
         internal RunSignUtil(ISignTool signTool, SignData signData)
         {
@@ -36,7 +39,7 @@ internal void Go()
         {
             // First validate our inputs and give a useful error message about anything that happens to be missing
             // from the binaries directory.
-            ValidateBinaries();
+            var vsixDataMap = VerifyBeforeSign();
 
             // Next remove public sign from all of the assemblies.  It can interfere with the signing process.
             RemovePublicSign();
@@ -45,7 +48,7 @@ internal void Go()
             SignAssemblies();
 
             // Last we sign the VSIX files (being careful to take into account nesting)
-            SignVsixes();
+            SignVsixes(vsixDataMap);
         }
 
         private void RemovePublicSign()
@@ -54,8 +57,7 @@ private void RemovePublicSign()
             foreach (var name in _signData.AssemblyNames)
             {
                 Console.WriteLine($"\t{name}");
-                var path = Path.Combine(_signData.RootBinaryPath, name);
-                RemovePublicSign(path);
+                RemovePublicSign(name.FullPath);
             }
         }
 
@@ -67,36 +69,38 @@ private void SignAssemblies()
             Console.WriteLine("Signing assemblies");
             foreach (var name in _signData.AssemblyNames)
             {
-                Console.WriteLine($"\t{name}");
+                Console.WriteLine($"\t{name.RelativePath}");
             }
 
-            _signTool.Sign(_signData.AssemblyNames.Select(x => Path.Combine(_signData.RootBinaryPath, x)));
+            _signTool.Sign(_signData.AssemblyNames.Select(x => x.FullPath));
         }
 
         /// <summary>
         /// Sign all of the VSIX files.  It is possible for VSIX to nest other VSIX so we must consider this when 
         /// picking the order.
         /// </summary>
-        private void SignVsixes()
+        private void SignVsixes(Dictionary<BinaryName, VsixData> vsixDataMap)
         {
             var round = 0;
-            var signedSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            var signedSet = new HashSet<BinaryName>(_signData.AssemblyNames);
             var toSignList = _signData.VsixNames.ToList();
             do
             {
                 Console.WriteLine($"Signing VSIX round {round}");
-                var list = new List<string>();
+                var list = new List<BinaryName>();
                 var i = 0;
                 var progress = false;
                 while (i < toSignList.Count)
                 {
                     var vsixName = toSignList[i];
-                    if (AreNestedVsixSigned(vsixName, signedSet))
+                    var vsixData = vsixDataMap[vsixName];
+                    var areNestedBinariesSigned = vsixData.NestedBinaryParts.All(x => signedSet.Contains(x.BinaryName));
+                    if (areNestedBinariesSigned)
                     {
                         list.Add(vsixName);
                         toSignList.RemoveAt(i);
                         Console.WriteLine($"\tRepacking {vsixName}");
-                        Repack(vsixName);
+                        Repack(vsixData);
                         progress = true;
                     }
                     else
@@ -111,7 +115,7 @@ private void SignVsixes()
                 }
 
                 Console.WriteLine($"\tSigning ...");
-                _signTool.Sign(list.Select(x => Path.Combine(_signData.RootBinaryPath, x)));
+                _signTool.Sign(list.Select(x => x.FullPath));
 
                 // Signing is complete so now we can update the signed set.
                 list.ForEach(x => signedSet.Add(x));
@@ -123,24 +127,20 @@ private void SignVsixes()
         /// <summary>
         /// Repack the VSIX with the signed parts from the binaries directory.
         /// </summary>
-        private void Repack(string vsixName)
+        private void Repack(VsixData vsixData)
         {
-            var vsixPath = Path.Combine(_signData.RootBinaryPath, vsixName);
-            using (var package = Package.Open(vsixPath, FileMode.Open, FileAccess.ReadWrite))
+            using (var package = Package.Open(vsixData.Name.FullPath, FileMode.Open, FileAccess.ReadWrite))
             {
                 foreach (var part in package.GetParts())
                 {
                     var relativeName = GetPartRelativeFileName(part);
-                    var name = Path.GetFileName(relativeName);
-
-                    // Only need to repack assemblies and VSIX parts.
-                    if (!IsVsix(name) && !IsAssembly(name))
+                    var vsixPart = vsixData.GetNestedBinaryPart(relativeName);
+                    if (!vsixPart.HasValue)
                     {
                         continue;
                     }
 
-                    var signedPath = Path.Combine(_signData.RootBinaryPath, name);
-                    using (var stream = File.OpenRead(signedPath))
+                    using (var stream = File.OpenRead(vsixPart.Value.BinaryName.FullPath))
                     using (var partStream = part.GetStream(FileMode.Open, FileAccess.ReadWrite))
                     {
                         stream.CopyTo(partStream);
@@ -153,12 +153,12 @@ private void Repack(string vsixName)
         /// <summary>
         /// Get the name of all VSIX which are nested inside this VSIX.
         /// </summary>
-        private IEnumerable<string> GetNestedVsixRelativeNames(string vsixName)
+        private IEnumerable<string> GetNestedVsixRelativeNames(BinaryName vsixName)
         {
             return GetVsixPartRelativeNames(vsixName).Where(x => IsVsix(x));
         }
 
-        private bool AreNestedVsixSigned(string vsixName, HashSet<string> signedSet)
+        private bool AreNestedVsixSigned(BinaryName vsixName, HashSet<string> signedSet)
         {
             foreach (var relativeName in GetNestedVsixRelativeNames(vsixName))
             {
@@ -175,11 +175,10 @@ private bool AreNestedVsixSigned(string vsixName, HashSet<string> signedSet)
         /// <summary>
         /// Return all the assembly and VSIX contents nested in the VSIX
         /// </summary>
-        private List<string> GetVsixPartRelativeNames(string vsixName)
+        private List<string> GetVsixPartRelativeNames(BinaryName vsixName)
         {
             var list = new List<string>();
-            var vsixPath = Path.Combine(_signData.RootBinaryPath, vsixName);
-            using (var package = Package.Open(vsixPath, FileMode.Open, FileAccess.Read))
+            using (var package = Package.Open(vsixName.FullPath, FileMode.Open, FileAccess.Read))
             {
                 foreach (var part in package.GetParts())
                 {
@@ -191,65 +190,108 @@ private List<string> GetVsixPartRelativeNames(string vsixName)
             return list;
         }
 
-        private void ValidateBinaries()
+        private Dictionary<BinaryName, VsixData> VerifyBeforeSign()
         {
-            if (!ValidateBinariesExist() || !ValidateVsixContents())
+            var allGood = true;
+            var map = VerifyBinariesBeforeSign(ref allGood);
+            var vsixDataMap = VerifyVsixContentsBeforeSign(map, ref allGood);
+
+            if (!allGood)
             {
                 throw new Exception("Errors validating the state before signing");
             }
+
+            return vsixDataMap;
         }
 
-        private bool ValidateBinariesExist()
+        /// <summary>
+        /// Validate all of the binaries which are specified to be signed exist on disk.  Compute their
+        /// checksums at this time so we can use it for VSIX content validation.
+        /// </summary>
+        private Dictionary<string, BinaryName> VerifyBinariesBeforeSign(ref bool allGood)
         {
-            var allGood = true;
+            var checksumToNameMap = new Dictionary<string, BinaryName>(StringComparer.Ordinal);
             foreach (var binaryName in _signData.BinaryNames)
             {
-                var path = Path.Combine(_signData.RootBinaryPath, binaryName);
-                if (!File.Exists(path))
+                if (!File.Exists(binaryName.FullPath))
                 {
-                    Console.WriteLine($"Did not find {binaryName} at {path}");
+                    Console.WriteLine($"Did not find {binaryName} at {binaryName.FullPath}");
                     allGood = false;
                 }
+
+                var checksum = _contentUtil.GetChecksum(binaryName.FullPath);
+                checksumToNameMap[checksum] = binaryName;
             }
 
-            return allGood;
+            return checksumToNameMap;
         }
 
-        private bool ValidateVsixContents()
+        private Dictionary<BinaryName, VsixData> VerifyVsixContentsBeforeSign(Dictionary<string, BinaryName> checksumToNameMap, ref bool allGood)
         {
-            var allBinaryNames = _signData.BinaryNames.Concat(_signData.ExcludeBinaryNames);
-            var allBinarySet = new HashSet<string>(allBinaryNames, StringComparer.OrdinalIgnoreCase);
-            var allGood = true;
+            var vsixDataMap = new Dictionary<BinaryName, VsixData>();
             foreach (var vsixName in _signData.VsixNames)
             {
-                if (!ValidateVsixContents(vsixName, allBinarySet))
-                {
-                    allGood = false;
-                }
+                var data = VerifyVsixContentsBeforeSign(vsixName, checksumToNameMap, ref allGood);
+                vsixDataMap[vsixName] = data;
             }
 
-            return allGood;
+            return vsixDataMap;
         }
 
-        private bool ValidateVsixContents(string vsixName, HashSet<string> allBinarySet)
+        private VsixData VerifyVsixContentsBeforeSign(BinaryName vsixName, Dictionary<string, BinaryName> checksumToNameMap, ref bool allGood)
         {
-            var allGood = true;
-            foreach (var relativeName in GetVsixPartRelativeNames(vsixName))
+            var nestedExternalBinaries = new List<string>();
+            var nestedParts = new List<VsixPart>();
+            using (var package = Package.Open(vsixName.FullPath, FileMode.Open, FileAccess.Read))
             {
-                var name = Path.GetFileName(relativeName);
-                if (!IsVsix(name) && !IsAssembly(name))
+                foreach (var part in package.GetParts())
                 {
-                    continue;
-                }
+                    var relativeName = GetPartRelativeFileName(part);
+                    var name = Path.GetFileName(relativeName);
+                    if (!IsVsix(name) && !IsAssembly(name))
+                    {
+                        continue;
+                    }
 
-                if (!allBinarySet.Contains(name))
-                {
-                    Console.WriteLine($"Vsix {vsixName} contains assembly {name} which is not in the binary list");
-                    allGood = false;
+                    if (_signData.ExternalBinaryNames.Contains(name))
+                    {
+                        nestedExternalBinaries.Add(name);
+                        continue;
+                    }
+
+                    if (!_signData.BinaryNames.Any(x => FilePathComparer.Equals(x.Name, name)))
+                    {
+                        allGood = false;
+                        Console.WriteLine($"VSIX {vsixName} has part {name} which is not listed in the sign or external list");
+                        continue;
+                    }
+
+                    // This represents a binary that we need to sign.  Ensure the content in the VSIX is the same as the 
+                    // content in the binaries directory by doing a chekcsum match.
+                    using (var stream = part.GetStream())
+                    {
+                        string checksum = _contentUtil.GetChecksum(stream);
+                        BinaryName checksumName;
+                        if (!checksumToNameMap.TryGetValue(checksum, out checksumName))
+                        {
+                            allGood = false;
+                            Console.WriteLine($"{vsixName} has part {name} which does not match the content in the binaries directory");
+                            continue;
+                        }
+
+                        if (!FilePathComparer.Equals(checksumName.Name, name))
+                        {
+                            allGood = false;
+                            Console.WriteLine($"{vsixName} has part {name} with a different name in the binaries directory: {checksumName}");
+                            continue;
+                        }
+
+                        nestedParts.Add(new VsixPart(relativeName, checksumName));
+                    }
                 }
             }
 
-            return allGood;
+            return new VsixData(vsixName, nestedParts.ToImmutableArray(), nestedExternalBinaries.ToImmutableArray());
         }
 
         private static string GetPartRelativeFileName(PackagePart part)
@@ -299,6 +341,5 @@ private void RemovePublicSign(string assemblyPath, bool force = false)
                 writer.Write((UInt32)(peReader.PEHeaders.CorHeader.Flags | CorFlags.StrongNameSigned));
             }
         }
-
     }
 }

src/Tools/SignRoslyn/SignData.cs

+using System;
+using System.Linq;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+
+namespace SignRoslyn
+{
+    internal sealed class SignData
+    {
+        /// <summary>
+        /// The path where the binaries are built to: e:\path\to\source\Binaries\Debug
+        /// </summary>
+        internal string RootBinaryPath { get; }
+
+        /// <summary>
+        /// The names of the binaries to be signed.  These are all relative paths off of the <see cref="RootBinaryPath"/>
+        /// property.
+        /// </summary>
+        internal ImmutableArray<BinaryName> BinaryNames { get; }
+
+        /// <summary>
+        /// These are binaries which are included in our VSIX files but are already signed.  This list is used for 
+        /// validation purpsoes.  These are all flat names and cannot be relative paths.
+        /// </summary>
+        internal ImmutableArray<string> ExternalBinaryNames { get;}
+
+        /// <summary>
+        /// Names of assemblies that need to be signed.  This is a subste of <see cref="BinaryNames"/>
+        /// </summary>
+        internal ImmutableArray<BinaryName> AssemblyNames { get; }
+
+        /// <summary>
+        /// Names of VSIX that need to be signed.  This is a subste of <see cref="BinaryNames"/>
+        /// </summary>
+        internal ImmutableArray<BinaryName> VsixNames { get; }
+
+        internal SignData(string rootBinaryPath, IEnumerable<string> binaryNames, IEnumerable<string> externalBinaryNames)
+        {
+            RootBinaryPath = rootBinaryPath;
+
+            // Use order by to make the output of this tool as predictable as possible.
+            BinaryNames = binaryNames.OrderBy(x => x).Select(x => new BinaryName(rootBinaryPath, x)).ToImmutableArray();
+            ExternalBinaryNames = externalBinaryNames.OrderBy(x => x).ToImmutableArray();
+
+            AssemblyNames = BinaryNames.Where(x => x.IsAssembly).ToImmutableArray();
+            VsixNames = BinaryNames.Where(x => x.IsVsix).ToImmutableArray();
+        }
+    }
+}
+

src/Tools/SignRoslyn/SignRoslyn.csproj

@@ -22,6 +22,8 @@
     <Reference Include="WindowsBase" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="BinaryName.cs" />
+    <Compile Include="ContentUtil.cs" />
     <Compile Include="FileJson.cs" />
     <Compile Include="ISignTool.cs" />
     <Compile Include="NopSignTool.cs" />
@@ -30,6 +32,7 @@
     <Compile Include="RunSignUtil.cs" />
     <Compile Include="SignData.cs" />
     <Compile Include="SignTool.cs" />
+    <Compile Include="VsixData.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />

src/Tools/SignRoslyn/VsixData.cs

+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace SignRoslyn
+{
+    internal sealed class VsixData
+    {
+        /// <summary>
+        /// Name of the VSIX
+        /// </summary>
+        internal BinaryName Name { get; }
+
+        /// <summary>
+        /// The set of binaries nested inside this VSIX.
+        /// </summary>
+        internal ImmutableArray<VsixPart> NestedBinaryParts;
+
+        /// <summary>
+        /// The set of external binaries this VSIX depends on.
+        /// </summary>
+        internal ImmutableArray<string> NestedExternalNames { get; }
+
+        internal VsixData(BinaryName name, ImmutableArray<VsixPart> nestedBinaryParts, ImmutableArray<string> nestedExternalNames)
+        {
+            Name = name;
+            NestedBinaryParts = nestedBinaryParts;
+            NestedExternalNames = nestedExternalNames;
+        }
+
+        internal VsixPart? GetNestedBinaryPart(string relativeName)
+        {
+            foreach (var part in NestedBinaryParts)
+            {
+                if (relativeName == part.RelativeName)
+                {
+                    return part;
+                }
+            }
+
+            return null;
+        }
+    }
+
+    internal struct VsixPart
+    {
+        internal string RelativeName { get; }
+        internal BinaryName BinaryName { get; }
+
+        internal VsixPart(string relativeName, BinaryName binaryName)
+        {
+            RelativeName = relativeName;
+            BinaryName = binaryName;
+        }
+
+        public override string ToString() => $"{RelativeName} -> {BinaryName.RelativePath}";
+    }
+}