Package verification

src/Tools/RepoUtil/Data.cs

@@ -7,7 +7,7 @@
 
 namespace RepoUtil
 {
-    class Data
+    internal static class Data
     {
         /// <summary>
         /// The dependencies in these project.json files are expected to be regularly updated by 
@@ -87,8 +87,8 @@ class Data
             @"src\Tools\Source\RunTests\project.json",
         };
 
-        internal static readonly ImmutableArray<string> FloatingList = ImmutableArray.Create(s_floatingList);
-        internal static readonly ImmutableArray<string> StaticList = ImmutableArray.Create(s_staticList);
+        internal static readonly ImmutableArray<string> FloatingList = s_floatingList.OrderBy(x => x).ToImmutableArray();
+        internal static readonly ImmutableArray<string> StaticList = s_staticList.OrderBy(x => x).ToImmutableArray();
 
         internal static ImmutableArray<FileName> GetFloatingFileNames(string sourcesPath)
         {

src/Tools/RepoUtil/FileName.cs

@@ -20,6 +20,12 @@ internal FileName(string rootPath, string relativePath)
             RelativePath = relativePath;
         }
 
+        internal static FileName FromFullPath(string rootPath, string fullPath)
+        {
+            fullPath = fullPath.Substring(rootPath.Length + 1);
+            return new FileName(rootPath, fullPath);
+        }
+
         public static bool operator ==(FileName left, FileName right) => left.FullPath == right.FullPath;
         public static bool operator !=(FileName left, FileName right) => !(left == right);
         public bool Equals(FileName other) => this == other;

src/Tools/RepoUtil/Program.cs

@@ -33,13 +33,14 @@ private static bool Run(string[] args)
                 return false;
             }
 
+            var repoData = RepoData.ReadFrom(Path.Combine(AppContext.BaseDirectory, "RepoData.json"));
             switch (mode)
             {
                 case Mode.Usage:
                     Usage();
                     return true;
                 case Mode.Verify:
-                    return Verify(sourcesPath);
+                    return VerifyUtil.Go(sourcesPath, repoData);
                 case Mode.Consumes:
                     return Consumes(sourcesPath);
                 default:
@@ -47,49 +48,27 @@ private static bool Run(string[] args)
             }
         }
 
-        /// <summary>
-        /// Verify our repo is consistent and this tool has enough information to make changes to the state.
-        ///
-        /// TOOD: should this run before every command?
-        /// </summary>
-        private static bool Verify(string sourcesPath)
-        {
-            var fileNames = Data
-                .StaticList
-                .Concat(Data.FloatingList)
-                .Select(x => new FileName(sourcesPath, x));
-            var allGood = false;
-            foreach (var fileName in fileNames)
-            {
-                if (!File.Exists(fileName.FullPath))
-                {
-                    Console.WriteLine($"Project file {fileName} does not exist");
-                    allGood = false;
-                }
-            }
-
-            if (!allGood)
-            {
-                return false;
-            }
-
-            return ProjectJsonUtil.VerifyTracked(sourcesPath, fileNames);
-        }
-
         private static bool Consumes(string sourcesPath)
         {
-            var map = new Dictionary<string, NuGetReference>(StringComparer.Ordinal);
-            foreach (var fileName in Data.GetFloatingFileNames(sourcesPath))
+            var set = new HashSet<string>(StringComparer.Ordinal);
+            var list = new List<string>();
+            var all = Data.FloatingList.Concat(Data.StaticList).Select(x => new FileName(sourcesPath, x));
+            foreach (var fileName in all)
             {
                 foreach (var nugetRef in ProjectJsonUtil.GetDependencies(fileName.FullPath))
                 {
-                    map[nugetRef.Name] = nugetRef;
+                    var key = $@"""{nugetRef.Name}"" : ""{nugetRef.Version}""";
+                    if (set.Add(key))
+                    {
+                        list.Add(key);
+                    }
                 }
             }
 
-            foreach (var nugetRef in map.Values.OrderBy(x => x.Name))
+            list.Sort();
+            foreach (var key in list)
             {
-                Console.WriteLine($@"""{nugetRef.Name}"" : ""{nugetRef.Version}""");
+                Console.WriteLine(key);
             }
 
             return true;

src/Tools/RepoUtil/ProjectJsonUtil.cs

@@ -20,6 +20,7 @@ internal static bool NeedsTracking(string filePath)
             return GetDependencies(filePath).Length > 0;
         }
 
+        // TOOD: use FileName here
         internal static ImmutableArray<NuGetReference> GetDependencies(string filePath)
         {
             // Need to track any file that has dependencies

src/Tools/RepoUtil/RepoData.cs

+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace RepoUtil
+{
+    /// <summary>
+    /// Packages in the repo fall into the following groups:
+    /// 
+    /// Static Packages:
+    /// 
+    /// These are packages which should never change.  In other words if there was a scenario where a new version of the 
+    /// package was available the reference should not update to the new version.  For all time it should remain at the 
+    /// specified version.
+    ///
+    /// Because they are fixed it's possible to have multiple vesions of the same package.  For instance it's okay to 
+    /// have many versions of Newtonsoft.Json referenced here because there is no need to unify.  Or at least it's stated
+    /// that we don't need to unify.
+    ///
+    /// Floating Packages:
+    /// 
+    /// These are packages which are expected to change when new versions are available.  These are tools, dependencies, etc ...
+    /// which are expected to evolve over time and we need to move forward with those dependencies. 
+    /// 
+    /// Generally these fall into two categories:
+    ///
+    ///     Build Dependencies
+    ///     Toolset Dependencies
+    ///
+    /// This distinction is necessary to help break circular references for repos when constructing build graphs.
+    /// </summary>
+    internal class RepoData
+    {
+        /// <summary>
+        /// Fixed references which do not change during a build.
+        /// </summary>
+        internal ImmutableArray<NuGetReference> StaticPackages { get; }
+
+        /// <summary>
+        /// This is a map of static package names to the list of supported versions.
+        /// </summary>
+        internal ImmutableDictionary<string, ImmutableArray<string>> StaticPackagesMap { get; }
+
+        internal ImmutableArray<string> FloatingBuildPackages { get; }
+
+        internal ImmutableArray<string> FloatingToolsetPackages { get; }
+
+        internal ImmutableArray<string> FloatingPackages { get; }
+
+        internal RepoData(IEnumerable<NuGetReference> staticPackages, IEnumerable<string> floatingBuildPackages, IEnumerable<string> floatingToolsetPackages)
+        {
+            StaticPackages = staticPackages.OrderBy(x => x.Name).ToImmutableArray();
+
+            // TODO: Validate duplicate names in the floating lists
+            FloatingBuildPackages = floatingBuildPackages.OrderBy(x => x).ToImmutableArray();
+            FloatingToolsetPackages = floatingToolsetPackages.OrderBy(x => x).ToImmutableArray();
+            FloatingPackages = FloatingBuildPackages.Concat(floatingToolsetPackages).ToImmutableArray();
+
+            var map = new Dictionary<string, List<string>>();
+            foreach (var nugetRef in staticPackages)
+            {
+                List<string> list;
+                if (!map.TryGetValue(nugetRef.Name, out list))
+                {
+                    list = new List<string>(capacity: 1);
+                    map[nugetRef.Name] = list;
+                }
+
+                list.Add(nugetRef.Version);
+            }
+
+            StaticPackagesMap = ImmutableDictionary<string, ImmutableArray<string>>.Empty;
+            foreach (var pair in map)
+            {
+                StaticPackagesMap = StaticPackagesMap.Add(pair.Key, pair.Value.ToImmutableArray());
+            }
+        }
+
+        internal static RepoData ReadFrom(string jsonFilePath)
+        {
+            // Need to track any file that has dependencies
+            var obj = JObject.Parse(File.ReadAllText(jsonFilePath));
+            var staticPackages = (JObject)obj["staticPackages"];
+            var staticPackagesList = ImmutableArray.CreateBuilder<NuGetReference>();
+            foreach (var prop in staticPackages.Properties())
+            {
+                if (prop.Value.Type == JTokenType.String)
+                {
+                    var version = (string)prop.Value;
+                    var nugetRef = new NuGetReference(prop.Name, version);
+                    staticPackagesList.Add(nugetRef);
+                }
+                else
+                {
+                    foreach (var version in ((JArray)prop.Value).Values<string>())
+                    {
+                        var nugetRef = new NuGetReference(prop.Name, version);
+                        staticPackagesList.Add(nugetRef);
+                    }
+                }
+            }
+
+            var floatingPackages = (JObject)obj["floatingPackages"];
+            var build = (JArray)floatingPackages.Property("build").Value;
+            var toolset = (JArray)floatingPackages.Property("toolset").Value;
+
+            return new RepoData(
+                staticPackagesList,
+                build.Values<string>(),
+                toolset.Values<string>());
+        }
+    }
+}

src/Tools/RepoUtil/RepoData.json

+{
+    "staticPackages": {
+        "Microsoft.Build.Framework": [ "0.1.0-preview-00005", "0.1.0-preview-00023-160527" ],
+        "Microsoft.Build.Tasks.Core": [ "0.1.0-preview-00005", "0.1.0-preview-00023-160527" ],
+        "Microsoft.Build.Utilities.Core": [ "0.1.0-preview-00005", "0.1.0-preview-00023-160527" ],
+        "Microsoft.VisualStudio.Composition": "14.2.19-pre",
+        "Microsoft.VisualStudio.Data.Core": "9.0.21022",
+        "Microsoft.VisualStudio.Data.Services": "9.0.21022",
+        "Microsoft.VisualStudio.Debugger.Engine": [ "14.3.25420", "14.3.25421" ],
+        "Microsoft.VisualStudio.Debugger.Metadata": [ "14.3.25420", "14.3.25421" ],
+        "Microsoft.VisualStudio.Designer.Interfaces": "1.1.4322",
+        "Microsoft.VisualStudio.Editor": "14.3.25407",
+        "Microsoft.VisualStudio.ImageCatalog": "14.3.25407",
+        "Microsoft.VisualStudio.Imaging": "14.3.25407",
+        "Microsoft.VisualStudio.Imaging.Interop.14.0.DesignTime": "14.3.25407",
+        "Microsoft.VisualStudio.Language.Intellisense": [ "14.3.25407", "15.0.25123-Dev15Preview" ],
+        "Microsoft.VisualStudio.Language.StandardClassification": "14.3.25407",
+        "Microsoft.VisualStudio.ManagedInterfaces": "8.0.50727",
+        "Microsoft.VisualStudio.OLE.Interop": "7.10.6070",
+        "Microsoft.VisualStudio.SDK.EmbedInteropTypes": "14.1.2",
+        "Microsoft.VisualStudio.Shell.14.0": "14.3.25407",
+        "Microsoft.VisualStudio.Shell.Design": "14.3.25407",
+        "Microsoft.VisualStudio.Shell.Interop.10.0": "10.0.30319",
+        "Microsoft.VisualStudio.Shell.Interop.11.0": "11.0.61030",
+        "Microsoft.VisualStudio.Shell.Interop.12.1.DesignTime": "12.1.30328",
+        "Microsoft.VisualStudio.Shell.Interop.14.0.DesignTime": "14.3.25407",
+        "Microsoft.VisualStudio.Text.Data": "14.3.25407",
+        "Microsoft.VisualStudio.Text.Logic": "14.3.25407",
+        "Microsoft.VisualStudio.Text.UI": "14.3.25407",
+        "Microsoft.VisualStudio.Text.UI.Wpf": "14.3.25407",
+        "Microsoft.VisualStudio.TextManager.Interop.10.0": "10.0.30319",
+        "Microsoft.VisualStudio.TextManager.Interop.12.0": "12.0.30110",
+        "Microsoft.VisualStudio.TextManager.Interop.12.1.DesignTime": "12.1.30328",
+        "Microsoft.VisualStudio.WCFReference.Interop": "9.0.30729",
+        "Newtonsoft.Json": [ "6.0.4", "7.0.1", "8.0.2", "9.0.1" ],
+        "xunit.runner.console": [ "2.1.0", "2.2.0-beta1-build3239" ]
+    },
+
+    "floatingPackages": {
+        "build": [ ],
+        "toolset": [ ]
+    }
+}

src/Tools/RepoUtil/RepoUtil.csproj

@@ -29,9 +29,14 @@
     <Compile Include="NuGetReference.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="ProjectJsonUtil.cs" />
+    <Compile Include="RepoData.cs" />
+    <Compile Include="VerifyUtil.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />
+    <None Include="RepoData.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="project.json" />
   </ItemGroup>
   <ItemGroup>

src/Tools/RepoUtil/VerifyUtil.cs

+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace RepoUtil
+{
+    /// <summary>
+    /// This utility is used to verify the repo is in a consistent state with respect to NuGet references. 
+    /// </summary>
+    internal sealed class VerifyUtil
+    {
+        private struct NuGetReferenceSource
+        {
+            internal NuGetReference NuGetReference { get; }
+            internal FileName FileName { get; }
+
+            internal NuGetReferenceSource(NuGetReference nugetRef, FileName fileName)
+            {
+                NuGetReference = nugetRef;
+                FileName = fileName;
+            }
+        }
+
+        private readonly string _sourcesPath;
+        private readonly RepoData _repoData;
+        private readonly Dictionary<string, NuGetReferenceSource> _floatingPackageMap = new Dictionary<string, NuGetReferenceSource>(StringComparer.Ordinal);
+
+        internal VerifyUtil(string sourcesPath, RepoData repoData)
+        {
+            _sourcesPath = sourcesPath;
+            _repoData = repoData;
+        }
+
+        internal static bool Go(string sourcesPath, RepoData repoData)
+        {
+            var util = new VerifyUtil(sourcesPath, repoData);
+            return util.Go();
+        }
+
+        private bool Go()
+        {
+            return VerifyPackages();
+        }
+
+        private bool VerifyPackages()
+        {
+            var allGood = false;
+            foreach (var filePath in GetProjectJsonFiles())
+            {
+                var fileName = FileName.FromFullPath(_sourcesPath, filePath);
+                foreach (var nugetRef in ProjectJsonUtil.GetDependencies(filePath))
+                {
+                    if (_repoData.StaticPackages.Any(x => x.Name == nugetRef.Name))
+                    {
+                        if (!VerifyStaticPackage(nugetRef, fileName))
+                        {
+                            allGood = false;
+                        }
+                    }
+                    else
+                    {
+                        if (!VerifyFloatingPackage(nugetRef, fileName))
+                        {
+                            allGood = false;
+                        }
+                    }
+                }
+            }
+
+            return allGood;
+        }
+
+        private bool VerifyFloatingPackage(NuGetReference nugetRef, FileName fileName)
+        {
+            NuGetReferenceSource source;
+            if (_floatingPackageMap.TryGetValue(nugetRef.Name, out source))
+            {
+                if (source.NuGetReference.Version == nugetRef.Version)
+                {
+                    return true;
+                }
+
+                Console.WriteLine($"Package {nugetRef.Name} version differs in:");
+                Console.WriteLine($"\t{fileName} at {nugetRef.Version}");
+                Console.WriteLine($"\t{source.FileName} at {source.NuGetReference.Version}");
+                return false;
+            }
+
+            _floatingPackageMap.Add(nugetRef.Name, new NuGetReferenceSource(nugetRef, fileName));
+            return true;
+        }
+
+        private bool VerifyStaticPackage(NuGetReference nugetRef, FileName fileName)
+        {
+            Debug.Assert(_repoData.StaticPackagesMap.ContainsKey(nugetRef.Name));
+            var versions = _repoData.StaticPackagesMap[nugetRef.Name];
+            if (!versions.Contains(nugetRef.Version))
+            {
+                Console.WriteLine($"Package {nugetRef.Name} at version {nugetRef.Version} in {fileName} is not a valid version");
+                return false;
+            }
+
+            return true;
+        }
+
+        // TODO: Need to include our toolset files not named project.json.
+        private IEnumerable<string> GetProjectJsonFiles()
+        {
+            return Directory.EnumerateFiles(_sourcesPath, "project.json", SearchOption.AllDirectories);
+        }
+    }
+}