Merge pull request #15421 from jaredpar/fix-build

Extend RepoUtil so it can support our internal repo

build/Targets/Dependencies.props

@@ -12,6 +12,7 @@
     <MicrosoftCodeAnalysisAnalyzersVersion>1.1.0</MicrosoftCodeAnalysisAnalyzersVersion>
     <MicrosoftCompositionVersion>1.0.27</MicrosoftCompositionVersion>
     <MicrosoftDiagnosticsRuntimeVersion>0.8.31-beta</MicrosoftDiagnosticsRuntimeVersion>
+    <MicrosoftDiagnosticsTracingTraceEventVersion>1.0.35</MicrosoftDiagnosticsTracingTraceEventVersion>
     <MicrosoftDiaSymReaderVersion>1.1.0-beta1-60625-03</MicrosoftDiaSymReaderVersion>
     <MicrosoftDiaSymReaderNativeVersion>1.5.0-beta1</MicrosoftDiaSymReaderNativeVersion>
     <MicrosoftDiaSymReaderPortablePdbVersion>1.2.0-beta1-60831-01</MicrosoftDiaSymReaderPortablePdbVersion>
@@ -32,8 +33,10 @@
     <SystemDiagnosticsToolsVersion>4.0.1</SystemDiagnosticsToolsVersion>
     <SystemDynamicRuntimeVersion>4.0.11</SystemDynamicRuntimeVersion>
     <SystemGlobalizationVersion>4.0.11</SystemGlobalizationVersion>
+    <SystemIdentityModelTokensJwtVersion>5.0.0</SystemIdentityModelTokensJwtVersion>
     <SystemIOVersion>4.1.0</SystemIOVersion>
     <SystemIOCompressionVersion>4.1.0</SystemIOCompressionVersion>
+    <SystemIOCompressionZipFileVersion>4.0.1</SystemIOCompressionZipFileVersion>
     <SystemIOFileSystemVersion>4.0.1</SystemIOFileSystemVersion>
     <SystemIOFileSystemDriveInfoVersion>4.0.0</SystemIOFileSystemDriveInfoVersion>
     <SystemIOFileSystemPrimitivesVersion>4.0.1</SystemIOFileSystemPrimitivesVersion>

build/ToolsetPackages/closed.project.json

+{
+    "dependencies": {
+        "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.35",
+        "System.IdentityModel.Tokens.Jwt": "5.0.0",
+        "System.IO.Compression.ZipFile": "4.0.1"
+    },
+    "frameworks": {
+        "net46": {}
+    }
+}

src/Tools/RepoUtil/ChangeCommand.cs

@@ -17,10 +17,12 @@ namespace RepoUtil
     internal sealed class ChangeCommand : ICommand
     {
         private readonly RepoData _repoData;
+        private readonly string _generateDirectory;
 
-        internal ChangeCommand(RepoData repoData)
+        internal ChangeCommand(RepoData repoData, string generateDir)
         {
             _repoData = repoData;
+            _generateDirectory = generateDir;
         }
 
         public bool Run(TextWriter writer, string[] args)
@@ -183,7 +185,7 @@ private void ChangeAllCore(ImmutableDictionary<NuGetPackage, NuGetPackage> chang
         private void ChangeProjectJsonFiles(ImmutableDictionary<NuGetPackage, NuGetPackage> changeMap)
         {
             Console.WriteLine("Changing project.json files");
-            foreach (var filePath in ProjectJsonUtil.GetProjectJsonFiles(_repoData.SourcesPath))
+            foreach (var filePath in ProjectJsonUtil.GetProjectJsonFiles(_repoData.SourcesDirectory))
             {
                 if (ProjectJsonUtil.ChangeDependencies(filePath, changeMap))
                 {
@@ -197,7 +199,7 @@ private void ChangeGeneratedFiles()
             var msbuildData = _repoData.RepoConfig.MSBuildGenerateData;
             if (msbuildData.HasValue)
             {
-                var fileName = new FileName(_repoData.SourcesPath, msbuildData.Value.RelativeFileName);
+                var fileName = new FileName(_generateDirectory, msbuildData.Value.RelativeFilePath);
                 var packages = GenerateUtil.GetFilteredPackages(msbuildData.Value, _repoData);
                 GenerateUtil.WriteMSBuildContent(fileName, packages);
             }

src/Tools/RepoUtil/GenerateData.cs

@@ -14,12 +14,12 @@ namespace RepoUtil
     /// </summary>
     internal struct GenerateData
     {
-        internal string RelativeFileName { get; }
+        internal string RelativeFilePath { get; }
         internal ImmutableArray<Regex> Packages { get; }
 
         internal GenerateData(string relativeFileName, ImmutableArray<Regex> packages)
         {
-            RelativeFileName = relativeFileName;
+            RelativeFilePath = relativeFileName;
             Packages = packages;
         }
     }

src/Tools/RepoUtil/Program.cs

@@ -13,11 +13,12 @@ internal static class Program
         private sealed class ParsedArgs
         {
             internal string RepoUtilDataPath { get; set; }
-            internal string SourcesPath { get; set; }
+            internal string SourcesDirectory { get; set; }
+            internal string GenerateDirectory { get; set; }
             internal string[] RemainingArgs { get; set; }
         }
 
-        private delegate ICommand CreateCommand(RepoConfig repoConfig, string sourcesPath);
+        private delegate ICommand CreateCommand(RepoConfig repoConfig, string sourcesDir, string generateDir);
 
         internal static int Main(string[] args)
         {
@@ -54,11 +55,12 @@ private static bool Run(string[] args)
             CreateCommand func;
             if (!TryParseCommandLine(args, out parsedArgs, out func))
             {
+                Usage();
                 return false;
             }
 
             var repoConfig = RepoConfig.ReadFrom(parsedArgs.RepoUtilDataPath);
-            var command = func(repoConfig, parsedArgs.SourcesPath);
+            var command = func(repoConfig, parsedArgs.SourcesDirectory, parsedArgs.GenerateDirectory);
             if (command.Run(Console.Out, parsedArgs.RemainingArgs))
             {
                 return true;
@@ -89,8 +91,9 @@ private static bool TryParseCommandLine(string[] args, out ParsedArgs parsedArgs
                 return false;
             }
 
-            parsedArgs.SourcesPath = parsedArgs.SourcesPath ?? GetDirectoryName(AppContext.BaseDirectory, 5);
-            parsedArgs.RepoUtilDataPath = parsedArgs.RepoUtilDataPath ?? Path.Combine(parsedArgs.SourcesPath, @"build\config\RepoUtilData.json");
+            parsedArgs.SourcesDirectory = parsedArgs.SourcesDirectory ?? GetDirectoryName(AppContext.BaseDirectory, 5);
+            parsedArgs.GenerateDirectory = parsedArgs.GenerateDirectory ?? parsedArgs.SourcesDirectory;
+            parsedArgs.RepoUtilDataPath = parsedArgs.RepoUtilDataPath ?? Path.Combine(parsedArgs.SourcesDirectory, @"build\config\RepoUtilData.json");
             parsedArgs.RemainingArgs = index >= args.Length
                 ? Array.Empty<string>()
                 : args.Skip(index).ToArray();
@@ -124,7 +127,7 @@ private static bool TryParseCommon(string[] args, ref int index, ParsedArgs pars
                         {
                             if (index < args.Length)
                             {
-                                parsedArgs.SourcesPath = args[index];
+                                parsedArgs.SourcesDirectory = args[index];
                                 index++;
                             }
                             else
@@ -134,6 +137,20 @@ private static bool TryParseCommon(string[] args, ref int index, ParsedArgs pars
                             }
                             break;
                         }
+                    case "-generatepath":
+                        {
+                            if (index < args.Length)
+                            {
+                                parsedArgs.GenerateDirectory = args[index];
+                                index++;
+                            }
+                            else
+                            {
+                                Console.WriteLine($"The -generatePath switch needs a value");
+                                return false;
+                            }
+                            break;
+                        }
                     case "-config":
                         {
                             if (index < args.Length)
@@ -171,19 +188,19 @@ private static bool TryParseCommand(string[] args, ref int index, out CreateComm
             switch (name)
             {
                 case "verify":
-                    func = (c, s) => new VerifyCommand(c, s);
+                    func = (c, s, g) => new VerifyCommand(c, s,g );
                     break;
                 case "view":
-                    func = (c, s) => new ViewCommand(c, s);
+                    func = (c, s, g) => new ViewCommand(c, s);
                     break;
                 case "consumes":
-                    func = (c, s) => new ConsumesCommand(RepoData.Create(c, s));
+                    func = (c, s, g) => new ConsumesCommand(RepoData.Create(c, s));
                     break;
                 case "change":
-                    func = (c, s) => new ChangeCommand(RepoData.Create(c, s));
+                    func = (c, s, g) => new ChangeCommand(RepoData.Create(c, s), g);
                     break;
                 case "produces":
-                    func = (c, s) => new ProducesCommand(c, s);
+                    func = (c, s, g) => new ProducesCommand(c, s);
                     break;
                 default:
                     Console.Write($"Command {name} is not recognized");
@@ -193,5 +210,10 @@ private static bool TryParseCommand(string[] args, ref int index, out CreateComm
             index++;
             return true;
         }
+
+        private static void Usage()
+        {
+            Console.WriteLine("RepoUtil [-sourcesPath <sources path>] [-generatePath <generate path>] [-config <config path>] [verify|view|consumes|change|produces]");
+        }
     }
 }

src/Tools/RepoUtil/RepoData.cs

@@ -12,7 +12,7 @@ namespace RepoUtil
 {
     internal sealed class RepoData
     {
-        internal string SourcesPath { get; }
+        internal string SourcesDirectory { get; }
         internal RepoConfig RepoConfig { get; }
         internal ImmutableArray<NuGetFeed> NuGetFeeds { get; }
         internal ImmutableArray<NuGetPackage> FloatingBuildPackages { get; }
@@ -21,9 +21,9 @@ internal sealed class RepoData
         internal ImmutableArray<NuGetPackage> FixedPackages => RepoConfig.FixedPackages;
         internal ImmutableArray<NuGetPackage> AllPackages { get; }
 
-        private RepoData(RepoConfig config, string sourcesPath, IEnumerable<NuGetFeed> nugetFeeds, IEnumerable<NuGetPackage> floatingPackages)
+        private RepoData(RepoConfig config, string sourcesDir, IEnumerable<NuGetFeed> nugetFeeds, IEnumerable<NuGetPackage> floatingPackages)
         {
-            SourcesPath = sourcesPath;
+            SourcesDirectory = sourcesDir;
             RepoConfig = config;
             NuGetFeeds = nugetFeeds.ToImmutableArray();
             FloatingToolsetPackages = floatingPackages
@@ -56,10 +56,10 @@ private static ImmutableArray<NuGetPackage> Combine(params ImmutableArray<NuGetP
         /// state of the repo and add in the current data.  If any conflicting package definitions are detected this method 
         /// will throw.
         /// </summary>
-        internal static RepoData Create(RepoConfig config, string sourcesPath)
+        internal static RepoData Create(RepoConfig config, string sourcesDir)
         {
             List<NuGetPackageConflict> conflicts;
-            var repoData = Create(config, sourcesPath, out conflicts);
+            var repoData = Create(config, sourcesDir, out conflicts);
             if (conflicts?.Count > 0)
             {
                 throw new ConflictingPackagesException(conflicts);
@@ -68,10 +68,10 @@ internal static RepoData Create(RepoConfig config, string sourcesPath)
             return repoData;
         }
 
-        internal static RepoData Create(RepoConfig config, string sourcesPath, out List<NuGetPackageConflict> conflicts)
+        internal static RepoData Create(RepoConfig config, string sourcesDir, out List<NuGetPackageConflict> conflicts)
         {
             var nugetFeeds = new List<NuGetFeed>();
-            foreach (var nugetConfig in NuGetConfigUtil.GetNuGetConfigFiles(sourcesPath))
+            foreach (var nugetConfig in NuGetConfigUtil.GetNuGetConfigFiles(sourcesDir))
             {
                 var nugetFeed = NuGetConfigUtil.GetNuGetFeeds(nugetConfig);
                 nugetFeeds.AddRange(nugetFeed);
@@ -81,14 +81,14 @@ internal static RepoData Create(RepoConfig config, string sourcesPath, out List<
 
             var fixedPackageSet = new HashSet<NuGetPackage>(config.FixedPackages, default(Constants.IgnoreGenerateNameComparer));
             var floatingPackageMap = new Dictionary<string, NuGetPackageSource>(Constants.NugetPackageNameComparer);
-            foreach (var filePath in ProjectJsonUtil.GetProjectJsonFiles(sourcesPath))
+            foreach (var filePath in ProjectJsonUtil.GetProjectJsonFiles(sourcesDir))
             {
                 if (config.ProjectJsonExcludes.Any(x => x.IsMatch(filePath)))
                 {
                     continue;
                 }
 
-                var fileName = FileName.FromFullPath(sourcesPath, filePath);
+                var fileName = FileName.FromFullPath(sourcesDir, filePath);
                 foreach (var package in ProjectJsonUtil.GetDependencies(filePath))
                 {
                     if (fixedPackageSet.Contains(package))
@@ -116,7 +116,7 @@ internal static RepoData Create(RepoConfig config, string sourcesPath, out List<
                 }
             }
 
-            return new RepoData(config, sourcesPath, nugetFeeds, floatingPackageMap.Values.Select(x => x.NuGetPackage));
+            return new RepoData(config, sourcesDir, nugetFeeds, floatingPackageMap.Values.Select(x => x.NuGetPackage));
         }
     }
 }

src/Tools/RepoUtil/VerifyCommand.cs

@@ -14,13 +14,15 @@ namespace RepoUtil
     /// </summary>
     internal sealed class VerifyCommand : ICommand
     {
-        private readonly string _sourcesPath;
+        private readonly string _sourcesDirectory;
+        private readonly string _generateDirectory;
         private readonly RepoConfig _repoConfig;
 
-        internal VerifyCommand(RepoConfig repoConfig, string sourcesPath)
+        internal VerifyCommand(RepoConfig repoConfig, string sourcesDir, string generateDir)
         {
             _repoConfig = repoConfig;
-            _sourcesPath = sourcesPath;
+            _sourcesDirectory = sourcesDir;
+            _generateDirectory = generateDir;
         }
 
         public bool Run(TextWriter writer, string[] args)
@@ -46,7 +48,7 @@ private bool VerifyProjectJsonContents(TextWriter writer, out RepoData repoData)
             writer.WriteLine($"Verifying project.json contents");
 
             List<NuGetPackageConflict> conflicts;
-            repoData = RepoData.Create(_repoConfig, _sourcesPath, out conflicts);
+            repoData = RepoData.Create(_repoConfig, _sourcesDirectory, out conflicts);
             if (conflicts?.Count > 0)
             { 
                 foreach (var conflict in conflicts)
@@ -72,7 +74,7 @@ private bool VerifyRepoConfig(TextWriter writer)
         {
             writer.WriteLine($"Verifying RepoData.json");
             var packages = ProjectJsonUtil
-                .GetProjectJsonFiles(_sourcesPath)
+                .GetProjectJsonFiles(_sourcesDirectory)
                 .SelectMany(x => ProjectJsonUtil.GetDependencies(x));
             var set = new HashSet<NuGetPackage>(packages, default(Constants.IgnoreGenerateNameComparer));
             var allGood = true;
@@ -99,7 +101,7 @@ private bool VerifyGeneratedFiles(TextWriter writer, RepoData repoData)
                 var packages = GenerateUtil.GetFilteredPackages(data, repoData);
 
                 // Need to verify the contents of the generated file are correct.
-                var fileName = new FileName(_sourcesPath, data.RelativeFileName);
+                var fileName = new FileName(_generateDirectory, data.RelativeFilePath);
                 var actualContent = File.ReadAllText(fileName.FullPath, GenerateUtil.Encoding);
                 var expectedContent = GenerateUtil.GenerateMSBuildContent(packages);
                 if (actualContent != expectedContent)

src/Tools/RepoUtil/ViewCommand.cs

@@ -131,7 +131,7 @@ private bool VerifyGeneratedFiles(TextWriter writer, RepoData repoData)
                 var packages = GenerateUtil.GetFilteredPackages(data, repoData);
 
                 // Need to verify the contents of the generated file are correct.
-                var fileName = new FileName(_sourcesPath, data.RelativeFileName);
+                var fileName = new FileName(_sourcesPath, data.RelativeFilePath);
                 var actualContent = File.ReadAllText(fileName.FullPath, GenerateUtil.Encoding);
                 var expectedContent = GenerateUtil.GenerateMSBuildContent(packages);
                 if (actualContent != expectedContent)