Add ability for crossgen2 to synthesize PGO histograms (#77683)

Add an option --synthesize-random-mibc. When passed, crossgen2 will use
metadata from the compilation group to synthesize PGO histograms for
methods that did not have any input profile data. Also supported for --embed-pgo-data,
in which case the synthesized PGO data gets embedded.

Adds a synthesizepgo argument to the src/tests/run.cmd/src/tests/run.sh scripts which
enable this mode for compatible tests. So testing can be done locally via:

.\src\tests\build.cmd crossgen2 checked
.\src\tests\run.cmd checked runcrossgen2tests synthesizepgo

src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs

@@ -30,6 +30,8 @@
 using ILCompiler.DependencyAnalysis.ReadyToRun;
 #endif
 
+using DependencyList = ILCompiler.DependencyAnalysisFramework.DependencyNodeCore<ILCompiler.DependencyAnalysis.NodeFactory>.DependencyList;
+
 #pragma warning disable IDE0060
 
 namespace Internal.JitInterface
@@ -556,10 +558,15 @@ private void PublishCode()
                     }
                 }
             }
+
+            if (_synthesizedPgoDependencies != null)
+            {
+                Debug.Assert(_compilation.NodeFactory.InstrumentationDataTable != null, "Expected InstrumentationDataTable to be non-null with synthesized PGO data to embed");
+                _compilation.NodeFactory.InstrumentationDataTable.EmbedSynthesizedPgoDataForMethods(ref _additionalDependencies, _synthesizedPgoDependencies);
+            }
 #else
             var methodIL = (MethodIL)HandleToObject((void*)_methodScope);
             CodeBasedDependencyAlgorithm.AddDependenciesDueToMethodCodePresence(ref _additionalDependencies, _compilation.NodeFactory, MethodBeingCompiled, methodIL);
-            _methodCodeNode.InitializeNonRelocationDependencies(_additionalDependencies);
             _methodCodeNode.InitializeDebugInfo(_debugInfo);
 
             LocalVariableDefinition[] locals = methodIL.GetLocals();
@@ -569,6 +576,8 @@ private void PublishCode()
 
             _methodCodeNode.InitializeLocalTypes(localTypes);
 #endif
+
+            _methodCodeNode.InitializeNonRelocationDependencies(_additionalDependencies);
         }
 
         private void PublishROData()
@@ -659,12 +668,12 @@ private void CompileMethodCleanup()
 
             _gcInfo = null;
             _ehClauses = null;
+            _additionalDependencies = null;
 
 #if !READYTORUN
             _debugInfo = null;
-
-            _additionalDependencies = null;
 #endif
+
             _debugLocInfos = null;
             _debugVarInfos = null;
             _lastException = null;
@@ -677,6 +686,7 @@ private void CompileMethodCleanup()
             _stashedPrecodeFixups.Clear();
             _stashedInlinedMethods.Clear();
             _ilBodiesNeeded = null;
+            _synthesizedPgoDependencies = null;
 #endif
 
             _instantiationToJitVisibleInstantiation = null;
@@ -3427,6 +3437,8 @@ private bool getTailCallHelpers(ref CORINFO_RESOLVED_TOKEN callToken, CORINFO_SI
         private byte[] _gcInfo;
         private CORINFO_EH_CLAUSE[] _ehClauses;
 
+        private DependencyList _additionalDependencies;
+
         private void allocMem(ref AllocMemArgs args)
         {
             args.hotCodeBlock = (void*)GetPin(_code = new byte[args.hotCodeSize]);
@@ -3951,11 +3963,6 @@ private uint getJitFlags(ref CORJIT_FLAGS flags, uint sizeInBytes)
             return (uint)sizeof(CORJIT_FLAGS);
         }
 
-        private PgoSchemaElem[] getPgoInstrumentationResults(MethodDesc method)
-        {
-            return _compilation.ProfileData[method]?.SchemaData;
-        }
-
         private MemoryStream _cachedMemoryStream = new MemoryStream();
 
         public static void ComputeJitPgoInstrumentationSchema(Func<object, IntPtr> objectToHandle, PgoSchemaElem[] pgoResultsSchemas, out PgoInstrumentationSchema[] nativeSchemas, MemoryStream instrumentationData, Func<TypeDesc, bool> typeFilter = null)
@@ -4033,7 +4040,19 @@ public static void ComputeJitPgoInstrumentationSchema(Func<object, IntPtr> objec
 
             if (!_pgoResults.TryGetValue(methodDesc, out PgoInstrumentationResults pgoResults))
             {
-                var pgoResultsSchemas = getPgoInstrumentationResults(methodDesc);
+#if READYTORUN
+                PgoSchemaElem[] pgoResultsSchemas = _compilation.ProfileData.GetAllowSynthesis(_compilation, methodDesc, out bool isSynthesized)?.SchemaData;
+
+                if (pgoResultsSchemas != null && isSynthesized && _compilation.ProfileData.EmbedPgoDataInR2RImage)
+                {
+                    if (_synthesizedPgoDependencies == null)
+                        _synthesizedPgoDependencies = new HashSet<MethodDesc>();
+
+                    _synthesizedPgoDependencies.Add(methodDesc);
+                }
+#else
+                PgoSchemaElem[] pgoResultsSchemas = _compilation.ProfileData[methodDesc]?.SchemaData;
+#endif
                 if (pgoResultsSchemas == null)
                 {
                     pgoResults.hr = HRESULT.E_NOTIMPL;

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/InstrumentationDataTableNode.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -22,13 +23,12 @@ public class InstrumentationDataTableNode : HeaderTableNode
     {
         private readonly NodeFactory _factory;
         private ReadyToRunSymbolNodeFactory _symbolNodeFactory;
-        private readonly MethodDesc[] _instrumentationDataMethods;
         private readonly ProfileDataManager _profileDataManager;
+        private readonly HashSet<MethodDesc> _methodsWithSynthesizedPgoData = new HashSet<MethodDesc>();
 
-        public InstrumentationDataTableNode(NodeFactory factory, MethodDesc[] instrumentationDataMethods, ProfileDataManager profileDataManager)
+        public InstrumentationDataTableNode(NodeFactory factory, ProfileDataManager profileDataManager)
         {
             _factory = factory;
-            _instrumentationDataMethods = instrumentationDataMethods;
             _profileDataManager = profileDataManager;
         }
 
@@ -183,12 +183,44 @@ public override void AppendMangledName(NameMangler nameMangler, Utf8StringBuilde
             sb.Append("__ReadyToRunInstrumentationDataTable");
         }
 
+        // Register some MDs that had synthesized PGO data created to be physically embedded by this node, and add
+        // the appropriate dependencies of the embedding to a dependency list.
+        public void EmbedSynthesizedPgoDataForMethods(ref DependencyList dependencies, IEnumerable<MethodDesc> mds)
+        {
+            PgoValueEmitter pgoEmitter = new PgoValueEmitter(_factory.CompilationModuleGroup, _symbolNodeFactory, false);
+            foreach (MethodDesc md in mds)
+            {
+                PgoSchemaElem[] schema = _profileDataManager[md].SchemaData;
+                Debug.Assert(schema != null);
+
+                lock (_methodsWithSynthesizedPgoData)
+                {
+                    _methodsWithSynthesizedPgoData.Add(md);
+                }
+
+                PgoProcessor.EncodePgoData(schema, pgoEmitter, false);
+            }
+
+            foreach (Import imp in pgoEmitter.ReferencedImports)
+            {
+                dependencies ??= new DependencyList();
+                dependencies.Add(imp, "Dependency of synthesized PGO data");
+            }
+        }
+
         protected override DependencyList ComputeNonRelocationBasedDependencies(NodeFactory factory)
         {
             PgoValueEmitter pgoEmitter = new PgoValueEmitter(_factory.CompilationModuleGroup, _symbolNodeFactory, false);
-            foreach (MethodDesc method in _instrumentationDataMethods)
+            foreach (EcmaModule inputModule in _factory.CompilationModuleGroup.CompilationModuleSet)
             {
-                PgoProcessor.EncodePgoData(_profileDataManager[method].SchemaData, pgoEmitter, false);
+                foreach (MethodDesc method in _profileDataManager.GetInputProfileDataMethodsForModule(inputModule))
+                {
+                    PgoSchemaElem[] schema = _profileDataManager[method].SchemaData;
+                    if (schema != null)
+                    {
+                        PgoProcessor.EncodePgoData(schema, pgoEmitter, false);
+                    }
+                }
             }
             DependencyListEntry[] symbols = new DependencyListEntry[pgoEmitter.ReferencedImports.Count];
             for (int i = 0; i < symbols.Length; i++)
@@ -199,7 +231,6 @@ protected override DependencyList ComputeNonRelocationBasedDependencies(NodeFact
             return new DependencyList(symbols);
         }
 
-
         public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false)
         {
             if (relocsOnly)
@@ -216,7 +247,25 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false)
 
             Dictionary<byte[], BlobVertex> uniqueInstrumentationData = new Dictionary<byte[], BlobVertex>(ByteArrayComparer.Instance);
 
-            foreach (MethodDesc method in _instrumentationDataMethods)
+            HashSet<MethodDesc> methodsToInsert = new();
+            foreach (EcmaModule inputModule in _factory.CompilationModuleGroup.CompilationModuleSet)
+            {
+                foreach (MethodDesc method in _profileDataManager.GetInputProfileDataMethodsForModule(inputModule))
+                {
+                    PgoSchemaElem[] schema = _profileDataManager[method].SchemaData;
+                    if (schema != null)
+                    {
+                        methodsToInsert.Add(method);
+                    }
+                }
+            }
+
+            methodsToInsert.UnionWith(_methodsWithSynthesizedPgoData);
+
+            MethodDesc[] methods = methodsToInsert.ToArray();
+            methods.MergeSort(TypeSystemComparer.Instance.Compare);
+
+            foreach (MethodDesc method in methods)
             {
                 pgoEmitter.Clear();
                 PgoProcessor.EncodePgoData(CorInfoImpl.ConvertTypeHandleHistogramsToCompactTypeHistogramFormat(_profileDataManager[method].SchemaData, factory.CompilationModuleGroup), pgoEmitter, false);

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodWithGCInfo.cs

@@ -6,6 +6,7 @@
 using System.Diagnostics;
 using System.Linq;
 using Internal.JitInterface;
+using Internal.Pgo;
 using Internal.Text;
 using Internal.TypeSystem;
 using Internal.TypeSystem.Ecma;
@@ -29,6 +30,7 @@ public class MethodWithGCInfo : ObjectNode, IMethodBodyNode, ISymbolDefinitionNo
         private List<ISymbolNode> _fixups;
         private MethodDesc[] _inlinedMethods;
         private bool _lateTriggeredCompilation;
+        private DependencyList _nonRelocationDependencies;
 
         public MethodWithGCInfo(MethodDesc methodDesc)
         {
@@ -272,6 +274,11 @@ protected override DependencyList ComputeNonRelocationBasedDependencies(NodeFact
                 dependencyList.Add(node, "classMustBeLoadedBeforeCodeIsRun");
             }
 
+            if (_nonRelocationDependencies != null)
+            {
+                dependencyList.AddRange(_nonRelocationDependencies);
+            }
+
             return dependencyList;
         }
 
@@ -382,6 +389,11 @@ public void InitializeInliningInfo(MethodDesc[] inlinedMethods, NodeFactory fact
                 RegisterInlineeModuleIndices(factory);
         }
 
+        public void InitializeNonRelocationDependencies(DependencyList dependencies)
+        {
+            _nonRelocationDependencies = dependencies;
+        }
+
         public int Offset => 0;
         public override bool IsShareable => throw new NotImplementedException();
         public override bool ShouldSkipEmittingObjectNode(NodeFactory factory) => IsEmpty;

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs

@@ -779,22 +779,26 @@ public void AttachToDependencyGraph(DependencyAnalyzerBase<NodeFactory> graph, I
             if ((ProfileDataManager != null) && (ProfileDataManager.EmbedPgoDataInR2RImage))
             {
                 // Profile instrumentation data attaches here
-                HashSet<MethodDesc> methodsToInsertInstrumentationDataFor = new HashSet<MethodDesc>();
-                foreach (EcmaModule inputModule in CompilationModuleGroup.CompilationModuleSet)
+
+                bool HasAnyProfileDataForInput()
                 {
-                    foreach (MethodDesc method in ProfileDataManager.GetMethodsForModuleDesc(inputModule))
+                    foreach (EcmaModule inputModule in CompilationModuleGroup.CompilationModuleSet)
                     {
-                        if (ProfileDataManager[method].SchemaData != null)
+                        foreach (MethodDesc method in ProfileDataManager.GetInputProfileDataMethodsForModule(inputModule))
                         {
-                            methodsToInsertInstrumentationDataFor.Add(method);
+                            if (ProfileDataManager[method].SchemaData != null)
+                            {
+                                return true;
+                            }
                         }
                     }
+
+                    return false;
                 }
-                if (methodsToInsertInstrumentationDataFor.Count != 0)
+
+                if (ProfileDataManager.SynthesizeRandomPgoData || HasAnyProfileDataForInput())
                 {
-                    MethodDesc[] methodsToInsert = methodsToInsertInstrumentationDataFor.ToArray();
-                    methodsToInsert.MergeSort(TypeSystemComparer.Instance.Compare);
-                    InstrumentationDataTable = new InstrumentationDataTableNode(this, methodsToInsert, ProfileDataManager);
+                    InstrumentationDataTable = new InstrumentationDataTableNode(this, ProfileDataManager);
                     Header.Add(Internal.Runtime.ReadyToRunSectionType.PgoInstrumentationData, InstrumentationDataTable, InstrumentationDataTable);
                 }
             }

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ProfileData.cs

@@ -62,11 +62,8 @@ public abstract class ProfileData
         public abstract IEnumerable<MethodProfileData> GetAllMethodProfileData();
         public abstract byte[] GetMethodBlockCount(MethodDesc m);
 
-        public static void MergeProfileData(ref bool partialNgen, Dictionary<MethodDesc, MethodProfileData> mergedProfileData, ProfileData profileData)
+        public static void MergeProfileData(Dictionary<MethodDesc, MethodProfileData> mergedProfileData, ProfileData profileData)
         {
-            if (profileData.PartialNGen)
-                partialNgen = true;
-
             PgoSchemaElem[][] schemaElemMergerArray = new PgoSchemaElem[2][];
 
             foreach (MethodProfileData data in profileData.GetAllMethodProfileData())

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs

@@ -61,9 +61,6 @@ public sealed class ReadyToRunCodegenCompilationBuilder : CompilationBuilder
             _ilProvider = new ReadyToRunILProvider(group);
             _inputFiles = inputFiles;
             _compositeRootPath = compositeRootPath;
-
-            // R2R field layout needs compilation group information
-            ((ReadyToRunCompilerContext)context).SetCompilationGroup(group);
         }
 
         public override CompilationBuilder UseBackendOptions(IEnumerable<string> options)

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunProfilingRootProvider.cs

@@ -23,7 +23,7 @@ public class ReadyToRunProfilingRootProvider : ICompilationRootProvider
         public ReadyToRunProfilingRootProvider(EcmaModule module, ProfileDataManager profileDataManager)
         {
             _module = module;
-            _profileData = profileDataManager.GetMethodsForModuleDesc(module);
+            _profileData = profileDataManager.GetInputProfileDataMethodsForModule(module);
         }
 
         public void AddCompilationRoots(IRootingServiceProvider rootProvider)

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunSingleAssemblyCompilationModuleGroup.cs

@@ -29,7 +29,7 @@ public sealed override bool ContainsMethodBody(MethodDesc method, bool unboxingS
 
             if (_profileGuidedCompileRestriction != null)
             {
-                if (!_profileGuidedCompileRestriction.IsMethodInProfileData(method))
+                if (!_profileGuidedCompileRestriction.IsMethodInInputProfileData(method))
                     return false;
             }
 

src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs

@@ -452,6 +452,7 @@ unsafe partial class CorInfoImpl
         private List<ISymbolNode> _precodeFixups;
         private List<EcmaMethod> _ilBodiesNeeded;
         private Dictionary<TypeDesc, bool> _preInitedTypes = new Dictionary<TypeDesc, bool>();
+        private HashSet<MethodDesc> _synthesizedPgoDependencies;
 
         public bool HasColdCode { get; private set; }
 

src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/ProfileDataManager.cs

@@ -26,12 +26,10 @@ public class ProfileDataManager
                 }
             }
 
-            bool dummy = false;
-
             // Merge all data together
             foreach (ProfileData profileData in _inputData)
             {
-                ProfileData.MergeProfileData(ref dummy, _mergedProfileData, profileData);
+                ProfileData.MergeProfileData(_mergedProfileData, profileData);
             }
         }
 

src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs

@@ -14,8 +14,6 @@
 using ILCompiler;
 using ILCompiler.DependencyAnalysis;
 
-using DependencyList = ILCompiler.DependencyAnalysisFramework.DependencyNodeCore<ILCompiler.DependencyAnalysis.NodeFactory>.DependencyList;
-
 #if SUPPORT_JIT
 using MethodCodeNode = Internal.Runtime.JitSupport.JitMethodCodeNode;
 using RyuJitCompilation = ILCompiler.Compilation;
@@ -39,7 +37,6 @@ internal unsafe partial class CorInfoImpl
         private DebugVarInfo[] _debugVarInfos;
         private readonly UnboxingMethodDescFactory _unboxingThunkFactory = new UnboxingMethodDescFactory();
         private bool _isFallbackBodyCompilation;
-        private DependencyList _additionalDependencies;
 
         public CorInfoImpl(RyuJitCompilation compilation)
             : this()

src/coreclr/tools/aot/crossgen2/Crossgen2RootCommand.cs

@@ -178,6 +178,8 @@ internal class Crossgen2RootCommand : RootCommand
             new(new[] { "--make-repro-path" }, "Path where to place a repro package");
         public Option<bool> HotColdSplitting { get; } =
             new(new[] { "--hot-cold-splitting" }, SR.HotColdSplittingOption);
+        public Option<bool> SynthesizeRandomMibc { get; } =
+            new(new[] { "--synthesize-random-mibc" });
 
         public bool CompositeOrInputBubble { get; private set; }
         public OptimizationMode OptimizationMode { get; private set; }
@@ -243,6 +245,7 @@ public Crossgen2RootCommand(string[] args) : base(SR.Crossgen2BannerText)
             AddOption(CallChainProfileFile);
             AddOption(MakeReproPath);
             AddOption(HotColdSplitting);
+            AddOption(SynthesizeRandomMibc);
 
             this.SetHandler(context =>
             {

src/coreclr/tools/aot/crossgen2/Program.cs

@@ -244,7 +244,7 @@ public int Run()
 
             string systemModuleName = Get(_command.SystemModuleName) ?? Helpers.DefaultSystemModule;
             _typeSystemContext.SetSystemModule((EcmaModule)_typeSystemContext.GetModuleForSimpleName(systemModuleName));
-            CompilerTypeSystemContext typeSystemContext = _typeSystemContext;
+            ReadyToRunCompilerContext typeSystemContext = _typeSystemContext;
 
             if (_singleFileCompilation)
             {
@@ -292,7 +292,7 @@ public int Run()
             return 0;
         }
 
-        private void RunSingleCompilation(Dictionary<string, string> inFilePaths, InstructionSetSupport instructionSetSupport, string compositeRootPath, Dictionary<string, string> unrootedInputFilePaths, HashSet<ModuleDesc> versionBubbleModulesHash, CompilerTypeSystemContext typeSystemContext)
+        private void RunSingleCompilation(Dictionary<string, string> inFilePaths, InstructionSetSupport instructionSetSupport, string compositeRootPath, Dictionary<string, string> unrootedInputFilePaths, HashSet<ModuleDesc> versionBubbleModulesHash, ReadyToRunCompilerContext typeSystemContext)
         {
             //
             // Initialize output filename
@@ -470,6 +470,9 @@ private void RunSingleCompilation(Dictionary<string, string> inFilePaths, Instru
                         compilationGroup = new ReadyToRunSingleAssemblyCompilationModuleGroup(groupConfig);
                     }
 
+                    // R2R field layout needs compilation group information
+                    typeSystemContext.SetCompilationGroup(compilationGroup);
+
                     // Load any profiles generated by method call chain analyis
                     CallChainProfile jsonProfile = null;
                     string callChainProfileFile = Get(_command.CallChainProfileFile);
@@ -503,7 +506,8 @@ private void RunSingleCompilation(Dictionary<string, string> inFilePaths, Instru
                         compilationGroup,
                         Get(_command.EmbedPgoData),
                         Get(_command.SupportIbc),
-                        crossModuleInlineableCode.Count == 0 ? compilationGroup.VersionsWithMethodBody : compilationGroup.CrossModuleInlineable);
+                        crossModuleInlineableCode.Count == 0 ? compilationGroup.VersionsWithMethodBody : compilationGroup.CrossModuleInlineable,
+                        Get(_command.SynthesizeRandomMibc));
 
                     bool partial = Get(_command.Partial);
                     compilationGroup.ApplyProfileGuidedOptimizationData(profileDataManager, partial);

src/coreclr/tools/dotnet-pgo/Program.cs

@@ -428,13 +428,12 @@ private int InnerMergeMain()
             {
                 var tsc = new TypeRefTypeSystem.TypeRefTypeSystemContext(mibcReaders);
 
-                bool partialNgen = false;
                 Dictionary<MethodDesc, MethodProfileData> mergedProfileData = new Dictionary<MethodDesc, MethodProfileData>();
                 for (int i = 0; i < mibcReaders.Length; i++)
                 {
                     var peReader = mibcReaders[i];
                     PrintDetailedMessage($"Merging {paths[i]}");
-                    ProfileData.MergeProfileData(ref partialNgen, mergedProfileData, MIbcProfileParser.ParseMIbcFile(tsc, peReader, assemblyNamesInBubble, onlyDefinedInAssembly: null));
+                    ProfileData.MergeProfileData(mergedProfileData, MIbcProfileParser.ParseMIbcFile(tsc, peReader, assemblyNamesInBubble, onlyDefinedInAssembly: null));
                 }
 
                 MibcConfig mergedConfig = ParseMibcConfigsAndMerge(tsc, mibcReaders);

src/tests/Common/CLRTest.CrossGen.targets

@@ -105,6 +105,11 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then
         echo -r:$CORE_ROOT/Microsoft.*.dll>>$__ResponseFile
         echo -r:$CORE_ROOT/mscorlib.dll>>$__ResponseFile
         echo --verify-type-and-field-layout>>$__ResponseFile
+        echo --method-layout:random>>$__ResponseFile
+        if [ ! -z ${CrossGen2SynthesizePgo+x} ]%3B then
+            echo --synthesize-random-mibc>>$__ResponseFile
+            echo --embed-pgo-data>>$__ResponseFile
+        fi
         echo --targetarch:$(TargetArchitecture)>>$__ResponseFile
         echo -O>>$__ResponseFile
 
@@ -248,6 +253,10 @@ if defined RunCrossGen2 (
     echo --targetarch:$(TargetArchitecture)>>!__ResponseFile!
     echo --verify-type-and-field-layout>>!__ResponseFile!
     echo --method-layout:random>>!__ResponseFile!
+    if defined CrossGen2SynthesizePgo (
+        echo --synthesize-random-mibc>>!__ResponseFile!
+        echo --embed-pgo-data>>!__ResponseFile!
+    )
     echo -r:!CORE_ROOT!\System.*.dll>>!__ResponseFile!
     echo -r:!CORE_ROOT!\Microsoft.*.dll>>!__ResponseFile!
     echo -r:!CORE_ROOT!\mscorlib.dll>>!__ResponseFile!

src/tests/Common/CLRTest.Execute.Bash.targets

@@ -115,6 +115,14 @@ then
   exit $(IncompatibleTestBashScriptExitCode)
 fi
       ]]></BashCLRTestEnvironmentCompatibilityCheck>
+      <BashCLRTestEnvironmentCompatibilityCheck Condition="'$(SynthesizedPgoIncompatible)' == 'true'"><![CDATA[
+$(BashCLRTestEnvironmentCompatibilityCheck)
+if [ ! -z "$CrossGen2SynthesizePgo" ]
+then
+  echo SKIPPING EXECUTION BECAUSE CrossGen2SynthesizePgo IS SET
+  exit $(IncompatibleTestBashScriptExitCode)
+fi
+      ]]></BashCLRTestEnvironmentCompatibilityCheck>
 
       <BashCLRTestExitCodePrep Condition="$(_CLRTestNeedsToRun)">
 <![CDATA[

src/tests/Common/CLRTest.Execute.Batch.targets

@@ -125,6 +125,15 @@ IF NOT "%RunningIlasmRoundTrip%"=="" (
 )
       ]]></BatchCLRTestEnvironmentCompatibilityCheck>
 
+      <BatchCLRTestEnvironmentCompatibilityCheck Condition="'$(SynthesizedPgoIncompatible)' == 'true'"><![CDATA[
+$(BatchCLRTestEnvironmentCompatibilityCheck)
+IF NOT "%CrossGen2SynthesizePgo"=="" (
+  ECHO SKIPPING EXECUTION BECAUSE CrossGen2SynthesizePgo IS SET
+  popd
+  Exit /b 0
+)
+      ]]></BatchCLRTestEnvironmentCompatibilityCheck>
+
       <BatchCLRTestExitCodePrep Condition="$(_CLRTestNeedsToRun)">
         <![CDATA[
 if not defined CLRTestExpectedExitCode (set CLRTestExpectedExitCode=$(CLRTestExitCode))

src/tests/Loader/classloader/MethodImpl/CovariantReturns/UnitTest/CompatibleWithTest.ilproj

 <Project Sdk="Microsoft.NET.Sdk.IL">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
+    <!-- Synthesized PGO may cause the expected TypeLoadExceptions in this test to appear at unexpected places -->
+    <SynthesizedPgoIncompatible>true</SynthesizedPgoIncompatible>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="CompatibleWithTest.il" />

src/tests/run.cmd

@@ -63,6 +63,7 @@ if /i "%1" == "printlastresultsonly"                    (set __PrintLastResultsO
 if /i "%1" == "runcrossgen2tests"                       (set RunCrossGen2=1&shift&goto Arg_Loop)
 REM This test feature is currently intentionally undocumented
 if /i "%1" == "runlargeversionbubblecrossgen2tests"     (set RunCrossGen2=1&set CrossgenLargeVersionBubble=1&shift&goto Arg_Loop)
+if /i "%1" == "synthesizepgo"                           (set CrossGen2SynthesizePgo=1&shift&goto Arg_Loop)
 if /i "%1" == "link"                                    (set DoLink=true&set ILLINK=%2&shift&shift&goto Arg_Loop)
 if /i "%1" == "gcname"                                  (set DOTNET_GCName=%2&shift&shift&goto Arg_Loop)
 if /i "%1" == "timeout"                                 (set __TestTimeout=%2&shift&shift&goto Arg_Loop)
@@ -140,6 +141,10 @@ if defined CrossgenLargeVersionBubble (
     set __RuntestPyArgs=%__RuntestPyArgs% --large_version_bubble
 )
 
+if defined CrossGen2SynthesizePgo {
+    set __RuntestPyArgs=%__RuntestPyArgs% --synthesize_pgo
+}
+
 if defined __PrintLastResultsOnly (
     set __RuntestPyArgs=%__RuntestPyArgs% --analyze_results_only
 )
@@ -390,6 +395,7 @@ echo ^<build_type^>              - Specifies build type: Debug, Release, or Chec
 echo TestEnv ^<test_env_script^> - Run a custom script before every test to set custom test environment settings.
 echo sequential                - Run tests sequentially (no parallelism).
 echo RunCrossgen2Tests         - Runs ReadytoRun tests compiled with Crossgen2
+echo synthesizepgo             - Enabled synthesizing PGO data in CrossGen2
 echo jitstress ^<n^>             - Runs the tests with DOTNET_JitStress=n
 echo jitstressregs ^<n^>         - Runs the tests with DOTNET_JitStressRegs=n
 echo jitminopts                - Runs the tests with DOTNET_JITMinOpts=1

src/tests/run.py

@@ -93,6 +93,7 @@ parser.add_argument("--gcsimulator", dest="gcsimulator", action="store_true", de
 parser.add_argument("--ilasmroundtrip", dest="ilasmroundtrip", action="store_true", default=False)
 parser.add_argument("--run_crossgen2_tests", dest="run_crossgen2_tests", action="store_true", default=False)
 parser.add_argument("--large_version_bubble", dest="large_version_bubble", action="store_true", default=False)
+parser.add_argument("--synthesize_pgo", dest="synthesize_pgo", action="store_true", default=False)
 parser.add_argument("--skip_test_run", dest="skip_test_run", action="store_true", default=False, help="Does not run tests.")
 parser.add_argument("--sequential", dest="sequential", action="store_true", default=False)
 
@@ -874,13 +875,17 @@ def run_tests(args,
         print("Large Version Bubble enabled")
         os.environ["LargeVersionBubble"] = "1"
 
+    if args.synthesize_pgo:
+        print("Synthesizing PGO")
+        os.environ["CrossGen2SynthesizePgo"] = "1"
+
     if args.limited_core_dumps:
         setup_coredump_generation(args.host_os)
 
     if args.run_in_context:
         print("Running test in an unloadable AssemblyLoadContext")
         os.environ["CLRCustomTestLauncher"] = args.runincontext_script_path
-        os.environ["RunInUnloadableContext"] = "1";
+        os.environ["RunInUnloadableContext"] = "1"
         per_test_timeout = 40*60*1000
 
     if args.tiering_test:
@@ -996,6 +1001,11 @@ def setup_args(args):
                               lambda unused: True,
                               "Error setting run_crossgen2_tests")
 
+    coreclr_setup_args.verify(args,
+                              "synthesize_pgo",
+                              lambda arg: True,
+                              "Error setting synthesize_pgo")
+
     coreclr_setup_args.verify(args,
                               "skip_test_run",
                               lambda arg: True,

src/tests/run.sh

@@ -18,6 +18,7 @@ function print_usage {
     echo '  --enableEventLogging             : Enable event logging through LTTNG.'
     echo '  --sequential                     : Run tests sequentially (default is to run in parallel).'
     echo '  --runcrossgen2tests              : Runs the ReadyToRun tests compiled with Crossgen2'
+    echo '  --synthesizepgo                  : Runs the tests allowing crossgen2 to synthesize PGO data'
     echo '  --jitstress=<n>                  : Runs the tests with DOTNET_JitStress=n'
     echo '  --jitstressregs=<n>              : Runs the tests with DOTNET_JitStressRegs=n'
     echo '  --jitminopts                     : Runs the tests with DOTNET_JITMinOpts=1'
@@ -140,6 +141,9 @@ do
         --runcrossgen2tests)
             export RunCrossGen2=1
             ;;
+        --synthesizepgo)
+            export CrossGen2SynthesizePgo=1
+            ;;
         --sequential)
             runSequential=1
             ;;
@@ -255,6 +259,10 @@ if [[ -n "$RunCrossGen2" ]]; then
     runtestPyArguments+=("--run_crossgen2_tests")
 fi
 
+if [[ -n "$CrossGen2SynthesizePgo" ]]; then
+    runtestPyArguments+=("--synthesize_pgo")
+fi
+
 if [[ "$limitedCoreDumps" == "ON" ]]; then
     runtestPyArguments+=("--limited_core_dumps")
 fi