Add support for NativeAOT testing (#62704)

Adds support for running runtime tests in NativeAOT configuration. It's similar to crossgen2 testing, but there's small differences.

* A layout for the AOT compiler is generated into `CORE_ROOT`. This includes the AOT compiler, NativeAOT corelib, and framework/test dependencies we otherwise dump into `CORE_ROOT`.
* The test execution script is amended to first AOT compile the test using the AOT compiler in `CORE_ROOT` and then run the AOT compiled test.
* The test is compiled by running MSBuild on a generated project file. This is different from crossgen2 testing that directly invokes crossgen2. The extra project file is annoying, but also the NativeAOT compiler has more command line arguments and we also need to invoke the platform linker with another set of command line arguments. Having MSBuild do that for us (using the shipping .target/.props that ship with NativeAOT) saves a some trouble there.
* This also includes support for multimodule testing (where each managed .dll is compiled into a single .o/.obj that we link with the native linker). This needs an extra step during test build to precompile the entire framework into a .a/.lib file.

src/tests/Common/CLRTest.Execute.Bash.targets

@@ -321,6 +321,10 @@ then
 else
     LAUNCHER="$_DebuggerFullPath $_DebuggerArgsSeparator $(CLRTestRunFile)"
 fi
+if [ ! -z "$RunNativeAot" ]
+then
+    LAUNCHER="$_DebuggerFullPath"
+fi
 
 $(BashIlrtTestLaunchCmds)
 
@@ -336,6 +340,13 @@ CLRTestExitCode=$?
 if [ ! -z ${RunCrossGen+x} ]%3B then
   ReleaseLock
 fi
+if [ ! -z "$RunNativeAot" ]
+then
+    if [ -z "$CLRTestNoCleanup" ];
+    then
+      rm -rf native
+    fi
+fi
 
 $(BashLinkerTestCleanupCmds)
       ]]></BashCLRTestLaunchCmds>

src/tests/Common/CLRTest.Execute.Batch.targets

@@ -306,6 +306,9 @@ IF NOT "%CLRCustomTestLauncher%"=="" (
 ) ELSE (
   set LAUNCHER=%_DebuggerFullPath% $(CLRTestRunFile)
 )
+IF NOT "%RunNativeAot%"=="" (
+  set LAUNCHER=%_DebuggerFullPath%
+)
 
 $(BatchIlrtTestLaunchCmds)
 
@@ -318,6 +321,11 @@ set CLRTestExitCode=!ERRORLEVEL!
 if defined RunCrossGen (
   call :ReleaseLock
 )
+if defined RunNativeAot (
+  if not defined CLRTestNoCleanup (
+    IF EXIST native rmdir /s /q native
+  )
+)
 $(BatchLinkerTestCleanupCmds)
       ]]></BatchCLRTestLaunchCmds>
       <BatchCLRTestLaunchCmds Condition="'$(CLRTestKind)' == 'BuildAndRun' And $(TargetOS) == 'Android' ">

src/tests/Common/CLRTest.Execute.targets

@@ -78,6 +78,7 @@ This file contains the logic for providing Execution Script generation.
 
   <Import Project="CLRTest.Jit.targets" />
   <Import Project="CLRTest.CrossGen.targets" />
+  <Import Project="CLRTest.NativeAot.targets" />
   <Import Project="CLRTest.GC.targets" />
   <Import Project="CLRTest.Execute.*.targets" />
   <Import Project="CLRTest.MockHosting.targets" Condition="'$(RequiresMockHostPolicy)' == 'true'" />

src/tests/Common/CLRTest.NativeAot.targets

+<!--
+***********************************************************************************************
+CLRTest.Execute.targets
+
+WARNING:  DO NOT MODIFY this file unless you are knowledgeable about MSBuild and have
+          created a backup copy.  Incorrect changes to this file will make it
+          impossible to load or build your projects from the command-line or the IDE.
+
+This file contains the logic for providing Execution Script generation.
+
+WARNING:   When setting properties based on their current state (for example:
+           <Foo Condition="'$(Foo)'==''>Bar</Foo>).  Be very careful.  Another script generation
+           target might be trying to do the same thing.  It's better to avoid this by instead setting a new property.
+
+           Additionally, be careful with itemgroups.  Include will propagate outside of the target too!
+
+***********************************************************************************************
+-->
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <BashScriptSnippetGen>$(BashScriptSnippetGen);GetNativeAotBashScript</BashScriptSnippetGen>
+    <BatchScriptSnippetGen>$(BatchScriptSnippetGen);GetNativeAotBatchScript</BatchScriptSnippetGen>
+  </PropertyGroup>
+
+  <!--
+    This returns the portion of the execution script that generates the required lines to run the test with native AOT.
+
+    Tests can opt out of ahead-of-time (AOT) compilation by setting this property in their project:
+
+      <NativeAotTest>false</NativeAotTest>
+  -->
+  <Target Name="GetNativeAotBashScript">
+
+    <PropertyGroup>
+      <NativeAotCleanupBashScript Condition="'$(CLRTestKind)' == 'BuildAndRun' and '$(NativeAotTest)' != 'false'">
+        <![CDATA[
+if [ ! -z ${RunNativeAot+x} ]%3B then
+    rm -rf native
+fi
+        ]]>
+    </NativeAotCleanupBashScript>
+
+      <NativeAotBashScript Condition="'$(CLRTestKind)' == 'BuildAndRun' and '$(NativeAotTest)' != 'false'">
+        <![CDATA[
+if [ ! -z ${RunNativeAot+x} ]%3B then
+    __Command=$_DebuggerFullPath
+    # Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path
+    if [ ! -z ${__TestDotNetCmd+x} ] %3B then
+        __Command+=" $__TestDotNetCmd"
+    else
+        __Command+=" dotnet"
+    fi
+
+    __Command+=" msbuild /p:IlcPath=$CORE_ROOT/nativeaot nativebuild.proj"
+    echo "Running Native AOT: $__Command"
+    $__Command
+    __naotExitCode=$?
+
+    if [ $__naotExitCode -ne 0 ]; then
+      echo Native AOT failed with exitcode: $__naotExitCode
+      exit 1
+    fi
+
+    # Copy the native component to the directory where the executable resides.
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        cp *.dylib  native/ 2>/dev/null
+    else
+        cp *.so  native/ 2>/dev/null
+    fi
+
+    ExePath=native/$(MSBuildProjectName)
+fi
+        ]]>
+      </NativeAotBashScript>
+
+      <BashCLRTestPreCommands>$(NativeAotCleanupBashScript);$(BashCLRTestPreCommands);$(NativeAotBashScript)</BashCLRTestPreCommands>
+    </PropertyGroup>
+  </Target>
+
+  <Target Name="GetNativeAotBatchScript">
+    <PropertyGroup>
+      <NativeAotMultimoduleIncompatibleScript Condition="'$(NativeAotMultimoduleIncompatible)' == 'true'">
+        <![CDATA[
+  ECHO SKIPPING EXECUTION BECAUSE TEST IS NOT COMPATIBLE WITH MULTIMODULE COMPILATION
+  popd
+  Exit /b 0
+        ]]>
+      </NativeAotMultimoduleIncompatibleScript>
+
+      <NativeAotCleanupBatchScript Condition="'$(CLRTestKind)' == 'BuildAndRun' and '$(NativeAotTest)' != 'false'">
+        <![CDATA[
+if defined RunNativeAot (
+    rem Delete any leftover native compilation bits
+    IF EXIST native rmdir /s /q native
+)
+        ]]>
+    </NativeAotCleanupBatchScript>
+
+    <NativeAotBatchScript Condition="'$(CLRTestKind)' == 'BuildAndRun' and '$(NativeAotTest)' != 'false'">
+        <![CDATA[
+if defined RunNativeAot (
+
+    set ExePath=native\$(MSBuildProjectName)
+
+    set __Command=!_DebuggerFullPath!
+    REM Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path
+    if defined __TestDotNetCmd (
+        set __Command=!__Command! "!__TestDotNetCmd!"
+    ) else (
+        set __Command=!__Command! "dotnet"
+    )
+
+    set __Command=!__Command! msbuild /p:IlcPath=!CORE_ROOT!\nativeaot nativebuild.proj
+
+    if defined NativeAotMultimodule (
+        set __Command=!__Command! /p:IlcMultiModule=true /p:DisableFrameworkLibGeneration=true /p:FrameworkLibPath=!CORE_ROOT!\nativeaot\sdk
+        $(NativeAotMultimoduleIncompatibleScript)
+    )
+
+    echo "!__Command!"
+    call !__Command!
+
+    if errorlevel 1 (
+        ECHO END COMPILATION - FAILED
+        ECHO FAILED
+        exit /b 1
+    )
+)
+        ]]>
+      </NativeAotBatchScript>
+
+      <CLRTestBatchPreCommands>$(NativeAotCleanupBatchScript);$(CLRTestBatchPreCommands);$(NativeAotBatchScript)</CLRTestBatchPreCommands>
+    </PropertyGroup>
+  </Target>
+
+  <Target Name="CreateNativeAotBuildProjectFile"
+    Condition="'$(_WillCLRTestProjectBuild)' == 'true'"
+    Inputs="$(MSBuildProjectFullPath)"
+    Outputs="$(OutputPath)\nativebuild.proj"
+    BeforeTargets="Build">
+    <PropertyGroup>
+      <_RootEntryAssemblyLine Condition="$(CLRTestFullTrimming) != 'true'">&lt;TrimmerDefaultAction&gt;copyused&lt;/TrimmerDefaultAction&gt;</_RootEntryAssemblyLine>
+
+      <_NativeAotBuildProjectFile>
+        <![CDATA[
+<Project DefaultTargets="LinkNative" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+
+  <PropertyGroup>
+    <TargetName>$(MSBuildProjectName)</TargetName>
+    <TargetExt>.dll</TargetExt>
+    <OutputType>Exe</OutputType>
+    <OutputPath>%24(MSBuildProjectDirectory)\</OutputPath>
+    <IntermediateOutputPath>%24(MSBuildProjectDirectory)\</IntermediateOutputPath>
+    <TargetArchitecture>$(TargetArchitecture)</TargetArchitecture>
+    <Optimize>$(Optimize)</Optimize>
+    <DebugSymbols>true</DebugSymbols>
+    <NETCoreSdkVersion>$(NETCoreSdkVersion)</NETCoreSdkVersion>
+    $(_RootEntryAssemblyLine)
+  </PropertyGroup>
+
+  $(NativeAotProjectLines)
+
+  <ItemGroup>
+    <!-- Some tests consist of multiple assemblies - make sure ILC sees them -->
+    <IlcCompileInput Include="%24(MSBuildProjectDirectory)\*.dll" Exclude="%24(MSBuildProjectDirectory)\$(MSBuildProjectName).dll" />
+
+    <!-- Copy RuntimeHostConfigurationOptions -->
+@(RuntimeHostConfigurationOption->'    <RuntimeHostConfigurationOption Include="%(Identity)" Value="%(Value)" />', '%0A')
+  </ItemGroup>
+
+  <!-- We don't do anything in the Compile step since the test is already compiled to IL - the AOT compiler hooks up after this -->
+  <Target Name="Compile" />
+
+  <Target Name="PrepareForILLink" />
+
+  <Import Project="%24(IlcPath)\build\Microsoft.NETCore.Native.targets" />
+
+</Project>
+      ]]>
+      </_NativeAotBuildProjectFile>
+    </PropertyGroup>
+
+    <!-- Write the file -->
+    <WriteLinesToFile
+      File="$(OutputPath)\nativebuild.proj"
+      Lines="$(_NativeAotBuildProjectFile)"
+      Overwrite="true" />
+  </Target>
+
+</Project>

src/tests/Common/Directory.Build.targets

@@ -31,7 +31,7 @@
           DependsOnTargets="ResolveAssemblyReferences;ResolveRuntimeFilesFromLocalBuild">
 
     <ItemGroup>
-      <RunTimeDependencyExclude Include="$(CORE_ROOT)\**\*.*"  />
+      <RunTimeDependencyExclude Include="$(CORE_ROOT)\**\*.*" Exclude="$(CORE_ROOT)\nativeaot\**\*.*" />
       <RunTimeDependencyExcludeFiles Include="@(RunTimeDependencyExclude -> '%(FileName)%(Extension)')" />
       <RunTimeDependencyExcludeFiles Include="@(RunTimeDependencyExclude -> '%(FileName).ni%(Extension)')" />
       <RunTimeDependencyExcludeFiles Include="@(RunTimeDependencyExclude -> '%(FileName).pdb')" />
@@ -171,6 +171,19 @@
       <RunTimeDependencyCopyLocal Include="$(CoreDisToolsLibrary)" Condition="$(CopyCoreDisToolsToCoreRoot)" />
     </ItemGroup>
 
+    <ItemGroup Condition="'$(TestBuildMode)' == 'nativeaot'">
+      <AotCompilerCopyLocal Include="$(CoreCLRArtifactsPath)ilc/**/*" TargetDir="nativeaot/tools/" />
+      <AotCompilerCopyLocal Include="$(CoreCLRArtifactsPath)aotsdk/**/*" TargetDir="nativeaot/sdk/" />
+      <AotCompilerCopyLocal Include="$(CoreCLRArtifactsPath)build/**/*" TargetDir="nativeaot/build/" />
+      <AotCompilerCopyLocal Include="$(TargetingPackPath)/*" TargetDir="nativeaot/framework/" />
+
+      <!-- Works around https://github.com/dotnet/runtime/issues/62372 -->
+      <_FixedRuntimeCopyLocalItems Include="@(RuntimeCopyLocalItems)" />
+      <_FixedRuntimeCopyLocalItems Remove="@(_FixedRuntimeCopyLocalItems)" Condition="'%(Filename)' == 'Microsoft.Interop.DllImportGenerator'" />
+
+      <AotCompilerCopyLocal Include="@(_FixedRuntimeCopyLocalItems)" TargetDir="nativeaot/framework/" />
+    </ItemGroup>
+
     <ItemGroup Condition="'$(RuntimeFlavor)' == 'mono' and '$(IsDesktopOS)' == 'true' " >
       <RuntimeDependencyCopyLocal Include="$(MonoArtifactsPath)/libcoreclr$(LibSuffix)" TargetDir=""  />
       <RuntimeDependencyCopyLocal Include="$(MonoArtifactsPath)/libmono-component-*" TargetDir=""  />
@@ -188,6 +201,17 @@
       <Output TaskParameter="DestinationFiles" ItemName="FileWrites" />
     </Copy>
 
+    <Copy
+      SourceFiles="@(AotCompilerCopyLocal)"
+      DestinationFiles="@(AotCompilerCopyLocal -> '$(CORE_ROOT)/%(TargetDir)%(RecursiveDir)%(Filename)%(Extension)')"
+      SkipUnchangedFiles="$(SkipCopyUnchangedFiles)"
+      OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
+      Retries="$(CopyRetryCount)"
+      RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
+      UseHardlinksIfPossible="$(CreateHardLinksForCopyFilesToOutputDirectoryIfPossible)">
+      <Output TaskParameter="DestinationFiles" ItemName="FileWrites" />
+    </Copy>
+
   </Target>
 
 </Project>

src/tests/build.cmd

@@ -117,6 +117,7 @@ if /i "%1" == "buildagainstpackages"  (echo error: Remove /BuildAgainstPackages
 if /i "%1" == "crossgen2"             (set __TestBuildMode=crossgen2&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "composite"             (set __CompositeBuildMode=1&set __TestBuildMode=crossgen2&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "pdb"                   (set __CreatePdb=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "nativeaot"             (set __TestBuildMode=nativeaot&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "perfmap"               (set __CreatePerfmap=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "Exclude"               (set __Exclude=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
 if /i "%1" == "-priority"             (set __Priority=%2&shift&set processedArgs=!processedArgs! %1=%2&shift&goto Arg_Loop)

src/tests/build.proj

@@ -552,4 +552,15 @@
       DependsOnTargets="MonoAotCompileTests"
       AfterTargets="ManagedBuild"
       Condition="'$(__BuildTestWrappersOnly)' != '1' and '$(__GenerateLayoutOnly)' != '1' and '$(__CopyNativeTestBinaries)' != '1' and ($(MonoAot) or $(MonoFullAot))" />
+
+  <Target Name="NativeAotCompileFramework" AfterTargets="CopyDependencyToCoreRoot" Condition="'$(NativeAotMultimodule)' == 'true'">
+    <!-- MSBuild doesn't understand the Checked configuration -->
+    <PropertyGroup>
+      <_OptimizeFlagValue Condition="'$(Configuration)' == 'Checked' or '$(Configuration)' == 'Release'">true</_OptimizeFlagValue>
+    </PropertyGroup>
+    <MSBuild Projects="$(CORE_ROOT)/nativeaot/build/BuildFrameworkNativeObjects.proj"
+      Targets="CreateLib"
+      Properties="FrameworkLibPath=$(CORE_ROOT)/nativeaot/sdk;FrameworkObjPath=$(BaseIntermediateOutputPath)/NativeAOTFX;IlcPath=$(CORE_ROOT)/nativeaot;Optimize=$(_OptimizeFlagValue);NETCoreSdkVersion=6.0.0" />
+  </Target>
+
 </Project>

src/tests/build.sh

@@ -150,6 +150,7 @@ usage_list+=("-dir:xxx - build all tests in a given directory");
 usage_list+=("-tree:xxx - build all tests in a given subtree");
 
 usage_list+=("-crossgen2: Precompiles the framework managed assemblies in coreroot using the Crossgen2 compiler.")
+usage_list+=("-nativeaot: Builds the tests for Native AOT compilation.")
 usage_list+=("-priority1: include priority=1 tests in the build.")
 usage_list+=("-composite: Use Crossgen2 composite mode (all framework gets compiled into a single native R2R library).")
 usage_list+=("-perfmap: emit perfmap symbol files when compiling the framework assemblies using Crossgen2.")
@@ -194,6 +195,10 @@ handle_arguments_local() {
             __TestBuildMode=crossgen2
             ;;
 
+        nativeaot|-nativeaot)
+            __TestBuildMode=nativeaot
+            ;;
+
         perfmap|-perfmap)
             __CreatePerfmap=1
             ;;

src/tests/run.cmd

@@ -61,6 +61,8 @@ if /i "%1" == "ilasmroundtrip"                          (set __IlasmRoundTrip=1&
 
 if /i "%1" == "printlastresultsonly"                    (set __PrintLastResultsOnly=1&shift&goto Arg_Loop)
 if /i "%1" == "runcrossgen2tests"                       (set RunCrossGen2=true&shift&goto Arg_Loop)
+if /i "%1" == "runnativeaottests"                       (set RunNativeAot=true&shift&goto Arg_Loop)
+if /i "%1" == "nativeaotmultimodule"                    (set NativeAotMultimodule=true&shift&goto Arg_Loop)
 REM This test feature is currently intentionally undocumented
 if /i "%1" == "runlargeversionbubblecrossgen2tests"     (set RunCrossGen2=true&set CrossgenLargeVersionBubble=true&shift&goto Arg_Loop)
 if /i "%1" == "link"                                    (set DoLink=true&set ILLINK=%2&shift&shift&goto Arg_Loop)
@@ -139,6 +141,14 @@ if defined CrossgenLargeVersionBubble (
     set __RuntestPyArgs=%__RuntestPyArgs% --large_version_bubble
 )
 
+if defined RunNativeAot (
+    set __RuntestPyArgs=%__RuntestPyArgs% --run_nativeaot_tests
+)
+
+if defined NativeAotMultimodule (
+    set __RuntestPyArgs=%__RuntestPyArgs% --nativeaot_multimodule
+)
+
 if defined __PrintLastResultsOnly (
     set __RuntestPyArgs=%__RuntestPyArgs% --analyze_results_only
 )

src/tests/run.py

@@ -92,6 +92,8 @@ parser.add_argument("--long_gc", dest="long_gc", action="store_true", default=Fa
 parser.add_argument("--gcsimulator", dest="gcsimulator", action="store_true", default=False)
 parser.add_argument("--ilasmroundtrip", dest="ilasmroundtrip", action="store_true", default=False)
 parser.add_argument("--run_crossgen2_tests", dest="run_crossgen2_tests", action="store_true", default=False)
+parser.add_argument("--run_nativeaot_tests", dest="run_nativeaot_tests", action="store_true", default=False)
+parser.add_argument("--nativeaot_multimodule", dest="nativeaot_multimodule", action="store_true", default=False)
 parser.add_argument("--large_version_bubble", dest="large_version_bubble", action="store_true", default=False)
 parser.add_argument("--skip_test_run", dest="skip_test_run", action="store_true", default=False, help="Does not run tests.")
 parser.add_argument("--sequential", dest="sequential", action="store_true", default=False)
@@ -869,6 +871,16 @@ def run_tests(args,
         print("Setting RunCrossGen2=true")
         os.environ["RunCrossGen2"] = "true"
 
+    if args.run_nativeaot_tests:
+        print("Running tests Native AOT")
+        print("Setting RunNativeAot=true")
+        os.environ["RunNativeAot"] = "true"
+
+    if args.nativeaot_multimodule:
+        print("Native AOT will be compiled in multimodule mode")
+        print("Setting NativeAotMultimodule=true")
+        os.environ["NativeAotMultimodule"] = "true"
+
     if args.large_version_bubble:
         print("Large Version Bubble enabled")
         os.environ["LargeVersionBubble"] = "true"
@@ -991,6 +1003,16 @@ def setup_args(args):
                               lambda unused: True,
                               "Error setting run_crossgen2_tests")
 
+    coreclr_setup_args.verify(args,
+                              "run_nativeaot_tests",
+                              lambda unused: True,
+                              "Error setting run_nativeaot_tests")
+
+    coreclr_setup_args.verify(args,
+                              "nativeaot_multimodule",
+                              lambda unused: True,
+                              "Error setting nativeaot_multimodule")
+
     coreclr_setup_args.verify(args,
                               "skip_test_run",
                               lambda arg: True,

src/tests/run.sh

@@ -18,6 +18,7 @@ function print_usage {
     echo '  --disableEventLogging            : Disable the events logged by both VM and Managed Code'
     echo '  --sequential                     : Run tests sequentially (default is to run in parallel).'
     echo '  --runcrossgen2tests              : Runs the ReadyToRun tests compiled with Crossgen2'
+    echo '  --runnativeaottests              : Runs the ready to run tests compiled with Native AOT'
     echo '  --jitstress=<n>                  : Runs the tests with COMPlus_JitStress=n'
     echo '  --jitstressregs=<n>              : Runs the tests with COMPlus_JitStressRegs=n'
     echo '  --jitminopts                     : Runs the tests with COMPlus_JITMinOpts=1'
@@ -167,6 +168,9 @@ do
         --runcrossgen2tests)
             export RunCrossGen2=1
             ;;
+        --runnativeaottests)
+            export RunNativeAot=1
+            ;;
         --sequential)
             runSequential=1
             ;;
@@ -281,6 +285,10 @@ if [[ -n "$RunCrossGen2" ]]; then
     runtestPyArguments+=("--run_crossgen2_tests")
 fi
 
+if [[ -n "$RunNativeAot" ]]; then
+    runtestPyArguments+=("--run_nativeaot_tests")
+fi
+
 if [[ "$limitedCoreDumps" == "ON" ]]; then
     runtestPyArguments+=("--limited_core_dumps")
 fi