Adding Official Build Support for runtimelab (#43146)

Co-authored-by: NSantiago Fernandez Madero <safern@microsoft.com>
Co-authored-by: NJose Perez Rodriguez <joperezr@microsoft.com>

eng/pipelines/common/global-build-job.yml

@@ -94,6 +94,7 @@ jobs:
           runtimeFlavor: ${{ parameters.runtimeFlavor }}
           helixQueues: ${{ parameters.helixQueues }}
           targetRid: ${{ parameters.targetRid }}
+          nameSuffix: ${{ parameters.nameSuffix }}
           ${{ insert }}: ${{ parameters.extraStepsParameters }}
 
     - task: PublishBuildArtifacts@1

eng/pipelines/installer/jobs/base-job.yml

@@ -10,6 +10,8 @@ parameters:
   container: ''
   buildSteps: []
   dependsOn: []
+  dependsOnGlobalBuild: false
+  globalBuildSuffix: ''
   variables: []
   name: ''
   displayName: ''
@@ -332,26 +334,38 @@ jobs:
   dependsOn:
   - checkout
   - ${{ parameters.dependsOn }}
-  - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
-    - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}',
-        parameters.runtimeFlavor,
-        parameters.runtimeVariant,
-        parameters.osGroup,
-        parameters.osSubgroup,
-        parameters.archType,
-        parameters.liveRuntimeBuildConfig) }}
-  - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
-    - libraries_build_${{ format('{0}{1}_{2}_{3}',
-        parameters.osGroup,
-        parameters.osSubgroup,
-        parameters.archType,
-        parameters.liveLibrariesBuildConfig) }}
-  - ${{ if eq(parameters.useOfficialAllConfigurations, true) }}:
-    - libraries_build_allconfigurations_Windows_NT_x64_Release
-  - ${{ if eq(parameters.buildFullPlatformManifest, true) }}:
-    - ${{ each platform in parameters.platforms }}:
-      - ${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_product_build_${{ platform }}_${{ parameters.liveRuntimeBuildConfig }}
-      - libraries_build_${{ platform }}_${{ parameters.liveLibrariesBuildConfig }}
+  - ${{ if eq(parameters.dependsOnGlobalBuild, false) }}:
+    - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
+        - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}',
+            parameters.runtimeFlavor,
+            parameters.runtimeVariant,
+            parameters.osGroup,
+            parameters.osSubgroup,
+            parameters.archType,
+            parameters.liveRuntimeBuildConfig) }}
+    - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
+      - libraries_build_${{ format('{0}{1}_{2}_{3}',
+          parameters.osGroup,
+          parameters.osSubgroup,
+          parameters.archType,
+          parameters.liveLibrariesBuildConfig) }}
+    - ${{ if eq(parameters.useOfficialAllConfigurations, true) }}:
+      - libraries_build_allconfigurations_Windows_NT_x64_Release
+    - ${{ if eq(parameters.buildFullPlatformManifest, true) }}:
+      - ${{ each platform in parameters.platforms }}:
+        - ${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_product_build_${{ platform }}_${{ parameters.liveRuntimeBuildConfig }}
+        - libraries_build_${{ platform }}_${{ parameters.liveLibrariesBuildConfig }}
+
+  - ${{ if eq(parameters.dependsOnGlobalBuild, true) }}:
+    - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
+        - ${{ format('build_{0}{1}_{2}_{3}_{4}',
+            parameters.osGroup,
+            parameters.osSubgroup,
+            parameters.archType,
+            parameters.liveRuntimeBuildConfig,
+            parameters.globalBuildSuffix) }}
+    - ${{ if eq(parameters.useOfficialAllConfigurations, true) }}:
+      - build_Windows_NT_x64_${{ parameters.liveLibrariesBuildConfig }}_All_Configurations
 
   steps:
 

eng/pipelines/runtimelab.yml

@@ -6,7 +6,7 @@ trigger:
   batch: true
   branches:
     include:
-    # Add Experiment branch here
+    - feature/*
   paths:
     include:
     - '*'
@@ -26,7 +26,7 @@ trigger:
 pr:
   branches:
     include:
-    # Add Experiment branch here
+    - feature/*
   paths:
     include:
     - '*'
@@ -45,141 +45,182 @@ pr:
 
 variables:
   - template: /eng/pipelines/common/variables.yml
+  - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
+    - name: TeamName
+      value: dotnet-core
 
-jobs:
-#
-# Checkout repository
-#
-- template: /eng/pipelines/common/checkout-job.yml
+stages:
+- stage: Build
+  jobs:
+  #
+  # Checkout repository
+  #
+  - template: /eng/pipelines/common/checkout-job.yml
 
-#
-# Build with Debug config and Checked runtimeConfiguration
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/common/global-build-job.yml
-    buildConfig: Checked
-    platforms:
-    - Linux_x64
-    - Windows_NT_x64
-    jobParameters:
-      testGroup: innerloop
-      buildArgs: -s clr+libs+installer -c debug -runtimeConfiguration Checked
-      extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml
+  #
+  # Build with Debug config and Checked runtimeConfiguration
+  #
+  - ${{ if ne(variables.isOfficialBuild, true) }}:
+    - template: /eng/pipelines/common/platform-matrix.yml
+      parameters:
+        jobTemplate: /eng/pipelines/common/global-build-job.yml
+        buildConfig: Checked
+        platforms:
+        - Linux_x64
+        - Windows_NT_x64
+        jobParameters:
+          timeoutInMinutes: 100
+          testGroup: innerloop
+          buildArgs: -s clr+libs+installer -c debug -runtimeConfiguration Checked
+          extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml
+          extraStepsParameters:
+            uploadRuntimeTests: true
 
-#
-# Build with Release config and Release runtimeConfiguration
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/common/global-build-job.yml
-    buildConfig: Release
-    platforms:
-    - Linux_x64
-    - Windows_NT_x64
-    jobParameters:
-      testGroup: innerloop
-      buildArgs: -s clr+libs+libs.tests+installer -c $(_BuildConfig) /p:ArchiveTests=true
-      extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml
-      extraStepsParameters:
-        uploadTests: true
+  #
+  # Build with Release config and Release runtimeConfiguration
+  #
+  - template: /eng/pipelines/common/platform-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/common/global-build-job.yml
+      buildConfig: Release
+      platforms:
+      - Linux_x64
+      - Windows_NT_x64
+      jobParameters:
+        isOfficialBuild: ${{ variables.isOfficialBuild }}
+        testGroup: innerloop
+        extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml
+        extraStepsParameters:
+          uploadLibrariesTests: ${{ eq(variables.isOfficialBuild, false) }}
+          uploadUnsignedArtifacts: false
+        ${{ if eq(variables.isOfficialBuild, false) }}:
+          buildArgs: -s clr+libs+libs.tests+installer -c $(_BuildConfig) /p:ArchiveTests=true
+        ${{ if eq(variables.isOfficialBuild, true) }}:
+          buildArgs: -s clr+libs -c $(_BuildConfig)
 
-#
-# Build with Release allConfigurations to produce packages
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/common/global-build-job.yml
-    buildConfig: release
-    platforms:
-    - Windows_NT_x64
-    jobParameters:
-      testGroup: innerloop
-      nameSuffix: All_Configurations
-      buildArgs: -s libs -c $(_BuildConfig) -allConfigurations
+  #
+  # Build with Release allConfigurations to produce packages
+  #
+  - template: /eng/pipelines/common/platform-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/common/global-build-job.yml
+      buildConfig: Release
+      platforms:
+      - Windows_NT_x64
+      jobParameters:
+        isOfficialBuild: ${{ variables.isOfficialBuild }}
+        testGroup: innerloop
+        nameSuffix: All_Configurations
+        buildArgs: -s libs -c $(_BuildConfig) -allConfigurations
+        ${{ if eq(variables.isOfficialBuild, true) }}:
+          extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml
+          extraStepsParameters:
+            uploadUnsignedArtifacts: true
+            isOfficialBuild: true
+            librariesBinArtifactName: libraries_bin_official_allconfigurations
 
-#
-# CoreCLR Test builds using live libraries release build
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
-    buildConfig: Checked
-    platforms:
-    - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
-    jobParameters:
-      testGroup: innerloop
-      liveLibrariesBuildConfig: Release
-      dependsOn:
-      - build_Linux_x64_Checked_
-      - build_Linux_x64_Release_
+  # Installer official builds need to build installers and need the libraries all configurations build
+  - ${{ if eq(variables.isOfficialBuild, true) }}:
+    - template: /eng/pipelines/installer/installer-matrix.yml
+      parameters:
+        jobParameters:
+          liveRuntimeBuildConfig: Release
+          liveLibrariesBuildConfig: Release
+          isOfficialBuild: ${{ variables.isOfficialBuild }}
+          useOfficialAllConfigurations: true
+          dependsOnGlobalBuild: true
+        platforms:
+        - Linux_x64
+        - Windows_NT_x64
 
-#
-# CoreCLR Test executions using live libraries
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
-    buildConfig: Checked
-    platforms:
-    - Linux_x64
-    helixQueueGroup: pr
-    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
-    jobParameters:
-      testGroup: innerloop
-      liveLibrariesBuildConfig: Release
-      dependsOn:
-      - coreclr_common_test_build_p0_AnyOS_AnyCPU_Checked
+  - ${{ if ne(variables.isOfficialBuild, true) }}:
+    #
+    # CoreCLR Test builds using live libraries release build
+    #
+    - template: /eng/pipelines/common/platform-matrix.yml
+      parameters:
+        jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
+        buildConfig: Checked
+        platforms:
+        - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
+        jobParameters:
+          testGroup: innerloop
+          liveLibrariesBuildConfig: Release
+          dependsOn:
+          - build_Linux_x64_Checked_
+          - build_Linux_x64_Release_
 
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
-    buildConfig: Checked
-    platforms:
-    - Windows_NT_x64
-    helixQueueGroup: pr
-    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
-    jobParameters:
-      testGroup: innerloop
-      liveLibrariesBuildConfig: Release
-      dependsOn:
-      - coreclr_common_test_build_p0_AnyOS_AnyCPU_Checked
-      - build_Windows_NT_x64_Checked_
-      - build_Windows_NT_x64_Release_
+    #
+    # CoreCLR Test executions using live libraries
+    #
+    - template: /eng/pipelines/common/platform-matrix.yml
+      parameters:
+        jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
+        buildConfig: Checked
+        platforms:
+        - Linux_x64
+        helixQueueGroup: pr
+        helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+        jobParameters:
+          testGroup: innerloop
+          liveLibrariesBuildConfig: Release
+          dependsOn:
+          - coreclr_common_test_build_p0_AnyOS_AnyCPU_Checked
 
-#
-# Libraries Release Test Execution against a release coreclr runtime
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/libraries/run-test-job.yml
-    buildConfig: Release
-    platforms:
-    - Linux_x64
-    helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
-    jobParameters:
-      isFullMatrix: false
-      isOfficialBuild: false
-      testScope: innerloop
-      liveRuntimeBuildConfig: Release
-      dependsOnTestBuildConfiguration: Release
-      dependsOnTestArchitecture: x64
-      dependsOn:
-      - build_Linux_x64_Release_
+    - template: /eng/pipelines/common/platform-matrix.yml
+      parameters:
+        jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
+        buildConfig: Checked
+        platforms:
+        - Windows_NT_x64
+        helixQueueGroup: pr
+        helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+        jobParameters:
+          testGroup: innerloop
+          liveLibrariesBuildConfig: Release
+          dependsOn:
+          - coreclr_common_test_build_p0_AnyOS_AnyCPU_Checked
+          - build_Windows_NT_x64_Checked_
+          - build_Windows_NT_x64_Release_
 
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/libraries/run-test-job.yml
-    buildConfig: Release
-    platforms:
-    - Windows_NT_x64
-    helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
-    jobParameters:
-      isFullMatrix: false
-      isOfficialBuild: false
-      testScope: innerloop
-      liveRuntimeBuildConfig: Release
-      dependsOnTestBuildConfiguration: Release
-      dependsOnTestArchitecture: x64
-      dependsOn:
-      - build_Windows_NT_x64_Release_
+    #
+    # Libraries Release Test Execution against a release coreclr runtime
+    #
+    - template: /eng/pipelines/common/platform-matrix.yml
+      parameters:
+        jobTemplate: /eng/pipelines/libraries/run-test-job.yml
+        buildConfig: Release
+        platforms:
+        - Linux_x64
+        helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
+        jobParameters:
+          isFullMatrix: false
+          isOfficialBuild: false
+          testScope: innerloop
+          liveRuntimeBuildConfig: Release
+          dependsOnTestBuildConfiguration: Release
+          dependsOnTestArchitecture: x64
+          dependsOn:
+          - build_Linux_x64_Release_
+
+    - template: /eng/pipelines/common/platform-matrix.yml
+      parameters:
+        jobTemplate: /eng/pipelines/libraries/run-test-job.yml
+        buildConfig: Release
+        platforms:
+        - Windows_NT_x64
+        helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
+        jobParameters:
+          isFullMatrix: false
+          isOfficialBuild: false
+          testScope: innerloop
+          liveRuntimeBuildConfig: Release
+          dependsOnTestBuildConfiguration: Release
+          dependsOnTestArchitecture: x64
+          dependsOn:
+          - build_Windows_NT_x64_Release_
+
+- ${{ if eq(variables.isOfficialBuild, true) }}:
+  - template: /eng/pipelines/official/stages/publish.yml
+    parameters:
+      isOfficialBuild: ${{ variables.isOfficialBuild }}

eng/pipelines/runtimelab/runtimelab-post-build-steps.yml

@@ -3,15 +3,23 @@ parameters:
   archType: ''
   osGroup: ''
   osSubgroup: ''
-  uploadTests: false
+  nameSuffix: ''
+  librariesBinArtifactName: ''
+  isOfficialBuild: false
+  uploadLibrariesTests: false
+  uploadRuntimeTests: false
+  uploadUnsignedArtifacts: false
 
 steps:
   # Build coreclr native test output
-  - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipstressdependencies skipmanaged skipgeneratelayout $(buildConfigUpper) ${{ parameters.archType }}
-    displayName: Build native test components
+  - ${{ if eq(parameters.uploadRuntimeTests, true) }}:
+    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipstressdependencies skipmanaged skipgeneratelayout $(buildConfigUpper) ${{ parameters.archType }}
+      displayName: Build native test components
 
   # Copy all build output into artifacts staging directory
   - template: /eng/pipelines/libraries/prepare-for-bin-publish.yml
+    parameters:
+      isOfficialBuild: ${{ parameters.isOfficialBuild }}
 
   # Zip CoreCLR Build Output
   - template: /eng/pipelines/common/upload-artifact-step.yml
@@ -25,7 +33,7 @@ steps:
       displayName: 'CoreCLR product build'
 
   # Zip Test Build
-  - ${{ if eq(parameters.uploadTests, true) }}:
+  - ${{ if eq(parameters.uploadLibrariesTests, true) }}:
     - template: /eng/pipelines/common/upload-artifact-step.yml
       parameters:
         rootFolder: $(Build.SourcesDirectory)/artifacts/helix
@@ -36,16 +44,17 @@ steps:
         artifactName: libraries_test_assets_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}
         displayName: Test Assets
 
-  # Zip product native assets for use by Tests
-  - template: /eng/pipelines/common/upload-artifact-step.yml
-    parameters:
-      rootFolder: $(Build.SourcesDirectory)/artifacts/tests/coreclr/obj/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)
-      includeRootFolder: false
-      archiveType: $(archiveType)
-      tarCompression: $(tarCompression)
-      archiveExtension: $(archiveExtension)
-      artifactName: CoreCLRNativeTestArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}
-      displayName: 'native test components'
+  # Zip runtime native assets for use by Tests
+  - ${{ if eq(parameters.uploadRuntimeTests, true) }}:
+    - template: /eng/pipelines/common/upload-artifact-step.yml
+      parameters:
+        rootFolder: $(Build.SourcesDirectory)/artifacts/tests/coreclr/obj/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)
+        includeRootFolder: false
+        archiveType: $(archiveType)
+        tarCompression: $(tarCompression)
+        archiveExtension: $(archiveExtension)
+        artifactName: CoreCLRNativeTestArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}
+        displayName: 'native test components'
 
   # Zip Libraries Build Output
   - template: /eng/pipelines/common/upload-artifact-step.yml
@@ -55,5 +64,11 @@ steps:
       tarCompression: $(tarCompression)
       includeRootFolder: false
       archiveExtension: $(archiveExtension)
-      artifactName: libraries_bin_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}
+      artifactName: ${{ coalesce(parameters.librariesBinArtifactName, format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig)) }}
       displayName: Build Assets
+
+  # Upload unsigned artifacts
+  - ${{ if eq(parameters.uploadUnsignedArtifacts, true) }}:
+    - template: /eng/pipelines/common/upload-unsigned-artifacts-step.yml
+      parameters:
+        name: packages_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}${{ parameters.nameSuffix }}