Merge pull request #5775 from microsoft/sammeluch/2.0-release-prep

June 2023 Update 3

.github/policies/resourceManagement.yml

@@ -1,83 +1,84 @@
-id: 
+id:
 name: GitOps.PullRequestIssueManagement
 description: GitOps.PullRequestIssueManagement primitive
-owner: 
+owner:
 resource: repository
 disabled: false
-where: 
+where:
 configuration:
   resourceManagementConfiguration:
     scheduledSearches: []
     eventResponderTasks:
-    - if:
-      - payloadType: Pull_Request
-      then:
       - if:
-        - includesModifiedFiles:
-            files:
-            - SPECS/
-            - SPECS-SIGNED/
+          - payloadType: Pull_Request
         then:
-        - addLabel:
-            label: Packaging
+          - if:
+              - includesModifiedFiles:
+                  files:
+                    - SPECS/
+                    - SPECS-EXTENDED/
+                    - SPECS-SIGNED/
+            then:
+              - addLabel:
+                  label: Packaging
+          - if:
+              - includesModifiedFiles:
+                  files:
+                    - toolkit/docs/
+            then:
+              - addLabel:
+                  label: documentation
+          - if:
+              - includesModifiedFiles:
+                  files:
+                    - toolkit/tools/
+                    - toolkit/scripts/
+            then:
+              - addLabel:
+                  label: Tools
+          - if:
+              - includesModifiedFiles:
+                  files:
+                    - toolkit/imageconfigs/
+                    - toolkit/tools/imagegen/configuration
+            then:
+              - addLabel:
+                  label: Schema
+        description:
       - if:
-        - includesModifiedFiles:
-            files:
-            - toolkit/docs/
+          - payloadType: Pull_Request
+          - filesMatchPattern:
+              pattern: (CVE|cve)-\d+-\d+\.(no)?patch
         then:
-        - addLabel:
-            label: documentation
+          - addLabel:
+              label: security
+        description:
       - if:
-        - includesModifiedFiles:
-            files:
-            - toolkit/tools/
-            - toolkit/scripts/
+          - payloadType: Pull_Request
+          - isAction:
+              action: Opened
+          - isActivitySender:
+              user: CBL-Mariner-Bot
+              issueAuthor: False
         then:
-        - addLabel:
-            label: Tools
+          - addLabel:
+              label: Automatic PR
+        description:
       - if:
-        - includesModifiedFiles:
-            files:
-            - toolkit/imageconfigs/
-            - toolkit/tools/imagegen/configuration
+          - payloadType: Pull_Request
         then:
-        - addLabel:
-            label: Schema
-      description: 
-    - if:
-      - payloadType: Pull_Request
-      - filesMatchPattern:
-          pattern: (CVE|cve)-\d+-\d+\.(no)?patch
-      then:
-      - addLabel:
-          label: security
-      description: 
-    - if:
-      - payloadType: Pull_Request
-      - isAction:
-          action: Opened
-      - isActivitySender:
-          user: CBL-Mariner-Bot
-          issueAuthor: False
-      then:
-      - addLabel:
-          label: Automatic PR
-      description: 
-    - if:
-      - payloadType: Pull_Request
-      then:
-      - if:
-        - targetsBranch:
-            branch: 1.0-dev
-        then:
-        - addLabel:
-            label: 1.0-dev
-      - if:
-        - targetsBranch:
-            branch: main
-        then:
-        - addLabel:
-            label: main
-      description: 
-onFailure: 
-onSuccess: 
+          - if:
+              - targetsBranch:
+                  branch: 1.0-dev
+            then:
+              - addLabel:
+                  label: 1.0-dev
+          - if:
+              - targetsBranch:
+                  branch: main
+            then:
+              - addLabel:
+                  label: main
+        description:
+onFailure:
+onSuccess:

.pipelines/livepatching/BuildLivepatch.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 set -e
 
 ROOT_DIR="$(git rev-parse --show-toplevel)"

.pipelines/livepatching/BuildLivepatchSigned.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 set -e
 
 ROOT_DIR="$(git rev-parse --show-toplevel)"

.pipelines/prchecks/PackageBuildPRCheck.yml

@@ -0,0 +1,93 @@
+# Since we're boosting our builds by using a private, pre-compiled raw toolchain
+# the pipeline requires defining the following variables outside of the YAML:
+# - rawToolchainCacheURL_AMD64
+# - rawToolchainCacheURL_ARM64
+# - agentPool_AMD64
+# - agentPool_ARM64
+# - agentVMName_AMD64
+# - agentVMName_ARM64
+
+trigger: none
+
+parameters:
+  - name: buildConfiguration
+    type: object
+    default:
+    - name: "AMD64"
+      agentPool: "$(agentPool_AMD64)"
+      agentVMName: "$(agentVMName_AMD64)"
+      rawToolchainCacheURL: "$(rawToolchainCacheURL_AMD64)"
+      rawToolchainExpectedHash: "f56df34b90915c93f772d3961bf5e9eeb8c1233db43dd92070214e4ce6b72894"
+    - name: "ARM64"
+      agentPool: "$(agentPool_ARM64)"
+      agentVMName: "$(agentVMName_ARM64)"
+      rawToolchainCacheURL: "$(rawToolchainCacheURL_ARM64)"
+      rawToolchainExpectedHash: "65de43b3bdcfdaac71df1f11fd1f830a8109b1eb9d7cb6cbc2e2d0e929d0ef76"
+
+resources:
+  repositories:
+    - repository: templates
+      type: git
+      name: OneBranch.Pipelines/GovernedTemplates
+      ref: refs/heads/main
+
+extends:
+  template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates
+  parameters:
+    featureFlags:
+      runOnHost: true
+    globalSdl:
+      credscan:
+        suppressionsFile: .config/CredScanSuppressions.json
+    stages:
+      - ${{ each configuration in parameters.buildConfiguration }}:
+        - stage: Toolchain_${{ configuration.name }}
+          jobs:
+            - job: Build
+              pool:
+                type: linux
+                isCustom: true
+                name: ${{ configuration.agentPool }}
+                vmImage: ${{ configuration.agentVMName }}
+              variables:
+                ob_outputDirectory: $(Build.ArtifactStagingDirectory)
+              steps:
+                - template: .pipelines/templates/RawToolchainDownload.yml@self
+                  parameters:
+                    rawToolchainCacheURL: ${{ configuration.rawToolchainCacheURL }}
+                    rawToolchainExpectedHash: ${{ configuration.rawToolchainExpectedHash }}
+
+                - template: .pipelines/templates/ToolchainBuild.yml@self
+                  parameters:
+                    buildArtifactsFolder: $(ob_outputDirectory)
+
+                # 1. Automatic publishing seems to be broken in the OneBranch templates when the build runs directly on the agent host.
+                #    Once that's fixed, we should remove this step.
+                # 2. The value for 'artifact' cannot be changed, as this is the only value OneBranch accepts.
+                #    We cannot also use pre-defined variables like "drop_$(System.StageName)_$(Agent.JobName)", to automatically track name changes.
+                - task: PublishPipelineArtifact@1
+                  inputs:
+                    artifact: drop_Toolchain_${{ configuration.name }}_Build
+                    targetPath: $(ob_outputDirectory)
+                  condition: always()
+                  displayName: 'Publish toolchain artifacts'
+
+        - stage: RPMs_${{ configuration.name }}
+          dependsOn: Toolchain_${{ configuration.name }}
+          jobs:
+            - job: Build
+              pool:
+                type: linux
+                isCustom: true
+                name: ${{ configuration.agentPool }}
+                vmImage: ${{ configuration.agentVMName }}
+              strategy:
+                matrix:
+                  regular:
+                    runCheck: 'false'
+                  ptest:
+                    runCheck: 'true'
+              steps:
+                - template: .pipelines/templates/PackageBuild.yml@self
+                  parameters:
+                    isCheckBuild: '$(runCheck)'

.pipelines/templates/RawToolchainDownload.yml

@@ -0,0 +1,37 @@
+parameters:
+  - name: coreRepoRoot
+    type: string
+    default: "$(Build.SourcesDirectory)"
+
+  - name: rawToolchainCacheURL
+    type: string
+
+  - name: rawToolchainExpectedHash
+    type: string
+
+steps:
+  - bash: |
+      set -e
+
+      raw_toolchain_file_path="${{ parameters.coreRepoRoot }}/build/toolchain/toolchain_from_container.tar.gz"
+
+      echo "-- Downloading cached raw toolchain."
+
+      mkdir -p "$(dirname "$raw_toolchain_file_path")"
+      if ! wget --quiet --timeout=30 --continue "$RAW_TOOLCHAIN_URL" -O "$raw_toolchain_file_path"; then
+          echo "-- ERROR: failed to download raw toolchain cache." >&2
+          exit 1
+      fi
+
+      # Verifying toolchain's SHA-256 hash.
+      cache_sha256=$(sha256sum "$raw_toolchain_file_path" | cut -d' ' -f1)
+      if [[ "$cache_sha256" != "${{ parameters.rawToolchainExpectedHash }}" ]]; then
+          echo "-- ERROR: raw toolchain hash verification failed. Expected (${{ parameters.rawToolchainExpectedHash }}). Got ($cache_sha256)." >&2
+          exit 1
+      fi
+      echo "-- Raw toolchain hash OK."
+
+      touch "$raw_toolchain_file_path"
+    env:
+      RAW_TOOLCHAIN_URL: ${{ parameters.rawToolchainCacheURL }}
+    displayName: "Populate raw toolchain"