From 39133cc0b377a6eb267e29b6173bac7e8fabba51 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 15 Jul 2024 16:22:42 +0000 Subject: [PATCH 1/6] Merge jobs in C-ACI CI --- .azure-pipelines-templates/deploy_aci.yml | 90 ++++++++++++++++------- .azure_pipelines_snp.yml | 2 - 2 files changed, 65 insertions(+), 27 deletions(-) diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index 79b8f435ea7d..e6a98c2e2ef6 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -89,27 +89,71 @@ jobs: env: CCF_AZURE_SUBSCRIPTION_ID: $(CCF_AZURE_SUBSCRIPTION_ID) - - job: cleanup_aci - displayName: "Cleanup ACI" - container: virtual - pool: - name: ado-virtual-ccf-sub # For access to managed identity - demands: - - WorkFolder -equals /mnt/storage - dependsOn: - - generate_ssh_key - - deploy_primary_aci - - ${{ parameters.used_by }} - condition: always() - variables: - Codeql.SkipTaskAutoInjection: true - skipComponentGovernanceDetection: true - IpAddresses: $[ dependencies.deploy_primary_aci.outputs['deploy_primary_aci.ipAddresses'] ] - sshKey: $[ dependencies.generate_ssh_key.outputs['generate_ssh_key.sshKey'] ] - steps: - - template: install_ssh_key.yml - parameters: - ssh_key: $(sshKey) + - script: | + set -ex + set -o pipefail + runOn=$[variables.ipAddresses] + mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') + ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' + echo ENV + env + echo ACI_ENV + cat /aci_env + ' + name: print_env + displayName: "Print Environment Variables" + + - script: | + set -ex + set -o pipefail + runOn=$[variables.ipAddresses] + mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') + ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' + set -ex + cd /CCF/build + npm config set cache /ccfci/workspace_$(Build.BuildNumber)/.npm + WORKSPACE=/ccfci/workspace_$(Build.BuildNumber) ELECTION_TIMEOUT_MS=10000 ./tests.sh -VV -T Test -LE "benchmark|perf|tlstest|vegeta|suite|snp_flaky" -E "lts_compatibility" + # Remove irrelevant and bulky data from workspace before uploading + find /ccfci/workspace_$(Build.BuildNumber) -type f -name cchost -delete + find /ccfci/workspace_$(Build.BuildNumber) -type f -name "*.so" -delete + find /ccfci/workspace_$(Build.BuildNumber) -type l -name "*.so" -delete + rm -rf /ccfci/workspace_$(Build.BuildNumber)/.npm + ' + name: run_ctest + displayName: "Run CTest" + + - script: | + set -ex + set -o pipefail + runOn=$[variables.ipAddresses] + mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') + ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' + dmesg + ' + name: run_dmesg + displayName: "Run DMesg" + condition: always() + + - script: | + set -ex + set -o pipefail + runOn=$[variables.ipAddresses] + mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') + scp -r agent@${IP_ADDR_LIST[0]}:/ccfci/workspace_$(Build.BuildNumber) . + name: fetch_workspace + displayName: "Fetch Workspace" + condition: always() + + - task: PublishPipelineArtifact@1 + inputs: + artifactName: "ACI Workspace" + targetPath: workspace_$(Build.BuildNumber) + condition: always() + + - script: rm -rf workspace_$(Build.BuildNumber) + name: cleanup_workspace + displayName: "Clean up Workspace" + condition: always() - script: | set -ex @@ -123,10 +167,6 @@ jobs: displayName: "Cleanup Workspace" continueOnError: true - - template: azure_cli.yml - parameters: - managed_identity_id: $(CCF_SNP_CI_MANAGED_IDENTITY_ID) - - script: | set -ex python3.8 -m venv ./scripts/azure_deployment/.env diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index 48d5f3bb4855..8e3d1641ccee 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -27,8 +27,6 @@ resources: options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: - - template: .azure-pipelines-templates/configure.yml - - template: .azure-pipelines-templates/deploy_aci.yml parameters: used_by: From ff67386277ebe278678ef6e61cefc5e78ac99983 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 15 Jul 2024 16:31:21 +0000 Subject: [PATCH 2/6] deduplicate --- .azure-pipelines-templates/deploy_aci.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index e6a98c2e2ef6..d1a9e07ca3b8 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -37,12 +37,6 @@ jobs: fetchDepth: 0 fetchTags: true - - script: | - set -ex - env - name: print_env - displayName: "Print Environment Variables" - - template: install_ssh_key.yml parameters: ssh_key: $(sshKey) @@ -150,13 +144,9 @@ jobs: targetPath: workspace_$(Build.BuildNumber) condition: always() - - script: rm -rf workspace_$(Build.BuildNumber) - name: cleanup_workspace - displayName: "Clean up Workspace" - condition: always() - - script: | set -ex + rm -rf workspace_$(Build.BuildNumber) mapfile -t IP_ADDR_LIST <<< $(echo "$(IpAddresses)" | awk '{print $2}') for IP_ADDR in "${IP_ADDR_LIST[@]}"; do ssh agent@$IP_ADDR -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' @@ -166,6 +156,7 @@ jobs: name: cleanup_workspace displayName: "Cleanup Workspace" continueOnError: true + condition: always() - script: | set -ex @@ -180,5 +171,6 @@ jobs: name: cleanup_primary_aci displayName: "Delete the primary ACIs and Azure Deployments" continueOnError: true + condition: always() env: CCF_AZURE_SUBSCRIPTION_ID: $(CCF_AZURE_SUBSCRIPTION_ID) From 3e98c7dee3ed43136d3c80ae56f526f46bd30a11 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 15 Jul 2024 16:51:35 +0000 Subject: [PATCH 3/6] update --- .azure-pipelines-templates/deploy_aci.yml | 14 ++++---------- .azure_pipelines_snp.yml | 15 +-------------- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index d1a9e07ca3b8..9f3d059f82fe 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -1,9 +1,3 @@ -parameters: - secondaries: - image: "" - count: 0 - path: "" - jobs: - job: generate_ssh_key displayName: "Generate SSH Key" @@ -86,7 +80,7 @@ jobs: - script: | set -ex set -o pipefail - runOn=$[variables.ipAddresses] + runOn=$(ipAddresses) mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' echo ENV @@ -100,7 +94,7 @@ jobs: - script: | set -ex set -o pipefail - runOn=$[variables.ipAddresses] + runOn=$(ipAddresses) mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' set -ex @@ -119,7 +113,7 @@ jobs: - script: | set -ex set -o pipefail - runOn=$[variables.ipAddresses] + runOn=$(ipAddresses) mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' dmesg @@ -131,7 +125,7 @@ jobs: - script: | set -ex set -o pipefail - runOn=$[variables.ipAddresses] + runOn=$(ipAddresses) mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') scp -r agent@${IP_ADDR_LIST[0]}:/ccfci/workspace_$(Build.BuildNumber) . name: fetch_workspace diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index 8e3d1641ccee..cad38d194061 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -27,17 +27,4 @@ resources: options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: - - template: .azure-pipelines-templates/deploy_aci.yml - parameters: - used_by: - - test_snp - - - template: .azure-pipelines-templates/test_on_remote.yml - parameters: - job_name: test_snp - display_name: "Test SNP" - depends_on: - - generate_ssh_key - - deploy_primary_aci - run_on: $[ dependencies.deploy_primary_aci.outputs['deploy_primary_aci.ipAddresses'] ] - ssh_key: $[ dependencies.generate_ssh_key.outputs['generate_ssh_key.sshKey'] ] + - template: .azure-pipelines-templates/deploy_aci.yml \ No newline at end of file From 60dd27ed016107a424c63a3cc16e3df8436644ac Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 15 Jul 2024 16:57:48 +0000 Subject: [PATCH 4/6] fmt --- .azure_pipelines_snp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index cad38d194061..b1b79aed94c4 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -27,4 +27,4 @@ resources: options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro jobs: - - template: .azure-pipelines-templates/deploy_aci.yml \ No newline at end of file + - template: .azure-pipelines-templates/deploy_aci.yml From 700e3631fd304a59347b933315a06e05509a3c68 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 17 Jul 2024 09:32:56 +0000 Subject: [PATCH 5/6] use file --- .azure-pipelines-templates/deploy_aci.yml | 19 ++-- .azure-pipelines-templates/test_on_remote.yml | 87 ------------------- .azure_pipelines_snp.yml | 1 - 3 files changed, 8 insertions(+), 99 deletions(-) delete mode 100644 .azure-pipelines-templates/test_on_remote.yml diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index 9f3d059f82fe..ab6e075cf49d 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -69,9 +69,10 @@ jobs: --aci-setup-timeout 300 \ --aci-private-key-b64 $(sshKey) \ --out ~/aci_ips - # Set a variable "ipAddresses" which is a list of ` ` separated by newlines. + # Create a ~/ipAddresses files which is a list of ` ` separated by newlines. source ./scripts/azure_deployment/escape_data.sh # Include escape_data to handle newlines. - echo "##vso[task.setvariable variable=ipAddresses;isOutput=true]$(escape_data "$(cat ~/aci_ips)")" + escape_data "$(cat ~/aci_ips)" > ~/ipAddresses + cat ~/ipAddresses name: deploy_primary_aci displayName: "Deploy Primary ACI" env: @@ -80,8 +81,7 @@ jobs: - script: | set -ex set -o pipefail - runOn=$(ipAddresses) - mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') + mapfile -t IP_ADDR_LIST <<< $(cat ~/ipAddresses | awk '{print $2}') ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' echo ENV env @@ -94,8 +94,7 @@ jobs: - script: | set -ex set -o pipefail - runOn=$(ipAddresses) - mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') + mapfile -t IP_ADDR_LIST <<< $(cat ~/ipAddresses | awk '{print $2}') ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' set -ex cd /CCF/build @@ -113,8 +112,7 @@ jobs: - script: | set -ex set -o pipefail - runOn=$(ipAddresses) - mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') + mapfile -t IP_ADDR_LIST <<< $(cat ~/ipAddresses | awk '{print $2}') ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' dmesg ' @@ -125,8 +123,7 @@ jobs: - script: | set -ex set -o pipefail - runOn=$(ipAddresses) - mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') + mapfile -t IP_ADDR_LIST <<< $(cat ~/ipAddresses | awk '{print $2}') scp -r agent@${IP_ADDR_LIST[0]}:/ccfci/workspace_$(Build.BuildNumber) . name: fetch_workspace displayName: "Fetch Workspace" @@ -141,7 +138,7 @@ jobs: - script: | set -ex rm -rf workspace_$(Build.BuildNumber) - mapfile -t IP_ADDR_LIST <<< $(echo "$(IpAddresses)" | awk '{print $2}') + mapfile -t IP_ADDR_LIST <<< $(cat ~/ipAddresses | awk '{print $2}') for IP_ADDR in "${IP_ADDR_LIST[@]}"; do ssh agent@$IP_ADDR -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' sudo rm -rf /ccfci/workspace_$(Build.BuildNumber) diff --git a/.azure-pipelines-templates/test_on_remote.yml b/.azure-pipelines-templates/test_on_remote.yml deleted file mode 100644 index 1884fa384b56..000000000000 --- a/.azure-pipelines-templates/test_on_remote.yml +++ /dev/null @@ -1,87 +0,0 @@ -parameters: - depends_on: "" - condition: "" - -jobs: - - job: ${{ parameters.job_name }} - displayName: ${{ parameters.display_name }} - dependsOn: ${{ parameters.depends_on }} - condition: ${{ parameters.condition }} - container: virtual - pool: - name: ado-virtual-ccf-sub - demands: - - WorkFolder -equals /mnt/storage - timeoutInMinutes: 120 - variables: - runOn: ${{ parameters.run_on }} - sshKey: ${{ parameters.ssh_key }} - Codeql.SkipTaskAutoInjection: true - skipComponentGovernanceDetection: true - - steps: - - template: install_ssh_key.yml - parameters: - ssh_key: $(sshKey) - - - script: | - set -ex - set -o pipefail - mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') - ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' - echo ENV - env - echo ACI_ENV - cat /aci_env - ' - name: print_env - displayName: "Print Environment Variables" - - - script: | - set -ex - set -o pipefail - mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') - ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' - set -ex - cd /CCF/build - npm config set cache /ccfci/workspace_$(Build.BuildNumber)/.npm - WORKSPACE=/ccfci/workspace_$(Build.BuildNumber) ELECTION_TIMEOUT_MS=10000 ./tests.sh -VV -T Test -LE "benchmark|perf|tlstest|vegeta|suite|snp_flaky" -E "lts_compatibility" - # Remove irrelevant and bulky data from workspace before uploading - find /ccfci/workspace_$(Build.BuildNumber) -type f -name cchost -delete - find /ccfci/workspace_$(Build.BuildNumber) -type f -name "*.so" -delete - find /ccfci/workspace_$(Build.BuildNumber) -type l -name "*.so" -delete - rm -rf /ccfci/workspace_$(Build.BuildNumber)/.npm - ' - name: run_ctest - displayName: "Run CTest" - - - script: | - set -ex - set -o pipefail - mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') - ssh agent@${IP_ADDR_LIST[0]} -o "StrictHostKeyChecking=no" -o ConnectTimeout=100 ' - dmesg - ' - name: run_dmesg - displayName: "Run DMesg" - condition: always() - - - script: | - set -ex - set -o pipefail - mapfile -t IP_ADDR_LIST <<< $(echo "$(runOn)" | awk '{print $2}') - scp -r agent@${IP_ADDR_LIST[0]}:/ccfci/workspace_$(Build.BuildNumber) . - name: fetch_workspace - displayName: "Fetch Workspace" - condition: always() - - - task: PublishPipelineArtifact@1 - inputs: - artifactName: "ACI Workspace" - targetPath: workspace_$(Build.BuildNumber) - condition: always() - - - script: rm -rf workspace_$(Build.BuildNumber) - name: cleanup_workspace - displayName: "Clean up Workspace" - condition: always() diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index b1b79aed94c4..072d1b3d7353 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -9,7 +9,6 @@ pr: - scripts/azure_deployment/* - .azure_pipelines_snp.yml - .azure-pipelines-templates/deploy_aci.yml - - .azure-pipelines-templates/test_on_remote.yml - .snpcc_canary schedules: From 36155332e275c428eafa98772b037d8d83ca72be Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 17 Jul 2024 09:50:00 +0000 Subject: [PATCH 6/6] Update base --- .azure-pipelines-templates/deploy_aci.yml | 6 +++--- docker/ccf_ci_built | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index ab6e075cf49d..8193053e33d9 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -42,8 +42,8 @@ jobs: - script: | set -ex docker login -u $ACR_TOKEN_NAME -p $ACR_CI_PUSH_TOKEN_PASSWORD $ACR_REGISTRY - docker pull $ACR_REGISTRY/ccf/ci:2024-06-26-snp-clang15 - docker build -f docker/ccf_ci_built . --build-arg="base=$BASE_IMAGE" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` + docker pull $BASE_IMAGE + docker build -f docker/ccf_ci_built . --build-arg="base=$BASE_IMAGE" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD` name: build_ci_image displayName: "Build CI SNP container" @@ -51,7 +51,7 @@ jobs: ACR_TOKEN_NAME: ci-push-token ACR_CI_PUSH_TOKEN_PASSWORD: $(ACR_CI_PUSH_TOKEN_PASSWORD) ACR_REGISTRY: ccfmsrc.azurecr.io - BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15 + BASE_IMAGE: ghcr.io/microsoft/ccf/ci/default:build-26-06-2024 - script: | set -ex diff --git a/docker/ccf_ci_built b/docker/ccf_ci_built index 4362f14b0428..ae3987a60497 100644 --- a/docker/ccf_ci_built +++ b/docker/ccf_ci_built @@ -1,10 +1,9 @@ -# CCF Continuous Integration image +# CCF Continuous Integration image for SNP # Contains CCF build dependencies and toolchain for target platform # Also contains CCF source and build directory # Latest image as of this change -ARG platform=sgx -ARG base=ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang-15 +ARG base=ghcr.io/microsoft/ccf/ci/default:build-26-06-2024 FROM ${base} # SSH. Note that this could (should) be done in the base ccf_ci image instead @@ -23,12 +22,11 @@ RUN useradd -m $user \ && chown -R $user:$user /home/$user/.ssh # Copy CCF source and build -ARG platform=sgx RUN mkdir /CCF COPY . /CCF/ RUN mkdir /CCF/build \ && cd /CCF/build \ - && cmake -GNinja -DCOMPILE_TARGET=${platform} .. \ + && cmake -GNinja -DCOMPILE_TARGET=snp .. \ && ninja \ && chmod -R 777 /CCF