diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index 8500fa89700..40b82cf60b9 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -23,7 +23,7 @@ resources: containers: - container: virtual image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE jobs: - template: .azure-pipelines-templates/deploy_aci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d47126e5895..67d070e7061 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,11 +37,11 @@ jobs: - name: virtual image: default nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] - options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE - name: snp image: default nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] - options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE runs-on: ${{ matrix.platform.nodes }} container: image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-14-01-2025 @@ -107,33 +107,26 @@ jobs: runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: image: mcr.microsoft.com/azurelinux/base/core:3.0 - options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE steps: - - name: "Install dependencies" + - name: "Checkout dependencies" + shell: bash run: | - set -ex gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY tdnf -y update - # Source control tdnf -y install ca-certificates git - # Build tools - tdnf -y install build-essential clang cmake ninja-build which - # libc++ - tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static - # Dependencies - tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel - # Test dependencies - tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect - # Install CDDL via rubygems - tdnf -y install rubygems - gem install cddl - shell: bash - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: "Install dependencies" + shell: bash + run: | + set -ex + ./scripts/install-azure-linux-deps.sh + - name: "Build Debug" run: | set -ex diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index a66e774cce0..8d435e90eb3 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -285,33 +285,26 @@ jobs: runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: image: mcr.microsoft.com/azurelinux/base/core:3.0 - options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE steps: - - name: "Install dependencies" + - name: "Checkout dependencies" + shell: bash run: | - set -ex gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY tdnf -y update - # Source control tdnf -y install ca-certificates git - # Build tools - tdnf -y install build-essential clang cmake ninja-build which - # libc++ - tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static - # Dependencies - tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel - # Test dependencies - tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect - # Install CDDL via rubygems - tdnf -y install rubygems - gem install cddl - shell: bash - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: "Install dependencies" + shell: bash + run: | + set -ex + ./scripts/install-azure-linux-deps.sh + - name: "Build Debug (Long Test)" run: | set -ex diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5a769279795..0114eb71a9d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,37 +62,65 @@ jobs: path: rel-notes.md build_release: - needs: release_notes name: Build Release + needs: release_notes strategy: matrix: platform: - name: virtual - image: default - nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + os: ubuntu + image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 + test_filter: "benchmark|unit" + - name: snp + os: ubuntu + image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 + - name: virtual + os: azure-linux + image: mcr.microsoft.com/azurelinux/base/core:3.0 + test_filter: "benchmark|unit|protocolstest|lts" - name: snp - image: default - nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] - runs-on: ${{ matrix.platform.nodes }} + os: azure-linux + image: mcr.microsoft.com/azurelinux/base/core:3.0 + + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: - image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-14-01-2025 - options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro ${{ matrix.platform.container_options }}" + image: ${{ matrix.platform.image }} + options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE" + steps: + - name: "Checkout dependencies" + if: ${{ matrix.platform.os == 'azure-linux' }} + shell: bash + run: | + gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY + tdnf -y update + tdnf -y install ca-certificates git + - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: "Install dependencies" + if: ${{ matrix.platform.os == 'azure-linux' }} + shell: bash + run: | + set -ex + ./scripts/install-azure-linux-deps.sh + # For packaging and release tests + tdnf -y install rpm-build + - name: "Build Release ${{ matrix.platform.name }}" + shell: bash run: | set -ex git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build - cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} ${{ matrix.platform.cmake_options }} -DCLIENT_PROTOCOLS_TEST=ON .. + CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DCMAKE_BUILD_TYPE=Release .. ninja -v | tee build.log - shell: bash - name: "Install Extended Testing Tools" + if: ${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }} run: | set -ex sudo apt-get -y update @@ -100,9 +128,9 @@ jobs: cd getting_started/setup_vm ansible-playbook ccf-extended-testing.yml shell: bash - if: ${{ matrix.platform.name != 'snp' }} - name: "Test ${{ matrix.platform.name }}" + if: "${{ matrix.platform.name == 'virtual' }}" run: | set -ex cd build @@ -114,23 +142,23 @@ jobs: # Suite tests ./tests.sh --timeout 600 --output-on-failure -L "suite" # Most tests - ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|unit|suite" + ./tests.sh --timeout 360 --output-on-failure -LE "suite|${{ matrix.platform.test_filter }}" shell: bash - if: "${{ matrix.platform.name != 'snp' }}" - - name: "Upload logs for ${{ matrix.platform.name }}" + - name: "Upload logs for ${{ matrix.platform.os }}-${{ matrix.platform.name }}" + if: success() || failure() uses: actions/upload-artifact@v4 with: - name: logs-${{ matrix.platform.name }} + name: logs-${{ matrix.platform.os }}-${{ matrix.platform.name }} path: | build/workspace/*/*.config.json build/workspace/*/out build/workspace/*/err build/workspace/*.ledger/* if-no-files-found: ignore - if: success() || failure() - name: "Make .deb Package" + if: "${{ matrix.platform.os == 'ubuntu' }}" id: make_deb run: | set -ex @@ -146,30 +174,56 @@ jobs: echo "name=$CCF_GITHUB_PKG" >> $GITHUB_OUTPUT shell: bash + - name: "Make .rpm Package" + if: "${{ matrix.platform.os == 'azure-linux' }}" + id: make_rpm + run: | + set -ex + set -o pipefail + cd build + cmake -L .. 2>/dev/null | grep CMAKE_INSTALL_PREFIX: | cut -d = -f 2 > /tmp/install_prefix + cpack -V -G RPM + INITIAL_PKG=`ls *.rpm` + CCF_GITHUB_PKG=${INITIAL_PKG//\~/_} + if [[ "$INITIAL_PKG" != "$CCF_GITHUB_PKG" ]]; then + mv $INITIAL_PKG $CCF_GITHUB_PKG + fi + echo "name=$CCF_GITHUB_PKG" >> $GITHUB_OUTPUT + shell: bash + - name: "Install CCF Debian package" + if: "${{ matrix.platform.os == 'ubuntu' }}" run: | set -ex cd build sudo apt -y install ./${{ steps.make_deb.outputs.name }} shell: bash + - name: "Install CCF RPM package" + if: "${{ matrix.platform.os == 'azure-linux' }}" + run: | + set -ex + cd build + tdnf -y install ./${{ steps.make_rpm.outputs.name }} + shell: bash + - name: "Test Installed CCF" + if: "${{ matrix.platform.name == 'virtual' }}" run: | set -ex set -o pipefail cd build cat /tmp/install_prefix | xargs -i bash -c "PYTHON_PACKAGE_PATH=../python ./test_install.sh {}" shell: bash - if: "${{ matrix.platform.name != 'snp' }}" - name: "Recovery Benchmark for Installed CCF" + if: "${{ matrix.platform.name == 'virtual' }}" run: | set -ex set -o pipefail cd build cat /tmp/install_prefix | xargs -i bash -c "PYTHON_PACKAGE_PATH=../python ./recovery_benchmark.sh {}" shell: bash - if: "${{ matrix.platform.name != 'snp' }}" - name: "Test Building a Sample Against Installed CCF" run: | @@ -178,26 +232,35 @@ jobs: shell: bash - name: "Upload .deb Package" + if: "${{ matrix.platform.os == 'ubuntu' }}" uses: actions/upload-artifact@v4 with: - name: pkg-${{ matrix.platform.name }} + name: pkg-${{ matrix.platform.os }}-${{ matrix.platform.name }} path: build/${{ steps.make_deb.outputs.name }} + - name: "Upload .rpm Package" + if: "${{ matrix.platform.os == 'azure-linux' }}" + uses: actions/upload-artifact@v4 + with: + name: pkg-${{ matrix.platform.os }}-${{ matrix.platform.name }} + path: build/${{ steps.make_rpm.outputs.name }} + - name: "Upload Compatibility Report" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" uses: actions/upload-artifact@v4 with: name: compatibility path: build/compatibility_report.json - if: "${{ matrix.platform.name == 'virtual' }}" - name: "Upload TLS Report" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" uses: actions/upload-artifact@v4 with: name: tls path: build/tls_report.html - if: "${{ matrix.platform.name == 'virtual' }}" - name: "Build Python Wheel" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" id: build_wheel run: | set -ex @@ -209,16 +272,16 @@ jobs: WHL=`ls dist/*.whl` echo "name=$WHL" >> $GITHUB_OUTPUT shell: bash - if: "${{ matrix.platform.name == 'virtual' }}" - name: "Upload Python Wheel" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" uses: actions/upload-artifact@v4 with: name: wheel path: python/${{ steps.build_wheel.outputs.name }} - if: "${{ matrix.platform.name == 'virtual' }}" - name: "Build TS Package" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" id: build_tstgz run: | set -ex @@ -231,14 +294,13 @@ jobs: PKG=`ls *.tgz` echo "name=$PKG" >> $GITHUB_OUTPUT shell: bash - if: "${{ matrix.platform.name == 'virtual' }}" - name: "Upload TS Package" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" uses: actions/upload-artifact@v4 with: name: tstgz path: js/ccf-app/${{ steps.build_tstgz.outputs.name }} - if: "${{ matrix.platform.name == 'virtual' }}" create_release: needs: diff --git a/cmake/cpack_settings.cmake b/cmake/cpack_settings.cmake index 9626f3c2a93..557927c81ae 100644 --- a/cmake/cpack_settings.cmake +++ b/cmake/cpack_settings.cmake @@ -8,6 +8,8 @@ set(CPACK_RESOURCE_FILE_LICENSE "${CCF_DIR}/LICENSE") set(CPACK_PACKAGE_VERSION ${CCF_RELEASE_VERSION}) set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) +# DEB-specific settings + set(CPACK_DEBIAN_PACKAGE_VERSION "${CCF_RELEASE_VERSION}") if(CCF_VERSION_SUFFIX) @@ -32,6 +34,30 @@ endif() list(JOIN CCF_DEB_DEPENDENCIES ", " CPACK_DEBIAN_PACKAGE_DEPENDS) +message(STATUS "DEB package dependencies: ${CCF_DEB_DEPENDENCIES}") + set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) +# RPM-specific settings + +set(CPACK_RPM_PACKAGE_VERSION "${CCF_RELEASE_VERSION}") + +if(CCF_VERSION_SUFFIX) + set(CPACK_RPM_PACKAGE_VERSION + "${CPACK_RPM_PACKAGE_VERSION}~${CCF_VERSION_SUFFIX}" + ) +endif() + +message(STATUS "RPM package version: ${CPACK_RPM_PACKAGE_VERSION}") + +set(CCF_RPM_DEPENDENCIES + "libuv >= 1.34.2, openssl >= 3.3.0, nghttp2 >= 1.40.0, curl >= 7.68.0, libcxxabi >= 18.1.2" +) + +message(STATUS "RPM package dependencies: ${CCF_RPM_DEPENDENCIES}") + +set(CPACK_RPM_PACKAGE_REQUIRES "${CCF_RPM_DEPENDENCIES}") + +set(CPACK_RPM_FILE_NAME RPM-DEFAULT) + include(CPack) diff --git a/scripts/install-azure-linux-deps.sh b/scripts/install-azure-linux-deps.sh new file mode 100755 index 00000000000..f610712c291 --- /dev/null +++ b/scripts/install-azure-linux-deps.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +set -ex + +# Source control +tdnf -y install ca-certificates git + +# Build tools +tdnf -y install build-essential clang cmake ninja-build which + +# libc++ +tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static + +# Dependencies +tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel + +# Test dependencies +tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect procps + +# Install CDDL via rubygems +tdnf -y install rubygems +gem install cddl diff --git a/tests/test_install_build.sh b/tests/test_install_build.sh index 4252b16accb..546c0f31803 100755 --- a/tests/test_install_build.sh +++ b/tests/test_install_build.sh @@ -6,13 +6,14 @@ set -ex mkdir -p build_against_install cd build_against_install -CC=$(command -v clang-15 || true) -CXX=$(command -v clang++-15 || true) - -if [ "$CC" = "" ] || [ "$CXX" = "" ]; then - CC=$(command -v clang-11) - CXX=$(command -v clang++-11) +CC=$(which clang || true) +CXX=$(which clang++ || true) + +if [ "$CC" = "" ] || [ "$CXX" = "" ]; then + CC=$(command -v clang-15 || true) + CXX=$(command -v clang++-15 || true) fi CC=$CC CXX=$CXX cmake -GNinja "$@" ../samples/apps/logging/ + ninja \ No newline at end of file