diff --git a/.github/workflows/cve-scan-and-patching.yml b/.github/workflows/cve-scan-and-patching.yml index 35f1036a..14d4a8b5 100644 --- a/.github/workflows/cve-scan-and-patching.yml +++ b/.github/workflows/cve-scan-and-patching.yml @@ -11,6 +11,15 @@ on: - '!CVEs/MAINTENANCE.md' jobs: + set_docker_config_env_var: + runs-on: ubuntu-latest + steps: + - name: set DOCKER_CONFIG + id: set_docker_config_env + run: | + echo "DOCKER_CONFIG=/tmp/${GITHUB_RUN_ID}/.docker" >> $GITHUB_OUTPUT + outputs: + docker_config: ${{ steps.set_docker_config_env.outputs.DOCKER_CONFIG }} fetch_kfd_versions: runs-on: ubuntu-latest steps: @@ -26,9 +35,56 @@ jobs: kfd_versions_json: ${{ steps.set_output.outputs.KFD_VERSIONS_JSON }} kfd_versions_list: ${{ steps.set_output.outputs.KFD_VERSIONS_JSON }} today_date: ${{ steps.set_output.outputs.TODAY_DATE }} + install_tools: + runs-on: ubuntu-latest + needs: + - fetch_kfd_versions + steps: + - uses: actions/checkout@master + - name: cache tool used by jobs + id: cache_tools + uses: actions/cache@v4 + with: + path: | + /tmp/.cache/trivy + /usr/local/bin/trivy + /usr/local/bin/furyctl + /usr/local/bin/buildctl + /usr/local/bin/copa + key: cve-scan-patching-tools-${{ needs.fetch_kfd_versions.outputs.today_date }} + - name: install tools required by jobs + if: steps.cache_tools.outputs.cache-hit != 'true' + run: | + sudo apt-get install wget apt-transport-https gnupg + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get install trivy + trivy --version + cp /usr/bin/trivy /usr/local/bin/trivy + + wget https://github.com/sighupio/furyctl/releases/latest/download/furyctl-linux-amd64.tar.gz + tar -xzvf furyctl-linux-amd64.tar.gz -C /usr/local/bin/ + furyctl version + + wget https://github.com/moby/buildkit/releases/download/v0.16.0/buildkit-v0.16.0.linux-amd64.tar.gz + tar -xzvf buildkit-v0.16.0.linux-amd64.tar.gz -C /usr/local/bin/ --strip-components=1 + buildctl --version + + wget https://github.com/project-copacetic/copacetic/releases/download/v0.9.0/copa_0.9.0_linux_amd64.tar.gz + tar -xzvf copa_0.9.0_linux_amd64.tar.gz + chmod +x copa + sudo mv copa /usr/local/bin/ + copa --version + + cd CVEs + make trivy-download-db scan_pre_patch: runs-on: ubuntu-latest - needs: fetch_kfd_versions + needs: + - set_docker_config_env_var + - fetch_kfd_versions + - install_tools continue-on-error: true strategy: # max-parallel: 3 @@ -43,24 +99,33 @@ jobs: registry: registry.sighup.io username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install furyctl, trivy - run: | - sudo apt-get install wget apt-transport-https gnupg - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null - echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | sudo tee -a /etc/apt/sources.list.d/trivy.list - sudo apt-get update - sudo apt-get install trivy - trivy --version - - wget https://github.com/sighupio/furyctl/releases/latest/download/furyctl-linux-amd64.tar.gz - tar -xzvf furyctl-linux-amd64.tar.gz -C /usr/local/bin/ - furyctl version + env: + DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}} + - name: cache tools + uses: actions/cache@v4 + with: + fail-on-cache-miss: true + path: | + /tmp/.cache/trivy + /usr/local/bin/trivy + /usr/local/bin/furyctl + /usr/local/bin/buildctl + /usr/local/bin/copa + key: cve-scan-patching-tools-${{ needs.fetch_kfd_versions.outputs.today_date }} + - name: cache KFD vendors built manifest for KFD ${{ matrix.kfd_version }} + uses: actions/cache@v4 + with: + path: | + CVEs/${{ matrix.kfd_version }}/built.yaml + CVEs/${{ matrix.kfd_version }}/images.txt + key: kfd-${{ matrix.kfd_version }}-vendors-built-manifests - name: Execute CVEs scan for KFD ${{ matrix.kfd_version }} id: scan_pre_patch run: | cd CVEs - make trivy-download-db make scan-pre-patch KFD_VERSIONS="${{ matrix.kfd_version }}" + env: + DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}} - name: publish CVE scan pre patch output files for KFD ${{ matrix.kfd_version }} uses: actions/upload-artifact@v4 with: @@ -89,6 +154,9 @@ jobs: patch: runs-on: ubuntu-latest needs: + - install_tools + - fetch_kfd_versions + - set_docker_config_env_var - fetch_kfd_images_to_patch continue-on-error: true strategy: @@ -103,30 +171,33 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + env: + DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}} - name: Login to SIGHUP new Registry uses: docker/login-action@v3 with: registry: registry.sighup.io username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install buildkit, trivy and copa - run: | - sudo apt-get install wget apt-transport-https gnupg - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null - echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | sudo tee -a /etc/apt/sources.list.d/trivy.list - sudo apt-get update - sudo apt-get install trivy - trivy --version - - wget https://github.com/moby/buildkit/releases/download/v0.16.0/buildkit-v0.16.0.linux-amd64.tar.gz - tar -xzvf buildkit-v0.16.0.linux-amd64.tar.gz -C /usr/local/bin/ --strip-components=1 - buildctl --version - - wget https://github.com/project-copacetic/copacetic/releases/download/v0.8.0/copa_0.8.0_linux_amd64.tar.gz - tar -xzvf copa_0.8.0_linux_amd64.tar.gz - chmod +x copa - sudo mv copa /usr/local/bin/ - copa --version + env: + DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}} + - # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: amd64,arm64 + - name: cache tools + uses: actions/cache@v4 + with: + fail-on-cache-miss: true + path: | + /tmp/.cache/trivy + /usr/local/bin/trivy + /usr/local/bin/furyctl + /usr/local/bin/buildctl + /usr/local/bin/copa + key: cve-scan-patching-tools-${{ needs.fetch_kfd_versions.outputs.today_date }} - name: Execute CVEs patching id: patching run: | @@ -134,20 +205,24 @@ jobs: IMAGE_TO_PATCH_NORMALIZED=${IMAGE_TO_PATCH//[:\/]/_} cd CVEs mkdir -p reports - make trivy-download-db - DOCKER_CONFIG="${DOCKER_CONFIG}" make patch DRY_RUN=0 IMAGE_TO_PATCH="${IMAGE_TO_PATCH}" PATCH_REPORT_OUTPUT_FILE="reports/${IMAGE_TO_PATCH_NORMALIZED}.patched.md" - + make patch DRY_RUN=0 \ + IMAGE_TO_PATCH="${IMAGE_TO_PATCH}" \ + PATCH_REPORT_OUTPUT_FILE="reports/${IMAGE_TO_PATCH_NORMALIZED}.patched.md" echo "IMAGE_TO_PATCH_NORMALIZED=${IMAGE_TO_PATCH_NORMALIZED}" >> "$GITHUB_OUTPUT" + env: + DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}} - name: publish CVE patching report for ${{ matrix.image_to_patch }} uses: actions/upload-artifact@v4 with: name: patch-report-${{ steps.patching.outputs.IMAGE_TO_PATCH_NORMALIZED }}.md if-no-files-found: ignore path: | - CVEs/reports/*.patched.md + CVEs/reports/*.patched.*.md scan_post_patch: runs-on: ubuntu-latest needs: + - install_tools + - set_docker_config_env_var - fetch_kfd_versions - patch continue-on-error: true @@ -163,25 +238,30 @@ jobs: registry: registry.sighup.io username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install trivy - run: | - sudo apt-get install wget apt-transport-https gnupg - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null - echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | sudo tee -a /etc/apt/sources.list.d/trivy.list - sudo apt-get update - sudo apt-get install trivy - trivy --version + env: + DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}} - name: download CVE scan pre patch output files uses: actions/download-artifact@v4 with: path: CVEs + - name: cache tools + uses: actions/cache@v4 + with: + fail-on-cache-miss: true + path: | + /tmp/.cache/trivy + /usr/local/bin/trivy + /usr/local/bin/furyctl + /usr/local/bin/buildctl + /usr/local/bin/copa + key: cve-scan-patching-tools-${{ needs.fetch_kfd_versions.outputs.today_date }} - name: Execute CVEs scan post patch for KFD ${{ matrix.kfd_version }} id: scan_post_patch run: | cd CVEs - find . - make trivy-download-db make scan-post-patch KFD_VERSIONS="${{ matrix.kfd_version }}" + env: + DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}} - name: publish CVE scan post patch output files for KFD ${{ matrix.kfd_version }} uses: actions/upload-artifact@v4 with: @@ -207,7 +287,7 @@ jobs: with: name: cve-patch-reports-by-image path: | - reports/*.patched.md + reports/*.patched.*.md - uses: geekyeggo/delete-artifact@v5 with: name: | diff --git a/.github/workflows/dry.yml b/.github/workflows/dry.yml index fe2317ba..562da9c2 100644 --- a/.github/workflows/dry.yml +++ b/.github/workflows/dry.yml @@ -2,19 +2,43 @@ name: "Dry run" on: push: + paths: + - '.github/workflows/dry.yml' + - '.github/workflows/sync.yml' + - 'modules/**' + - '!README.md' + - '!single_sync.sh' + - '!single_sync_v2.sh' + - 'single_sync_v3.sh' jobs: + fetch_modules_to_sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Set output + id: set_output + run: | + MODULES_TO_SYNC_JSON=$(find modules -type d -mindepth 1 -maxdepth 1 | cut -d/ -f2 | sort | jq -R | jq -cs . ) + echo "MODULES_TO_SYNC_JSON=${MODULES_TO_SYNC_JSON}" >> $GITHUB_OUTPUT + outputs: + modules_to_sync: ${{ steps.set_output.outputs.MODULES_TO_SYNC_JSON }} sync: runs-on: ubuntu-latest + needs: fetch_modules_to_sync + strategy: + fail-fast: false + matrix: + module: ${{ fromJson(needs.fetch_modules_to_sync.outputs.modules_to_sync) }} steps: - uses: actions/checkout@master - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: registry.sighup.io username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} @@ -28,4 +52,4 @@ jobs: run: | yq --version docker run --rm quay.io/skopeo/stable:v1.13 --version - ./sync.sh --dry-run + ./single_sync_v3.sh modules/${{ matrix.module }}/images.yml true diff --git a/.github/workflows/sync-auth.yml b/.github/workflows/sync-auth.yml deleted file mode 100644 index 7a6da0ba..00000000 --- a/.github/workflows/sync-auth.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync AUTH" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/auth/images.yml false diff --git a/.github/workflows/sync-aws.yml b/.github/workflows/sync-aws.yml deleted file mode 100644 index c0b60a26..00000000 --- a/.github/workflows/sync-aws.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Sync AWS" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/aws/images.yml false diff --git a/.github/workflows/sync-dr.yml b/.github/workflows/sync-dr.yml deleted file mode 100644 index 2f93757a..00000000 --- a/.github/workflows/sync-dr.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync DR" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/dr/images.yml false diff --git a/.github/workflows/sync-extra.yml b/.github/workflows/sync-extra.yml deleted file mode 100644 index bedf4854..00000000 --- a/.github/workflows/sync-extra.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Sync EXTRA" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/extra/images.yml false diff --git a/.github/workflows/sync-ingress.yml b/.github/workflows/sync-ingress.yml deleted file mode 100644 index 906bae7c..00000000 --- a/.github/workflows/sync-ingress.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Sync INGRESS" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/ingress/images.yml false diff --git a/.github/workflows/sync-kafka.yml b/.github/workflows/sync-kafka.yml deleted file mode 100644 index 07ba1402..00000000 --- a/.github/workflows/sync-kafka.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync KAFKA" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/kafka/images.yml false diff --git a/.github/workflows/sync-keycloak.yml b/.github/workflows/sync-keycloak.yml deleted file mode 100644 index 75e5b2bc..00000000 --- a/.github/workflows/sync-keycloak.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync KEYCLOAK" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/keycloak/images.yml false diff --git a/.github/workflows/sync-kong.yml b/.github/workflows/sync-kong.yml deleted file mode 100644 index 81a9f5cf..00000000 --- a/.github/workflows/sync-kong.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync KONG" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/kong/images.yml false diff --git a/.github/workflows/sync-kuma.yml b/.github/workflows/sync-kuma.yml deleted file mode 100644 index e1adeed1..00000000 --- a/.github/workflows/sync-kuma.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Sync KUMA" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/kuma/images.yml false diff --git a/.github/workflows/sync-logging.yml b/.github/workflows/sync-logging.yml deleted file mode 100644 index 121c0fce..00000000 --- a/.github/workflows/sync-logging.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Sync LOGGING" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/logging/images.yml false diff --git a/.github/workflows/sync-monitoring.yml b/.github/workflows/sync-monitoring.yml deleted file mode 100644 index 2afb7a9a..00000000 --- a/.github/workflows/sync-monitoring.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Sync MONITORING" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/monitoring/images.yml false diff --git a/.github/workflows/sync-networking.yml b/.github/workflows/sync-networking.yml deleted file mode 100644 index 25e0909b..00000000 --- a/.github/workflows/sync-networking.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Sync NETWORKING" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/networking/images.yml false \ No newline at end of file diff --git a/.github/workflows/sync-on-premises.yml b/.github/workflows/sync-on-premises.yml deleted file mode 100644 index 2aa6fed1..00000000 --- a/.github/workflows/sync-on-premises.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Sync ON-PREMISES" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/on-premises/images.yml false diff --git a/.github/workflows/sync-opa.yml b/.github/workflows/sync-opa.yml deleted file mode 100644 index 6d6f6485..00000000 --- a/.github/workflows/sync-opa.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Sync OPA" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/opa/images.yml false diff --git a/.github/workflows/sync-postgresql.yml b/.github/workflows/sync-postgresql.yml deleted file mode 100644 index 7b8366dc..00000000 --- a/.github/workflows/sync-postgresql.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync POSTGRESQL" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/postgresql/images.yml false diff --git a/.github/workflows/sync-redis.yml b/.github/workflows/sync-redis.yml deleted file mode 100644 index 8415d396..00000000 --- a/.github/workflows/sync-redis.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync REDIS" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/redis/images.yml false diff --git a/.github/workflows/sync-registry.yml b/.github/workflows/sync-registry.yml deleted file mode 100644 index e07e5ac8..00000000 --- a/.github/workflows/sync-registry.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync REGISTRY" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/registry/images.yml false diff --git a/.github/workflows/sync-service-mesh.yml b/.github/workflows/sync-service-mesh.yml deleted file mode 100644 index 30fcfd71..00000000 --- a/.github/workflows/sync-service-mesh.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Sync SERVICE MESH" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json - docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/service-mesh/images.yml false diff --git a/.github/workflows/sync-storage.yml b/.github/workflows/sync-storage.yml deleted file mode 100644 index 451e60f6..00000000 --- a/.github/workflows/sync-storage.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync STORAGE" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/storage/images.yml false diff --git a/.github/workflows/sync-tracing.yml b/.github/workflows/sync-tracing.yml deleted file mode 100644 index 2a1c335f..00000000 --- a/.github/workflows/sync-tracing.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync TRACING" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/tracing/images.yml false diff --git a/.github/workflows/sync-vault.yml b/.github/workflows/sync-vault.yml deleted file mode 100644 index cdeb2052..00000000 --- a/.github/workflows/sync-vault.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Sync VAULT" - -on: - push: - branches: - - "main" - schedule: - - cron: "0 2 * * *" - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 - with: - registry: registry.sighup.io - username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} - password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} - - name: Install yq - run: | - sudo apt-get update && sudo apt-get install -yqq wget - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Iterate - run: | - yq --version - docker run --rm quay.io/skopeo/stable:v1.13 --version - ./single_sync.sh modules/vault/images.yml false diff --git a/.github/workflows/sync-vsphere.yml b/.github/workflows/sync.yml similarity index 50% rename from .github/workflows/sync-vsphere.yml rename to .github/workflows/sync.yml index 8a5c6403..4d3710f7 100644 --- a/.github/workflows/sync-vsphere.yml +++ b/.github/workflows/sync.yml @@ -1,24 +1,49 @@ -name: "Sync vSphere" +name: "Sync images" on: push: branches: - "main" + paths: + - '.github/workflows/dry.yml' + - '.github/workflows/sync.yml' + - 'modules/**' + - '!README.md' + - '!single_sync.sh' + - '!single_sync_v2.sh' + - 'single_sync_v3.sh' + schedule: - cron: "0 2 * * *" jobs: + fetch_modules_to_sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Set output + id: set_output + run: | + MODULES_TO_SYNC_JSON=$(find modules -type d -mindepth 1 -maxdepth 1 | cut -d/ -f2 | sort | jq -R | jq -cs . ) + echo "MODULES_TO_SYNC_JSON=${MODULES_TO_SYNC_JSON}" >> $GITHUB_OUTPUT + outputs: + modules_to_sync: ${{ steps.set_output.outputs.MODULES_TO_SYNC_JSON }} sync: runs-on: ubuntu-latest + needs: fetch_modules_to_sync + strategy: + fail-fast: false + matrix: + module: ${{ fromJson(needs.fetch_modules_to_sync.outputs.modules_to_sync) }} steps: - uses: actions/checkout@master - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Login to SIGHUP new Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: registry.sighup.io username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }} @@ -28,10 +53,18 @@ jobs: sudo apt-get update && sudo apt-get install -yqq wget sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.19.1/yq_linux_amd64 -O /usr/bin/yq sudo chmod +x /usr/bin/yq + - # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + platforms: linux/amd64,linux/arm64 - name: Iterate run: | yq --version docker run --rm quay.io/skopeo/stable:v1.13 --version docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }} docker.io --authfile=/login/auth.json docker run -v ./login:/login --rm quay.io/skopeo/stable:v1.13 login --username ${{ secrets.SIGHUP_REGISTRY_USERNAME }} --password ${{ secrets.SIGHUP_REGISTRY_PASSWORD }} registry.sighup.io --authfile=/login/auth.json - ./single_sync_v2.sh modules/vsphere/images.yml false + ./single_sync_v3.sh modules/${{ matrix.module }}/images.yml false diff --git a/.github/workflows/sync-trivy.yml b/.github/workflows/trivy-db-sync.yml similarity index 100% rename from .github/workflows/sync-trivy.yml rename to .github/workflows/trivy-db-sync.yml diff --git a/CVEs/Makefile b/CVEs/Makefile index 8edb4bc1..29cdaa8e 100644 --- a/CVEs/Makefile +++ b/CVEs/Makefile @@ -7,18 +7,19 @@ scan-vulns \ concat-multiple-kfd-images-list -KFD_VERSIONS := $(shell find . -type d -name "v*" -maxdepth 1 -mindepth 1 | cut -d/ -f2 | sort ) +KFD_VERSIONS := $(shell find . -maxdepth 1 -mindepth 1 -type d -name "v*" | cut -d/ -f2 | sort ) DRY_RUN := 1 PATCH_FILE_IMAGE_LIST_TO_PATCHING := all_kfd_images.txt +TRIVY_CACHE_DIR := /tmp/.cache/trivy all: trivy-download-db scan-pre-patch concat-multiple-kfd-images-list patch scan-post-patch scan-pre-patch: @for version in $(KFD_VERSIONS); do \ ( \ - $(MAKE) download-deps KFD_VERSION=$$version; \ - $(MAKE) kustomize-build-all KFD_VERSION=$$version; \ - $(MAKE) generate-image-list-from-manifests KFD_VERSION=$$version; \ + [ ! -e $$version/built.yaml ] && $(MAKE) download-deps KFD_VERSION=$$version; \ + [ ! -e $$version/built.yaml ] && $(MAKE) kustomize-build-all KFD_VERSION=$$version; \ + [ ! -e $$version/images.txt ] && $(MAKE) generate-image-list-from-manifests KFD_VERSION=$$version; \ $(MAKE) scan-vulns KFD_VERSION=$$version LIST_FILE=$$version/images.txt OUTPUT_FILE=$$version/FURY-CVEs.md; \ ) & \ done; \ @@ -32,7 +33,7 @@ concat-multiple-kfd-images-list: rm $(PATCH_FILE_IMAGE_LIST_TO_PATCHING).tmp patch: - DOCKER_CONFIG="${DOCKER_CONFIG}" DRY_RUN=$(DRY_RUN) ./patch_images_with_copacetic.sh -i "$(IMAGE_TO_PATCH)" -l "$(PATCH_FILE_IMAGE_LIST_TO_PATCHING)" -o "$(PATCH_REPORT_OUTPUT_FILE)" + DRY_RUN=$(DRY_RUN) TRIVY_CACHE_DIR=$(TRIVY_CACHE_DIR) ./patch_images_with_copacetic.sh -i "$(IMAGE_TO_PATCH)" -l "$(PATCH_FILE_IMAGE_LIST_TO_PATCHING)" -o "$(PATCH_REPORT_OUTPUT_FILE)" scan-post-patch: @for version in $(KFD_VERSIONS); do \ @@ -67,11 +68,11 @@ generate-image-list-from-manifests: sort --unique $(KFD_VERSION)/images.tmp.txt --output $(KFD_VERSION)/images.txt && rm $(KFD_VERSION)/images.tmp.txt trivy-download-db: - trivy image --download-db-only --db-repository registry.sighup.io/fury-secured/aquasecurity/trivy-db:2 --no-progress - trivy image --download-java-db-only --java-db-repository registry.sighup.io/fury-secured/aquasecurity/trivy-java-db:1 --no-progress + trivy image --download-db-only --db-repository registry.sighup.io/fury-secured/aquasecurity/trivy-db:2 --no-progress --cache-dir $(TRIVY_CACHE_DIR) + trivy image --download-java-db-only --java-db-repository registry.sighup.io/fury-secured/aquasecurity/trivy-java-db:1 --no-progress --cache-dir $(TRIVY_CACHE_DIR) scan-vulns: - ./scan_vuln.sh -v "$(KFD_VERSION)" -l "$(LIST_FILE)" -o "$(OUTPUT_FILE)"; + TRIVY_CACHE_DIR=$(TRIVY_CACHE_DIR) ./scan_vuln.sh -v "$(KFD_VERSION)" -l "$(LIST_FILE)" -o "$(OUTPUT_FILE)"; diff --git a/CVEs/logging.sh b/CVEs/logging.sh index 30059c7d..6f00ed52 100644 --- a/CVEs/logging.sh +++ b/CVEs/logging.sh @@ -10,7 +10,7 @@ YELLOW='\033[1;33m' WHITE='\033[1;37m' function info() { - echo -e ">>\t${CYAN}[INFO]${NC} $(date +"%Y-%m-%dT%H:%M:%S.%3NZ"): $*" + echo -e ">>\t[${CYAN}INFO${NC}] $(date +"%Y-%m-%dT%H:%M:%S.%3NZ"): $*" } function warn() { @@ -22,9 +22,9 @@ function error() { } function success() { - echo -e ">>\t${GREEN}SUCCESS${NC} $(date +"%Y-%m-%dT%H:%M:%S.%3NZ"): $*" + echo -e ">>\t[${GREEN}SUCCESS${NC}] $(date +"%Y-%m-%dT%H:%M:%S.%3NZ"): $*" } function fail() { - echo -e ">>\t${RED}FAIL${NC} $(date +"%Y-%m-%dT%H:%M:%S.%3NZ"): $*" && exit 1 + echo -e ">>\t[${RED}FAIL${NC}] $(date +"%Y-%m-%dT%H:%M:%S.%3NZ"): $*" && exit 1 } \ No newline at end of file diff --git a/CVEs/patch_images_with_copacetic.sh b/CVEs/patch_images_with_copacetic.sh index 6ce5d872..3c485a56 100755 --- a/CVEs/patch_images_with_copacetic.sh +++ b/CVEs/patch_images_with_copacetic.sh @@ -1,6 +1,7 @@ #!/bin/bash -source logging.sh +source $(dirname $0)/utils.sh +source $(dirname $0)/logging.sh IMAGE_TO_PATCH= FILE_WITH_IMAGES_LIST_TO_PATCH= @@ -32,20 +33,27 @@ shift $((OPTIND-1)) [[ -z ${FILE_WITH_IMAGES_LIST_TO_PATCH} ]] && FILE_WITH_IMAGES_LIST_TO_PATCH=images.txt [[ -z ${PATCH_REPORT_OUTPUT_FILE} ]] && PATCH_REPORT_OUTPUT_FILE=PATCHED.md +DEFAULT_PATCH_REPORT_OUTPUT_FILE="${PATCH_REPORT_OUTPUT_FILE}" + TRIVY_SCAN_OUTPUT_DIR=.patching/scan COPA_PATCH_OUTPUT_DIR=.patching/patch DOCKERFILE_OUTPUT_DIR=.patching/dockerfile LOG_OUTPUT_DIR=.patching/log PATCH_ERROR_OUTPUT_FILE="${LOG_OUTPUT_DIR}/patch-error.log" -if [ -z "$(docker ps -f name=buildkitd -q)" ] +if [ -z "$(docker ps -f name=buildkitd -q 2> /dev/null)" ] then info "Start buildkitd instance for COPA" if [ -n "${DOCKER_CONFIG}" ] then docker_config_extra_args="-v ${DOCKER_CONFIG}:/root/.docker" fi - docker run --detach --rm --privileged ${docker_config_extra_args} -p 127.0.0.1:8888:8888/tcp --name buildkitd --entrypoint buildkitd registry.sighup.io/fury-secured/moby/buildkit:v0.16.0 --addr tcp://0.0.0.0:8888 # --platform linux/amd64 + if ! docker run --detach --rm --privileged ${docker_config_extra_args} -p 127.0.0.1:8888:8888/tcp --name buildkitd --entrypoint buildkitd registry.sighup.io/fury-secured/moby/buildkit:v0.16.0 --addr tcp://0.0.0.0:8888 2> /dev/null # --platform linux/amd64 + then + fail "can't start buildkit" + fi +else + warn "buildkit is already running" fi mkdir -p "${TRIVY_SCAN_OUTPUT_DIR}" "${COPA_PATCH_OUTPUT_DIR}" "${DOCKERFILE_OUTPUT_DIR}" "${LOG_OUTPUT_DIR}" @@ -53,109 +61,191 @@ echo -n "" > "${PATCH_ERROR_OUTPUT_FILE}" REGISTRY_BASE_URL='registry.sighup.io/fury/' REGISTRY_SECURED_BASE_URL='registry.sighup.io/fury-secured/' +RETURN_ERROR=0 +PATCH_IMAGE_RETURN_ERROR=0 function patch_image() { local image="$1" - secured_image=${image//"${REGISTRY_BASE_URL}"/"${REGISTRY_SECURED_BASE_URL}"} image_to_patch="${image}" + image_to_patch_repo=$(echo ${image_to_patch} | cut -d: -f1) + image_to_patch_tag=$(echo ${image_to_patch} | cut -d: -f2) - DOCKER_PULL_OUTPUT_FILE="${LOG_OUTPUT_DIR}/${image_to_patch//[:\/]/_}-image-pull.log" - if ! docker pull "${image_to_patch}" > "${DOCKER_PULL_OUTPUT_FILE}" 2>&1 - then - error "Failed pull ${image_to_patch}" - cat "${DOCKER_PULL_OUTPUT_FILE}" - return 1 - fi + secured_image=${image//"${REGISTRY_BASE_URL}"/"${REGISTRY_SECURED_BASE_URL}"} + secured_image_repo=$(echo ${secured_image} | cut -d: -f1) + + ARCHITECTURES=$(get_architecture_and_digest ${image_to_patch} | jq -r '.[].architecture' ) + [[ -z "${ARCHITECTURES}" ]] && error "no architectures found for ${image_to_patch}" && PATCH_IMAGE_RETURN_ERROR=$((PATCH_IMAGE_RETURN_ERROR + 1 )) && return $PATCH_IMAGE_RETURN_ERROR + + MULTI_ARCH_IMAGES="" - TRIVY_SCAN_OUTPUT_FILE=${TRIVY_SCAN_OUTPUT_DIR}/${image_to_patch//[:\/]/_}.json - COPA_REPORT_OUTPUT_FILE=${COPA_PATCH_OUTPUT_DIR}/${image_to_patch//[:\/]/_}.vex.json - COPA_PATCHING_LOG_FILE=${COPA_PATCH_OUTPUT_DIR}/${image_to_patch//[:\/]/_}.log - info "Scan ${image_to_patch} for CVEs" - trivy image --skip-db-update --skip-java-db-update --scanners vuln -q --vuln-type os --ignore-unfixed -f json -o "${TRIVY_SCAN_OUTPUT_FILE}" "${image_to_patch}" # --platform=linux/amd64 - info "Clean trivy scan cache for ${image_to_patch}" - trivy clean --scan-cache - info "Patching CVEs for ${image_to_patch}" - copa patch -r "${TRIVY_SCAN_OUTPUT_FILE}" -i "${image_to_patch}" --format="openvex" --output "$COPA_REPORT_OUTPUT_FILE" -a tcp://127.0.0.1:8888 2>&1 | tee "${COPA_PATCHING_LOG_FILE}" - copa_exit_code=${PIPESTATUS[0]} - - if [ "${copa_exit_code}" -eq 0 ] + for ARCHITECTURE in ${ARCHITECTURES[@]} + do + + image_to_patch_with_digest="${image_to_patch_repo}@$(get_architecture_and_digest ${image_to_patch} | jq -r \ + --arg arch ${ARCHITECTURE} \ + '.[] | select(.architecture == $arch) | .digest ' \ + )" + if ! docker pull "${image_to_patch_with_digest}" --platform linux/${ARCHITECTURE} > /dev/null 2>&1 + then + error "Failed pull ${image_to_patch_with_digest} for linux/${ARCHITECTURE}" + PATCH_IMAGE_RETURN_ERROR=$((PATCH_IMAGE_RETURN_ERROR + 1)) + continue + fi + # Replace with skopeo/podman if exists a command that get imageId + image_to_patch_image_id=$(docker inspect "${image_to_patch_with_digest}" --format '{{.Id}}') + docker rmi -f ${image_to_patch_with_digest} > /dev/null + patched_tag="${image_to_patch_tag}-${ARCHITECTURE}-patched" + # Handling image with no tag + [ ${image_to_patch_tag} = ${image_to_patch} ] && patched_tag="latest-${ARCHITECTURE}-patched" + secured_image_with_tag_arch="${secured_image}-${ARCHITECTURE}" + + TRIVY_SCAN_OUTPUT_FILE=${TRIVY_SCAN_OUTPUT_DIR}/${image_to_patch//[:\/]/_}-${ARCHITECTURE}.json + COPA_REPORT_OUTPUT_FILE=${COPA_PATCH_OUTPUT_DIR}/${image_to_patch//[:\/]/_}-${ARCHITECTURE}.vex.json + COPA_PATCHING_LOG_FILE=${COPA_PATCH_OUTPUT_DIR}/${image_to_patch//[:\/]/_}-${ARCHITECTURE}.log + info "Looking for CVEs in ${image_to_patch} for linux/${ARCHITECTURE}" + trivy image --platform=linux/${ARCHITECTURE} \ + --skip-db-update --skip-java-db-update \ + --cache-dir ${TRIVY_CACHE_DIR:-/tmp/.cache/trivy} \ + --scanners vuln -q --vuln-type os --ignore-unfixed \ + -f json -o "${TRIVY_SCAN_OUTPUT_FILE}" \ + "${image_to_patch_with_digest}" + # info "Clean trivy scan cache" + # trivy clean --scan-cache + info "Patching CVEs in ${image_to_patch} for linux/${ARCHITECTURE}" + copa patch -r "${TRIVY_SCAN_OUTPUT_FILE}" -i "${image_to_patch_with_digest}" --tag ${patched_tag} --format="openvex" --output "$COPA_REPORT_OUTPUT_FILE" -a tcp://127.0.0.1:8888 2>&1 | tee "${COPA_PATCHING_LOG_FILE}" + copa_exit_code=${PIPESTATUS[0]} + + if [ "${copa_exit_code}" -eq 0 ] + then + image_patched="${image_to_patch_repo}:${patched_tag}" + PATCH_REPORT_OUTPUT_FILE="${DEFAULT_PATCH_REPORT_OUTPUT_FILE%.md}.${ARCHITECTURE}.md" + { + [[ -n ${IMAGE_TO_PATCH} ]] && printf "# %s\n\n" "${IMAGE_TO_PATCH}" + printf "Last updated %s\n\n" "$(date +'%Y-%m-%d')"; + printf "## Arch %s\n\n" ${ARCHITECTURE}; + printf "## CVEs patched\n\n" ; + echo "| Source Image | Arch | Source Image ID | CVE | Severity | Description | Patched Image| Patched Image ID |" + echo "| --- | --- | --- | --- | --- |--- | --- | --- |" + } > "${PATCH_REPORT_OUTPUT_FILE}" + + FIXED_CVES=$(jq '.statements[] | select(.status=="fixed") | .vulnerability."@id"' -r < "${COPA_REPORT_OUTPUT_FILE}" | sort -r ) + info "CVEs patched in ${ARCHITECTURE} ${image_patched}: ${FIXED_CVES//[$'\r\n']/ }" + DOCKER_LABELS= + image_patched_image_id=$(docker inspect "${image_patched}" --format '{{.Id}}') + info "${image_patched} image id: ${image_patched_image_id}" + info "Update patching report for ${ARCHITECTURE} ${image_to_patch}" + for FIXED_CVE in ${FIXED_CVES[@]} + do + DOCKER_LABELS="--label io.sighup.secured.${FIXED_CVE}=fixed ${DOCKER_LABELS}" + jq -r \ + --arg cve "${FIXED_CVE}" \ + --arg image_to_patch "${image_to_patch}" \ + --arg image_to_patch_image_id "${image_to_patch_image_id}" \ + --arg image_patched "${image_patched}" \ + --arg image_patched_image_id "${image_patched_image_id}" \ + --arg arch ${ARCHITECTURE} \ + '[try .Results[].Vulnerabilities[] | select(.VulnerabilityID==$cve)][0] | "|" + $image_to_patch + " | " + $arch + " | " + $image_to_patch_image_id + " | " + .VulnerabilityID + " | " + .Severity + " | " + .Title + " | " + $image_patched + "|" + $image_patched_image_id + "|"' < "$TRIVY_SCAN_OUTPUT_FILE">> "$PATCH_REPORT_OUTPUT_FILE" + done + info "Tag ${image_patched} as secured image: ${secured_image_with_tag_arch}" + echo "FROM ${image_patched}" | DOCKER_BUILDKIT=0 docker build \ + ${DOCKER_LABELS} \ + --label io.sighup.secured.image.arch="${ARCHITECTURE}" \ + --label io.sighup.secured.image.created="$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ")" \ + --label io.sighup.secured.image.from.imageId="${image_to_patch_image_id}" \ + -t "${secured_image_with_tag_arch}" \ + -f - "${DOCKERFILE_OUTPUT_DIR}" &> /dev/null + secured_image_labeled_image_id=$(docker inspect "${secured_image_with_tag_arch}" --format '{{.Id}}') + if [ ${DRY_RUN:-1} -eq 0 ] + then + secured_image_labeled_digest=$(skopeo_run "skopeo inspect docker-daemon:${secured_image_with_tag_arch}" | jq -r '.Digest') + secured_image_with_digest=${secured_image_repo}@${secured_image_labeled_digest} + info "Push secure image: ${secured_image_with_digest}" + skopeo_run "skopeo copy \ + --authfile=\$DOCKER_CONFIG/config.json \ + docker-daemon:${secured_image_with_tag_arch} \ + docker://${secured_image_with_digest}" + if [ $? -eq 0 ] + then + success "${secured_image_with_digest} pushed with image id: ${secured_image_labeled_image_id}" + else + PATCH_IMAGE_RETURN_ERROR=$((PATCH_IMAGE_RETURN_ERROR + 1)) + error "failed to push ${secured_image_with_digest} with image id: ${secured_image_labeled_image_id}" + fi + MULTI_ARCH_IMAGES="${secured_image_with_digest} ${MULTI_ARCH_IMAGES}" + fi + sed -i'.unsecured' s#"${image_patched}"#"${secured_image}"# "${PATCH_REPORT_OUTPUT_FILE}" + sed -i'.unsecured' s#"${image_patched_image_id}"#"${secured_image_labeled_image_id}"# "${PATCH_REPORT_OUTPUT_FILE}" + rm "${PATCH_REPORT_OUTPUT_FILE}.unsecured" + info "Cleanup ${image_patched}" + buildctl --addr tcp://127.0.0.1:8888 prune + docker rmi -f "${image_patched}" + info "cleanup ${secured_image_with_tag_arch}" + docker rmi -f "${secured_image_with_tag_arch}" + else + if [ "${image_to_patch}" != "${secured_image}" ] # if image need to be patched + then + copa_error="$(awk -F'Error: ' '$0 ~ /Error:/ {print $2}' ${COPA_PATCHING_LOG_FILE})" + echo "linux/${ARCHITECTURE} ${secured_image}: ${copa_error}" >> "${PATCH_ERROR_OUTPUT_FILE}" + # Ignore accepted errors + if ( + [ "${copa_error}" == "no patchable vulnerabilities found" ] || + [ "${copa_error}" == "no scanning results for os-pkgs found" ] + ) + then + warn "${copa_error} in ${image_to_patch} for linux/${ARCHITECTURE}" + if [ ${DRY_RUN:-1} -eq 0 ] + then + secured_image_with_digest=${image_to_patch_with_digest//${REGISTRY_BASE_URL}/${REGISTRY_SECURED_BASE_URL}} + info "copy ${image_to_patch_with_digest} to ${secured_image_with_digest}" + skopeo_run "skopeo copy \ + --authfile=\$DOCKER_CONFIG/config.json \ + docker://${image_to_patch_with_digest} \ + docker://${secured_image_with_digest}" + MULTI_ARCH_IMAGES="${image_to_patch_with_digest//${REGISTRY_BASE_URL}/${REGISTRY_SECURED_BASE_URL}} ${MULTI_ARCH_IMAGES}" + success "pushed ${secured_image_with_digest} with image id: ${image_to_patch_image_id}" + fi + else + error "patching ${image_to_patch} for linux/${ARCHITECTURE}: ${copa_error}" + PATCH_IMAGE_RETURN_ERROR=$((PATCH_IMAGE_RETURN_ERROR + 1)) + fi + else + warn "${image_to_patch} is the same of ${secured_image}" + fi + fi + done + + if [ ${DRY_RUN:-1} -eq 0 ] && [[ $(echo ${MULTI_ARCH_IMAGES} | wc -w) -eq $(echo ${ARCHITECTURES} | wc -w) ]] then - { - [[ -n ${IMAGE_TO_PATCH} ]] && printf "# %s\n\n" "${IMAGE_TO_PATCH}" - printf "Last updated %s\n\n" "$(date +'%Y-%m-%d')"; - printf "## CVEs patched\n\n" ; - echo "| Source Image | Source Image Hash |CVE | Severity | Description | Patched Image| Patched Image Hash |" - echo "| --- | --- | --- | --- |--- | --- | --- |" - } > "${PATCH_REPORT_OUTPUT_FILE}" - - FIXED_CVES=$(jq '.statements[] | select(.status=="fixed") | .vulnerability."@id"' -r < "${COPA_REPORT_OUTPUT_FILE}" | sort -r ) - info "CVEs patched in ${image_to_patch}-patched: ${FIXED_CVES//[$'\r\n']/ }" - DOCKER_LABELS= - image_patched_hash=$(docker inspect "${image_to_patch}-patched" --format '{{.Id}}') - info "${image_to_patch}-patched hash: ${image_patched_hash}" - info "Update patching report for ${image_to_patch}" - image_to_patch_hash=$(jq -r '.Metadata.ImageID' < "${TRIVY_SCAN_OUTPUT_FILE}") - for FIXED_CVE in ${FIXED_CVES[@]} - do - DOCKER_LABELS="--label io.sighup.secured.${FIXED_CVE}=fixed ${DOCKER_LABELS}" - jq -r \ - --arg cve "${FIXED_CVE}" \ - --arg image_to_patch "${image_to_patch}" \ - --arg image_to_patch_hash "${image_to_patch}_hash" \ - --arg image_patched "${image_to_patch}-patched" \ - --arg image_patched_hash "${image_patched_hash}" \ - '[try .Results[].Vulnerabilities[] | select(.VulnerabilityID==$cve)][0] | "|" + $image_to_patch + " | " + $image_to_patch_hash + " | " + .VulnerabilityID + " | " + .Severity + " | " + .Title + " | " + $image_patched + "|" + $image_patched_hash + "|"' < "$TRIVY_SCAN_OUTPUT_FILE">> "$PATCH_REPORT_OUTPUT_FILE" - done - info "Tag ${image_to_patch}-patched as secured image: ${secured_image}" - echo "FROM ${image_to_patch}-patched" | DOCKER_BUILDKIT=0 docker build \ - ${DOCKER_LABELS} \ - --label io.sighup.secured.image.created="$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ")" \ - --label io.sighup.secured.image.from.hash="${image_to_patch_hash}" \ - -t "${secured_image}" \ - -f - "${DOCKERFILE_OUTPUT_DIR}" &> /dev/null - secured_labeled_image_hash=$(docker inspect "${secured_image}" --format '{{.Id}}') - sed -i'.unsecured' s#"${image_to_patch}-patched"#"${secured_image}"# "${PATCH_REPORT_OUTPUT_FILE}" - sed -i'.unsecured' s#"${image_patched_hash}"#"${secured_labeled_image_hash}"# "${PATCH_REPORT_OUTPUT_FILE}" - rm "${PATCH_REPORT_OUTPUT_FILE}.unsecured" - info "Push secure image: ${secured_image}" - [[ ${DRY_RUN:-1} -eq 0 ]] && docker push "${secured_image}" - info "Cleanup ${image_to_patch}-patched" - buildctl --addr tcp://127.0.0.1:8888 prune - docker rmi -f "${image_to_patch}-patched" - info "cleanup ${secured_image}" - docker rmi -f "${secured_image}" - success "${secured_image} pushed with id: ${secured_labeled_image_hash}" - else - if [ "${image_to_patch}" != "${secured_image}" ] + info "Create and push manifest ${secured_image}" + if podman_run "podman manifest create ${secured_image} ${MULTI_ARCH_IMAGES} && podman manifest push ${secured_image}" then - warn "No CVEs patched in ${image_to_patch}" - warn "Tag ${image_to_patch} as secured image: ${secured_image}" - docker tag "${image_to_patch}" "${secured_image}" - [[ ${DRY_RUN:-1} -eq 0 ]] && info "Push secured image: ${secured_image}" && docker push "${secured_image}" - info "cleanup ${secured_image}" - docker rmi -f "${secured_image}" - info "Update patching error log" - echo "${secured_image}: $(awk -F'Error:' '$0 ~ /Error:/ {print $2}' "${COPA_PATCHING_LOG_FILE}")" >> "${PATCH_ERROR_OUTPUT_FILE}" + success "manifest ${secured_image} pushed" else - error "${image_to_patch} is the same of ${secured_image}" + error "failed pushing manifest ${secured_image}" + PATCH_IMAGE_RETURN_ERROR=$((PATCH_IMAGE_RETURN_ERROR + 1)) fi fi - info "Cleanup ${image_to_patch}" - docker rmi -f "${image_to_patch}" - echo "" echo "================================================================" echo "" + return $PATCH_IMAGE_RETURN_ERROR +} - return 0 +function patch_from_list(){ + while IFS= read -r image; do + patch_image "${image}" + RETURN_ERROR=$(($RETURN_ERROR + PATCH_IMAGE_RETURN_ERROR)) + PATCH_IMAGE_RETURN_ERROR=0 + done } if [ -n "${IMAGE_TO_PATCH}" ] then patch_image "${IMAGE_TO_PATCH}" + RETURN_ERROR=$(($RETURN_ERROR + PATCH_IMAGE_RETURN_ERROR)) else [[ ! -f "${FILE_WITH_IMAGES_LIST_TO_PATCH}" ]] && fail "Missing image list files" - while IFS= read -r image; do - patch_image "${image}" - done < "${FILE_WITH_IMAGES_LIST_TO_PATCH}" -fi \ No newline at end of file + patch_from_list < "${FILE_WITH_IMAGES_LIST_TO_PATCH}" +fi + +exit $RETURN_ERROR \ No newline at end of file diff --git a/CVEs/scan_vuln.sh b/CVEs/scan_vuln.sh index bafd1392..f66689f3 100755 --- a/CVEs/scan_vuln.sh +++ b/CVEs/scan_vuln.sh @@ -4,6 +4,9 @@ KFD_VERSION= IMAGE_LIST_FILE= SCAN_RESULT_OUTPUT_FILE= +source $(dirname $0)/utils.sh +source $(dirname $0)/logging.sh + function usage() { echo "Usage: $0 -v KFD_VERSION [-l IMAGE_LIST_FILE] [-o SCAN_RESULT_OUTPUT_FILE]" } @@ -27,8 +30,6 @@ while getopts ":v:l:o:" o; do done shift $((OPTIND-1)) -source logging.sh - [[ -z ${KFD_VERSION} ]] && fail "Missing KFD VERSION" [[ -z ${IMAGE_LIST_FILE} ]] && IMAGE_LIST_FILE="${KFD_VERSION}/images.txt" [[ -z ${SCAN_RESULT_OUTPUT_FILE} ]] && SCAN_RESULT_OUTPUT_FILE="${KFD_VERSION}/CVEs.md" @@ -46,26 +47,50 @@ echo "" > "${SCAN_ERROR_OUTPUT_FILE}" printf "## CVEs\n\n"; } > "${SCAN_RESULT_OUTPUT_FILE}" -echo "| Image | Hash| Severity | CVE | Reason | Package Affected | Status | Fixed in versions |" >> "${SCAN_RESULT_OUTPUT_FILE}" -echo "| --- | --- | --- | --- | --- | --- | --- | --- |" >> "${SCAN_RESULT_OUTPUT_FILE}" +echo "| Image | Arch | Image ID | Severity | CVE | Reason | Package Affected | Status | Fixed in versions |" >> "${SCAN_RESULT_OUTPUT_FILE}" +echo "| --- | --- | --- | --- | --- | --- | --- | --- | --- |" >> "${SCAN_RESULT_OUTPUT_FILE}" mkdir -p "${TRIVY_SCAN_OUTPUT_DIR}" -for image in $IMAGE_LIST; do - TRIVY_SCAN_OUTPUT_FILE="${TRIVY_SCAN_OUTPUT_DIR}/scan-${image//[:\/]/_}.json" +RETURN_ERROR=0 - info "Scan $image for CVEs" - if ! trivy image --skip-db-update --skip-java-db-update --scanners vuln --no-progress --output "$TRIVY_SCAN_OUTPUT_FILE" --format json --severity CRITICAL "$image" - then - error "trivy failed to scan $image" - echo "$image | ERROR PROCESSING! " >> "${SCAN_ERROR_OUTPUT_FILE}" - else - src_image_hash=$(jq -r '.Metadata.ImageID' < "$TRIVY_SCAN_OUTPUT_FILE") - jq -r --arg image "$image" \ - --arg src_image_hash "$src_image_hash" \ - 'try .Results[].Vulnerabilities[] | "| " + $image + " | " + $src_image_hash +" | " + .Severity + " | " + .VulnerabilityID + " | " + .Title + " | " + .PkgName + " " + .InstalledVersion + " | " + .Status + " | " + .FixedVersion + " |" ' < "$TRIVY_SCAN_OUTPUT_FILE" >> "${SCAN_RESULT_OUTPUT_FILE}" - fi - trivy clean --scan-cache +for image in $IMAGE_LIST; do + info "Looking for linux architectures available for ${image}" + ARCHITECTURES=$(get_architecture_and_digest ${image} | jq -r '.[].architecture' ) + ARCHITECTURES=${ARCHITECTURES//[$'\r\n']/ } + [ ${#ARCHITECTURES} -eq 0 ] && error "are sure that ${image} exist?" && RETURN_ERROR=$((RETURN_ERROR + 1)) && continue + info "${image} - linux architectures found: ${ARCHITECTURES//[$'\r\n']/ }" + for ARCHITECTURE in ${ARCHITECTURES[@]} + do + TRIVY_SCAN_OUTPUT_FILE="${TRIVY_SCAN_OUTPUT_DIR}/scan-${image//[:\/]/_}-${ARCHITECTURE}.json" + IMAGE_REPO=$(echo $image | cut -d: -f1) + IMAGE_DIGEST=$(get_architecture_and_digest ${image} | jq -r \ + --arg arch ${ARCHITECTURE} \ + '.[] | select(.architecture == $arch) | .digest ' \ + ) + info "Looking for CVEs in $image for linux/${ARCHITECTURE}" + if ! trivy image \ + --platform linux/${ARCHITECTURE} \ + --cache-dir ${TRIVY_CACHE_DIR:-/tmp/.cache/trivy} \ + --skip-db-update --skip-java-db-update \ + --scanners vuln --no-progress \ + --output "$TRIVY_SCAN_OUTPUT_FILE" \ + --format json --severity CRITICAL \ + "$IMAGE_REPO@$IMAGE_DIGEST" + then + error "trivy failed to scan $image for linux/${ARCHITECTURE}" + RETURN_ERROR=$((RETURN_ERROR + 1)) + echo "${ARCHITECTURE} $image | ERROR PROCESSING! " >> "${SCAN_ERROR_OUTPUT_FILE}" + else + src_image_id=$(jq -r '.Metadata.ImageID' < "$TRIVY_SCAN_OUTPUT_FILE") + jq -r --arg image "$image" \ + --arg src_image_id "$src_image_id" \ + --arg src_image_arch ${ARCHITECTURE} \ + 'try .Results[].Vulnerabilities[] | "| " + $image + " | " + $src_image_arch + " | " + $src_image_id + " | " + .Severity + " | " + .VulnerabilityID + " | " + .Title + " | " + .PkgName + " " + .InstalledVersion + " | " + .Status + " | " + .FixedVersion + " |" ' < "$TRIVY_SCAN_OUTPUT_FILE" >> "${SCAN_RESULT_OUTPUT_FILE}" + fi + # trivy clean --scan-cache + done done - rm -rf "${TRIVY_SCAN_OUTPUT_DIR}" + +exit $RETURN_ERROR \ No newline at end of file diff --git a/CVEs/utils.sh b/CVEs/utils.sh new file mode 100644 index 00000000..3caaf51a --- /dev/null +++ b/CVEs/utils.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +source $(dirname $0)/logging.sh + +if [ -z $DOCKER_CONFIG ] +then + warn "missing env var: DOCKER_CONFIG. Set default to $HOME/.docker" + export DOCKER_CONFIG=$HOME/.docker +fi + +function podman_run(){ + docker run --rm -v ${DOCKER_CONFIG}:${DOCKER_CONFIG} -e DOCKER_CONFIG="${DOCKER_CONFIG}" --privileged quay.io/podman/stable:v5.2.5 bash -c "$*" +} + +function skopeo_run(){ + docker run --rm -v ${DOCKER_CONFIG}:${DOCKER_CONFIG} -v /var/run/docker.sock:/var/run/docker.sock -e DOCKER_CONFIG="${DOCKER_CONFIG}" --entrypoint bash quay.io/skopeo/stable:v1.16 -c "$*" +} + +function get_architecture_and_digest(){ + local image=$1 + + # Fetch the manifest using Podman + MANIFESTS_AS_JSON=$(podman_run podman manifest inspect ${image} 2> /dev/null | jq -cr ' + if .mediaType == "application/vnd.docker.distribution.manifest.list.v2+json" then + .manifests | [ + .[] | + select(.platform.os == "linux") | + select(.platform.architecture == "amd64" or .platform.architecture == "arm64") | + { + "architecture": .platform.architecture, + "digest": .digest + } + ] + else + null + end + ') + + # If the manifest is null, fall back to Skopeo inspection + if [ "${MANIFESTS_AS_JSON}" = "null" ]; then + MANIFESTS_AS_JSON=$(skopeo_run skopeo inspect docker://${image} | jq -cr '[{"digest": .Digest, "architecture": "amd64"}]') + fi + + # Check if the result is empty or invalid + if [ -z "$MANIFESTS_AS_JSON" ] || [ "$MANIFESTS_AS_JSON" = "null" ]; then +# error "unable to retrieve architecture and digest information for image ${image}" + MANIFESTS_AS_JSON=[] + fi + + echo "${MANIFESTS_AS_JSON}" +} + diff --git a/README.md b/README.md index 39096440..0b386688 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ Features: - Configurable via YAML files - Build custom images - Skips images if the layers between src and dest are the same using `skopeo` -- Everything is executed with two bash scripts: `sync.sh` and `single_sync.sh` -- Execute the vulnerability detection and patching of the synced images +- Everything is executed with bash script `single_sync_v3.sh` that by default will sync all image architectures +- Execute the vulnerability detection and patching of synced images with amd64 and arm64 architectures ## How it works diff --git a/single_sync_v3.sh b/single_sync_v3.sh new file mode 100755 index 00000000..63222b27 --- /dev/null +++ b/single_sync_v3.sh @@ -0,0 +1,163 @@ +#!/bin/bash + +RC_ERROR_MISSING_DESTINATION=1 # Define exit code for missing destination + +# Check if second argument ($2) is set to "true", which enables dry mode +if [ $2 = true ]; then + DRY=true # Set DRY mode to true + echo " - DRY MODE: $DRY" # Inform the user that dry mode is active +else + DRY=false # DRY mode is off + echo " - DRY MODE: $DRY" # Inform the user that dry mode is inactive +fi + +# Get the number of images from the input YAML file ($1) +IMAGES=$(yq e '.images | length' $1) + +# Loop through each image in the YAML file +for (( c=0; c<${IMAGES}; c++ )) +do + # Extract various attributes for the image (name, multi-arch) + NAME=$(yq e '.images['"${c}"'].name' $1) + MULTI_ARCH=$(yq e '.images['"${c}"'].multi-arch' $1) + + # If MULTI_ARCH is null or empty, set it to true by default + if [ "$MULTI_ARCH" == "null" ] || [ -z "$MULTI_ARCH" ]; then + MULTI_ARCH="true" # Default to true if null or empty + fi + + # Extract source and context for the sync + SRC=$(yq e '.images['"${c}"'].source' $1) + CONTEXT=$(yq e '.images['"${c}"'].build.context' $1) + echo " - Start ${NAME}" # Notify the start of processing this image + + # Get the number of destinations and tags + DST=$(yq e '.images['"${c}"'].destinations | length' $1) + TAG=$(yq e '.images['"${c}"'].tag | length' $1) + + # Check if at least one destination is specified + if [ ${#DST} -lt 1 ]; then + echo " - ERROR: at least one destination is required" + exit $RC_ERROR_MISSING_DESTINATION # Exit with error code if no destination + fi + + # Loop through each tag + for (( t=0; t<${TAG}; t++ )) + do + # Initialize flags for layer differences (for both amd64 and arm64) + AMD64_DIFF=1 + ARM64_DIFF=1 + LOCAL_TAG=$(yq e '.images['"${c}"'].tag['"${t}"']' $1) + + # Check if the context is null and if dry mode is active + if [ "${CONTEXT}" = "null" ]; then + if [ ${DRY} = true ]; then + echo " - DRY MODE is active, skipping layers check" # In dry mode, skip layer diff check + else + # Perform layer diff check for AMD64 + LOCAL_LAYERS=$(docker run --rm quay.io/skopeo/stable:v1.13 inspect -n --override-os linux --override-arch amd64 docker://${SRC}:${LOCAL_TAG} 2> /dev/null | yq .Layers) + TARGET_LAYERS=$(docker run --rm quay.io/skopeo/stable:v1.13 inspect -n --override-os linux --override-arch amd64 docker://$(yq e '.images['"${c}"'].destinations[0]' $1):${LOCAL_TAG} 2> /dev/null | yq .Layers) + + diff <(echo ${LOCAL_LAYERS}) <(echo ${TARGET_LAYERS}) > /dev/null + AMD64_DIFF=$? # Store the exit code of diff (0 if no difference) + echo " - AMD64 diff exit code is: $AMD64_DIFF" + + # If multi-arch is true, also perform the diff check for ARM64 + ARM64_DIFF=$AMD64_DIFF + if [ ${MULTI_ARCH} = true ]; then + LOCAL_LAYERS=$(docker run --rm quay.io/skopeo/stable:v1.13 inspect -n --override-os linux --override-arch arm64 docker://${SRC}:${LOCAL_TAG} 2> /dev/null | yq .Layers) + TARGET_LAYERS=$(docker run --rm quay.io/skopeo/stable:v1.13 inspect -n --override-os linux --override-arch arm64 docker://$(yq e '.images['"${c}"'].destinations[0]' $1):${LOCAL_TAG} 2> /dev/null | yq .Layers) + + diff <(echo ${LOCAL_LAYERS}) <(echo ${TARGET_LAYERS}) > /dev/null + ARM64_DIFF=$? # Store the exit code for ARM64 diff + echo " - ARM64 diff exit code is: $ARM64_DIFF" + fi + fi + fi + + # If no layer differences and dry mode is off, skip syncing + if [ $AMD64_DIFF -eq 0 ] && [ $ARM64_DIFF -eq 0 ] && [ ${DRY} = false ] && [ ${CONTEXT} = "null" ]; then + echo " - Skipping ${SRC}:${LOCAL_TAG} as it is already synced" + else + # Handle dry mode and perform either skip or sync actions + if [ ${DRY} = true ] && [ ${CONTEXT} = "null" ]; then + echo " - DRY MODE is active, skipping pull for ${SRC}:${LOCAL_TAG}" # Skip pull in dry mode + elif [ ${DRY} = true ] && [ ${CONTEXT} != "null" ]; then + echo " - DRY MODE is active, skipping build for ${SRC}:${LOCAL_TAG}" # Skip build in dry mode + elif [ ${CONTEXT} = "null" ]; then + echo " - Layers are different, syncing ${SRC}:${LOCAL_TAG}" # Sync the image if layers are different + if [ ${MULTI_ARCH} = true ]; then + echo " - No prepull, sync using skopeo" # Multi-arch sync using skopeo + else + docker pull ${SRC}:${LOCAL_TAG} # Regular pull if not multi-arch + fi + else + # Handle build arguments if context is specified + BUILD_ARGS="" + ARG=$(yq e '.images['"${c}"'].build.args | length' $1) + for (( a=0; a<${ARG}; a++ )) + do + ARG_NAME=$(yq e '.images['"${c}"'].build.args['"${a}"'].name' $1) + ARG_VALUE=$(yq e '.images['"${c}"'].build.args['"${a}"'].value' $1) + BUILD_ARGS=$BUILD_ARGS" --build-arg "$ARG_NAME"="$ARG_VALUE + done + + # Perform multi-arch build if needed + if [ ${MULTI_ARCH} = true ]; then + TARGET_IMAGE=$(yq e '.images['"${c}"'].destinations[0]' $1):${LOCAL_TAG} + docker buildx build --push \ + --platform linux/amd64,linux/arm64 \ + $BUILD_ARGS --build-arg IMAGETAG=${LOCAL_TAG} \ + -t ${TARGET_IMAGE} $(dirname $1)/${CONTEXT} + else + docker build $BUILD_ARGS --build-arg IMAGETAG=${LOCAL_TAG} -t ${SRC}:${LOCAL_TAG} $(dirname $1)/${CONTEXT} + fi + fi + + # Loop through each destination and push the image + for (( d=0; d<${DST}; d++ )) + do + TO=$(yq e '.images['"${c}"'].destinations['"${d}"']' $1):${LOCAL_TAG} + + # Handle dry mode and skip tag/push + if [ ${DRY} = true ]; then + echo " - DRY MODE is active, skipping tag and push to ${TO}" + else + if [ ${MULTI_ARCH} = true ]; then + if [ ${CONTEXT} != "null" ]; then + if [ ${d} -eq 0 ]; then + continue # Skip the first destination in multi-arch case + else + SRC=$(yq e '.images['"${c}"'].destinations[0]' $1) # Use the first destination as source as it as already built previously + fi + fi + # Use skopeo for multi-arch sync + echo " - Syncing image from ${SRC}:${LOCAL_TAG} to ${TO}" + docker run -v ./login:/login \ + --rm \ + quay.io/skopeo/stable:v1.13 \ + copy --authfile=/login/auth.json --multi-arch all \ + docker://${SRC}:${LOCAL_TAG} docker://${TO} + else + # Regular tag and push for single-arch images + docker tag ${SRC}:${LOCAL_TAG} ${TO} + docker push ${TO} + docker rmi ${TO} + fi + fi + done + + # Handle image removal in dry mode and for multi-arch images + if [ ${DRY} = true ]; then + echo " - DRY MODE is active, skipping image rmi for ${SRC}:${LOCAL_TAG}" + else + if [ ${MULTI_ARCH} = true ]; then + echo "not removing images while using skopeo" # Skip image removal for multi-arch sync + else + docker rmi ${SRC}:${LOCAL_TAG} # Regular image removal if not multi-arch + fi + fi + fi + done + echo " - Finish ${NAME}" # Notify the end of processing for this image +done diff --git a/sync.sh b/sync.sh deleted file mode 100755 index 6ff03787..00000000 --- a/sync.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# if parameter --dry-run set DRY to true -if [ "$1" == "-d" ] || [ "$1" == "--dry-run" ]; then - DRY=true -else - DRY=false -fi - -# loop through all the images.yml files in the directory modules -for f in modules/*/images.yml -do - # if the file exists - if [ -f "$f" ] - then - # run the sync script - ./single_sync.sh $f $DRY - fi -done