Upgrade to OE 0.19.0-rc1 (#5165)

.azure-pipelines-gh-pages.yml

@@ -7,7 +7,7 @@ trigger:
 
 jobs:
   - job: build_and_publish_docs
-    container: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-virtual
+    container: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-virtual
     pool:
       vmImage: ubuntu-20.04
 

.azure-pipelines-quictls.yml

@@ -17,7 +17,7 @@ parameters:
 
 jobs:
   - job: build_quictls
-    container: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-virtual
+    container: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-virtual
     pool: 1es-dv4-focal
 
     strategy:

.azure-pipelines.yml

@@ -29,15 +29,15 @@ schedules:
 resources:
   containers:
     - container: virtual
-      image: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-virtual-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-virtual-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: snp
-      image: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-snp-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-snp-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: sgx
-      image: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-sgx
+      image: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-sgx
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro
 
 variables:

.daily.yml

@@ -25,19 +25,19 @@ schedules:
 resources:
   containers:
     - container: virtual
-      image: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-virtual
+      image: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-virtual
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
 
     - container: virtualclang15
-      image: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-virtual-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-virtual-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
 
     - container: snp
-      image: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-snp-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-snp-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: sgx
-      image: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-sgx
+      image: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-sgx
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx
 
 jobs:

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "CCF Development Environment",
-  "image": "ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-virtual",
+  "image": "ccfmsrc.azurecr.io/ccf/ci:2023-04-17-virtual",
   "runArgs": [],
   "extensions": [
     "eamodio.gitlens",

.github/workflows/build-ci-container.yml

@@ -45,10 +45,10 @@ jobs:
         run: docker login -u $ACR_TOKEN_NAME -p ${{ secrets.ACR_CI_PUSH_TOKEN_PASSWORD }} $ACR_REGISTRY
 
       - name: Pull CI container
-        run: docker pull $ACR_REGISTRY/ccf/ci:2023-03-13-2-snp-clang15
+        run: docker pull $ACR_REGISTRY/ccf/ci:2023-04-17-snp-clang15
 
       - name: Build CCF CI SNP container
-        run: docker build -f docker/ccf_ci_built . --build-arg="base=ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-snp-clang15" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD`
+        run: docker build -f docker/ccf_ci_built . --build-arg="base=ccfmsrc.azurecr.io/ccf/ci:2023-04-17-snp-clang15" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD`
 
       - name: Push CI container
         run: docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD`

.github/workflows/ci-checks.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   checks:
     runs-on: ubuntu-latest
-    container: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-virtual
+    container: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-virtual
 
     steps:
       - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

.github/workflows/codeql-analysis.yml

@@ -64,7 +64,7 @@ jobs:
       - run: |
           mkdir build
           cd build
-          cmake -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DLVI_MITIGATIONS=OFF -DCMAKE_C_COMPILER=`which clang-10` -DCMAKE_CXX_COMPILER=`which clang++-10` ..
+          cmake -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DLVI_MITIGATIONS=OFF -DCMAKE_C_COMPILER=`which clang-11` -DCMAKE_CXX_COMPILER=`which clang++-11` ..
 
       - run: |
           cd build

.github/workflows/containers.yml

@@ -18,16 +18,16 @@ jobs:
         platform: [sgx, virtual, snp]
         type: [dev, run]
         run_js: [true, ""]
-        clang_version: ["10", "15"]
+        clang_version: ["11", "15"]
         exclude:
           - type: dev
             run_js: true
           - platform: sgx
             clang_version: "15"
           - platform: virtual
-            clang_version: "10"
+            clang_version: "11"
           - platform: snp
-            clang_version: "10"
+            clang_version: "11"
 
     steps:
       - uses: actions/checkout@v3

.multi-thread.yml

@@ -16,7 +16,7 @@ pr:
 resources:
   containers:
     - container: virtual
-      image: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-virtual-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-virtual-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
 jobs:

.stress.yml

@@ -20,7 +20,7 @@ schedules:
 resources:
   containers:
     - container: sgx
-      image: ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-sgx
+      image: ccfmsrc.azurecr.io/ccf/ci:2023-04-17-sgx
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx
 
 jobs:

CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 [4.0.0-rc2]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.0-rc2
 
+### Changed
+
+- Updated Open Enclave to 0.19.0 (#5165).
+- Updated snmalloc to 0.6.0. This may result in a slight increase in the reported memory usage (~2MB), with improved latency for small memory allocations, especially in multi-threaded scenarios (#5165).
+- Update to `clang-11` for SGX builds (#5165).
+
 ### Removed
 
 - Support for HTTP request signing has been removed (#5137). Governance requests must use COSE Sign1 signing instead, see [documentation](https://microsoft.github.io/CCF/main/use_apps/issue_commands.html#cose-sign1) for details.

cmake/ccf_app.cmake

@@ -22,12 +22,7 @@ message(STATUS "Compile target platform: ${COMPILE_TARGET}")
 include(${CCF_DIR}/cmake/open_enclave.cmake)
 
 list(APPEND COMPILE_LIBCXX -stdlib=libc++)
-if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9)
-  list(APPEND LINK_LIBCXX -lc++ -lc++abi -stdlib=libc++)
-else()
-  # Clang <9 needs to link libc++fs when using <filesystem>
-  list(APPEND LINK_LIBCXX -lc++ -lc++abi -lc++fs -stdlib=libc++)
-endif()
+list(APPEND LINK_LIBCXX -lc++ -lc++abi -stdlib=libc++)
 
 # Sign a built enclave library with oesign
 function(sign_app_library name app_oe_conf_path enclave_sign_key_path)

cmake/cpack_settings.cmake

@@ -21,10 +21,10 @@ message(STATUS "Debian package version: ${CPACK_DEBIAN_PACKAGE_VERSION}")
 set(CCF_DEB_BASE_DEPENDENCIES "libuv1 (>= 1.34.2);openssl (>=1.1.1)")
 set(CCF_DEB_DEPENDENCIES ${CCF_DEB_BASE_DEPENDENCIES})
 
-set(OE_VERSION "0.18.5")
+set(OE_VERSION "0.19.0~rc1")
 if(COMPILE_TARGET STREQUAL "sgx")
   list(APPEND CCF_DEB_DEPENDENCIES
-       "libc++1-10;libc++abi1-10;open-enclave (>=${OE_VERSION})"
+       "libc++1-11;libc++abi1-11;open-enclave (>=${OE_VERSION})"
   )
 else()
   list(

cmake/open_enclave.cmake

@@ -6,7 +6,7 @@ if(NOT COMPILE_TARGET STREQUAL "sgx")
 endif()
 
 # Find OpenEnclave package
-find_package(OpenEnclave 0.18.5 CONFIG REQUIRED)
+find_package(OpenEnclave 0.19.0 CONFIG REQUIRED)
 # As well as pulling in openenclave:: targets, this sets variables which can be
 # used for our edge cases (eg - for virtual libraries). These do not follow the
 # standard naming patterns, for example use OE_INCLUDEDIR rather than
@@ -25,31 +25,14 @@ if(COMPILE_TARGET STREQUAL "sgx")
   )
 
   option(LVI_MITIGATIONS "Enable LVI mitigations" ON)
-  if(LVI_MITIGATIONS)
-    string(APPEND OE_TARGET_LIBC -lvi-cfg)
-    list(TRANSFORM OE_TARGET_ENCLAVE_AND_STD APPEND -lvi-cfg)
-    list(TRANSFORM OE_TARGET_ENCLAVE_CORE_LIBS APPEND -lvi-cfg)
-  endif()
 
   function(add_lvi_mitigations name)
     if(LVI_MITIGATIONS)
-      apply_lvi_mitigation(${name})
-      # Necessary to make sure Spectre mitigations are applied until
-      # https://github.com/openenclave/openenclave/issues/4641 is fixed
-      target_link_libraries(${name} PRIVATE openenclave::oecore)
+      # Enable clang-11 built-in LVI mitigation
+      target_compile_options(${name} PRIVATE -mlvi-cfi)
     endif()
   endfunction()
 
-  if(LVI_MITIGATIONS)
-    set(LVI_MITIGATION_BINDIR
-        /opt/oe_lvi
-        CACHE STRING "Path to the LVI mitigation bindir."
-    )
-    find_package(
-      OpenEnclave-LVI-Mitigation CONFIG REQUIRED HINTS ${OpenEnclave_DIR}
-    )
-  endif()
-
   set(OE_HOST_LIBRARY openenclave::oehost)
 else()
   set(OE_HOST_LIBRARY openenclave::oehostverify)

cmake/preproject.cmake

@@ -3,20 +3,20 @@
 
 # Note: this needs to be done before project(), otherwise CMAKE_*_COMPILER is
 # already set by CMake. If the user has not expressed any choice, we attempt to
-# default to Clang >= 10 If they have expressed even a partial choice, the usual
-# CMake selection logic applies. If we cannot find both a suitable clang and a
-# suitable clang++, the usual CMake selection logic applies
+# default to Clang >= 11. If they have expressed even a partial choice, the
+# usual CMake selection logic applies. If we cannot find both a suitable clang
+# and a suitable clang++, the usual CMake selection logic applies
 if((NOT CMAKE_C_COMPILER)
    AND (NOT CMAKE_CXX_COMPILER)
    AND "$ENV{CC}" STREQUAL ""
    AND "$ENV{CXX}" STREQUAL ""
 )
-  find_program(FOUND_CMAKE_C_COMPILER NAMES clang-15 clang-10)
-  find_program(FOUND_CMAKE_CXX_COMPILER NAMES clang++-15 clang++-10)
+  find_program(FOUND_CMAKE_C_COMPILER NAMES clang-15 clang-11)
+  find_program(FOUND_CMAKE_CXX_COMPILER NAMES clang++-15 clang++-11)
   if(NOT (FOUND_CMAKE_C_COMPILER AND FOUND_CMAKE_CXX_COMPILER))
     message(
       WARNING
-        "Clang >= 10 not found, will use default compiler. "
+        "Clang 11 or Clang 15 not found, will use default compiler. "
         "Override the compiler by setting CC and CXX environment variables."
     )
   else()
@@ -28,8 +28,8 @@ if((NOT CMAKE_C_COMPILER)
 endif()
 
 if(CMAKE_C_COMPILER_ID MATCHES "Clang")
-  if(CMAKE_C_COMPILER_VERSION VERSION_LESS 10)
-    message(WARNING "CCF officially supports Clang >= 10 only, "
+  if(CMAKE_C_COMPILER_VERSION VERSION_LESS 11)
+    message(WARNING "CCF officially supports Clang >= 11 only, "
                     "but your Clang version (${CMAKE_C_COMPILER_VERSION}) "
                     "is older than that. Build problems may occur."
     )

doc/build_apps/install_bin.rst

@@ -21,7 +21,7 @@ The dependencies required to build and run CCF apps can be conveniently installe
     .. code-block:: bash
 
         $ cd <ccf_path>/getting_started/setup_vm/
-        $ ./run.sh app-dev.yml --extra-vars "platform=sgx" --extra-vars "clang_version=10"
+        $ ./run.sh app-dev.yml --extra-vars "platform=sgx" --extra-vars "clang_version=11"
 
 .. tab:: SNP
 

doc/operations/run_setup.rst

@@ -13,7 +13,7 @@ Then, to quickly set up the dependencies necessary to start CCF applications, si
     .. code-block:: bash
 
         $ cd /opt/ccf_sgx/getting_started/setup_vm
-        $ ./run.sh app-run.yml --extra-vars "platform=sgx" --extra-vars "clang_version=10"
+        $ ./run.sh app-run.yml --extra-vars "platform=sgx" --extra-vars "clang_version=11"
 
 .. tab:: SNP
 

docker/app_dev

@@ -21,7 +21,7 @@ FROM base-${platform} AS final
 
 ARG platform=sgx
 ARG ansible_vars
-ARG clang_version=10
+ARG clang_version=11
 
 RUN echo "APT::Acquire::Retries \"5\";" | tee /etc/apt/apt.conf.d/80-retries
 

docker/app_run

@@ -21,7 +21,7 @@ FROM base-${platform} AS final
 
 ARG platform=sgx
 ARG ansible_vars
-ARG clang_version=10
+ARG clang_version=11
 
 RUN echo "APT::Acquire::Retries \"5\";" | tee /etc/apt/apt.conf.d/80-retries
 

docker/ccf_ci

@@ -20,7 +20,7 @@ FROM ubuntu:20.04 AS base-virtual
 FROM base-${platform} AS final
 
 ARG platform=sgx
-ARG clang_version=10
+ARG clang_version=11
 ARG ansible_vars
 
 RUN echo "APT::Acquire::Retries \"5\";" | tee /etc/apt/apt.conf.d/80-retries

docker/ccf_ci_built

@@ -4,7 +4,7 @@
 
 # Latest image as of this change
 ARG platform=sgx
-ARG base=ccfmsrc.azurecr.io/ccf/ci:2023-03-13-2-snp-clang-15
+ARG base=ccfmsrc.azurecr.io/ccf/ci:2023-04-17-snp-clang-15
 FROM ${base}
 
 # SSH. Note that this could (should) be done in the base ccf_ci image instead

getting_started/setup_vm/app-dev.yml

@@ -2,7 +2,7 @@
   vars:
     run_only: false
     platform: "sgx"
-    clang_version: "10"
+    clang_version: "11"
   tasks:
     - import_role:
         name: llvm_repo

getting_started/setup_vm/app-run.yml

@@ -2,7 +2,7 @@
   vars:
     run_only: true
     platform: "sgx"
-    clang_version: "10"
+    clang_version: "11"
   tasks:
     - import_role:
         name: llvm_repo

getting_started/setup_vm/ccf-dev.yml

@@ -1,7 +1,7 @@
 - hosts: localhost
   vars:
     platform: "sgx"
-    clang_version: "10"
+    clang_version: "11"
   tasks:
     - import_role:
         name: llvm_repo

getting_started/setup_vm/roles/ccf_build/vars/clang10.yml → getting_started/setup_vm/roles/ccf_build/vars/clang11.yml

@@ -1,5 +1,5 @@
 workspace: "/tmp/"
-clang_ver: 10
+clang_ver: 11
 
 debs:
   - apt-transport-https
@@ -11,7 +11,7 @@ debs:
   - python3.8-venv
   - llvm-{{ clang_ver }}
   - clang-{{ clang_ver }}
-  - clang-format-{{ clang_ver }}
+  - clang-format-10 # On purpose, to avoid formatting conflicts
   - clang-tools-{{ clang_ver }}
   - build-essential
   - expect

getting_started/setup_vm/roles/ccf_run/vars/clang10.yml → getting_started/setup_vm/roles/ccf_run/vars/clang11.yml

@@ -1,5 +1,5 @@
 workspace: "/tmp/"
 debs:
-  - libc++abi1-10
-  - libc++1-10
+  - libc++abi1-11
+  - libc++1-11
   - libuv1