docker: more cleanup and reorga

Author: mloubout

.github/workflows/docker-bases.yml

@@ -13,24 +13,39 @@ on:
       - '/docker/Dockerfile.intel'
   workflow_dispatch:
     inputs:
+      cpu:
+        type: boolean
+        default: false
+      nvidia:
+        type: boolean
+        default: false
+      amd:
+        type: boolean
+        default: false
+      intel:
+        type: boolean
+        default: false
+
       tags:
         description: 'Build compiler bases'
   schedule:
-    # Run once a week
+    # Run once a month
     - cron: "0 0 1 * *"
 
 jobs:
 #######################################################
 ############## Basic gcc CPU ##########################
 #######################################################
   deploy-cpu-bases:
+    if: inputs.cpu
     name: "cpu-base"
     runs-on: ubuntu-latest
     env:
       DOCKER_BUILDKIT: "1"
+
     strategy:
-        matrix:
-          version: ['', '14']
+      matrix:
+        gcc: ["", "14"]
 
     steps:
       - name: Checkout devito
@@ -60,14 +75,15 @@ jobs:
           context: .
           file: './docker/Dockerfile.cpu'
           push: true
-          build-args: 'gcc=${{ matrix.version }}'
-          tags: 'devitocodes/bases:cpu-gcc${{ matrix.version }}'
+          build-args: 'gcc=${{ matrix.gcc }}'
+          tags: 'devitocodes/bases:cpu-gcc${{ matrix.gcc }}'
 
 
 #######################################################
 ############## Intel OneApi CPU #######################
 #######################################################
   deploy-oneapi-bases:
+    if: inputs.intel
     name: "oneapi-base"
     runs-on: ubuntu-latest
     env:
@@ -129,6 +145,7 @@ jobs:
 ################### Nvidia nvhpc ######################
 #######################################################
   deploy-nvidia-bases:
+    if: inputs.nvidia
     name: "nvidia-bases"
     runs-on: ["self-hosted", "nvidiagpu"]
     env:
@@ -190,6 +207,7 @@ jobs:
 ##################### AMD #############################
 #######################################################
   deploy-amd-bases:
+    if: inputs.amd
     name: "amd-base"
     runs-on: ["self-hosted", "amdgpu"]
     env:

.github/workflows/tutorials.yml

@@ -70,7 +70,7 @@ jobs:
     - name: Build docker image
       if: "contains(matrix.name, 'docker')"
       run: |
-          docker build . --file docker/Dockerfile.devito --tag devito_img
+          docker build . --pull --file docker/Dockerfile.devito --tag devito_img
 
     - name: Set run prefix
       run: |

docker/Dockerfile.amd

@@ -5,7 +5,7 @@
 
 ARG ROCM_VERSION=6.3.2
 
-FROM rocm/dev-ubuntu-22.04:${ROCM_VERSION}-complete as sdk-base
+FROM rocm/dev-ubuntu-22.04:${ROCM_VERSION}-complete AS sdk-base
 
 ARG UCX_BRANCH="v1.16.0"
 ARG OMPI_BRANCH="v5.0.x"
@@ -14,16 +14,12 @@ ARG OMPI_BRANCH="v5.0.x"
 RUN rm /etc/apt/sources.list.d/* \
   && apt-get update \
   && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    ca-certificates git ssh make vim libtinfo* initramfs-tools libelf-dev \
-    numactl wget tmux build-essential autoconf automake libtool \
-    pkg-config libnuma* gfortran flex hwloc && \
-    apt-get clean
+    dh-autoreconf python3-venv python3-dev python3-pip git \
+    ca-certificates ssh make libtinfo* initramfs-tools libelf-dev \
+    wget build-essential autoconf automake libtool \
+    pkg-config libnuma* gfortran flex hwloc cmake
 
-# Requires cmake > 3.22 
-RUN mkdir -p /opt/cmake  \
-  && wget --no-check-certificate --quiet -O - https://cmake.org/files/v3.22/cmake-3.22.2-linux-x86_64.tar.gz | tar --strip-components=1 -xz -C /opt/cmake
-
-ENV ROCM_HOME=/opt/rocm \
+ENV ROCM_HOME=/opt/rocm\
     UCX_HOME=/opt/ucx \
     OMPI_HOME=/opt/ompi
 
@@ -34,9 +30,6 @@ ENV PATH=$ROCM_HOME/bin:$PATH \
 # Until rocm base has it fixed
 RUN ln -s /opt/rocm/llvm/bin/offload-arch /opt/rocm/bin/offload-arch | echo "offload-arch already exis"
 
-# Install tmpi
-RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi
-
 # Install UCX
 RUN cd /tmp/ \
     && git clone --depth 1 https://github.com/openucx/ucx.git -b ${UCX_BRANCH} \
@@ -101,14 +94,6 @@ ENV PATH=$OMPI_HOME/bin:$UCX_HOME/bin:$PATH \
 ENV OMPI_ALLOW_RUN_AS_ROOT=1
 ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
 
-##############################################################
-# Additional Devito requirements
-##############################################################
-
-# Additional utils and python3
-RUN apt-get update && \
-    apt-get install -y dh-autoreconf python3-venv python3-dev python3-pip
-
 RUN apt-get clean && apt-get autoclean && apt-get autoremove -y && \
     rm -rf /var/lib/apt/lists/*
 
@@ -119,7 +104,7 @@ CMD ["/bin/bash"]
 # AOMP for GPUs (OpenMP offloading)
 ########################################################################
 # This will only trigger if arch is aomp since the final stage depends on it
-FROM sdk-base as amdclang
+FROM sdk-base AS amdclang
 
 # MPI env
 ENV OMPI_CC="amdclang"
@@ -134,7 +119,7 @@ ENV DEVITO_LANGUAGE="openmp"
 # HIPCC for GPUs (HIP)
 ########################################################################
 # This will only trigger if arch is hip since the final stage depends on it
-FROM sdk-base as hip
+FROM sdk-base AS hip
 
 # MPI env
 ENV OMPI_CC="hipcc"

docker/Dockerfile.cpu

@@ -16,19 +16,18 @@ RUN apt-get update && \
     apt-get install -y software-properties-common dh-autoreconf python3-venv python3-dev python3-pip
 
 # Install for basic base not containing it
-RUN apt-get install -y vim wget git flex libnuma-dev tmux \
-        numactl hwloc curl \
+RUN apt-get install -y wget flex libnuma-dev hwloc curl cmake git \
         autoconf libtool build-essential procps software-properties-common
 
-# Install compilersx
+# Install gcc
 RUN if [ -n "$gcc" ]; then \
         apt-get install libgmp-dev libmpc-dev libmpfr-dev libisl-dev binutils texinfo -y && \
         cd /tmp && mkdir gcc && \
         git clone git://gcc.gnu.org/git/gcc.git gcc && \
         cd gcc && git checkout releases/gcc-${gcc} && \
         mkdir build && cd build && \
         ../configure --prefix=/opt/gcc-${gcc} --enable-languages=c,c++,fortran --disable-multilib \
-                     --enable-shared --enable-lto --enable-threads=posix --enable-libstdcxx-time=yes --enable-libgomp && \
+                    --enable-shared --enable-lto --enable-libstdcxx-time=yes --enable-libgomp && \
         make -j $(nproc) && make install-strip && \
         # Update alternatives
         update-alternatives --install /usr/bin/gcc gcc /opt/gcc-${gcc}/bin/gcc 60 && \
@@ -37,14 +36,6 @@ RUN if [ -n "$gcc" ]; then \
         rm -rf /tmp/gcc; \
     fi;
 
-# Install tmpi
-RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi
-
-# Install OpenGL library, necessary for the installation of GemPy
-RUN apt-get install -y libgl1-mesa-glx && \
-    apt-get clean && apt-get autoclean && apt-get autoremove  -y && \
-    rm -rf /var/lib/apt/lists/*
-
 ARG OMPI_BRANCH="v5.0.x"
 # Install OpenMPI
 RUN cd /tmp && mkdir openmpi && \
@@ -62,6 +53,9 @@ RUN cd /tmp && mkdir openmpi && \
 ENV PATH=${PATH}:/opt/openmpi/bin
 ENV LD_LIBRARY_PATH=/opt/openmpi/lib
 
+# Cleanup
+RUN apt-get clean && apt-get autoclean && apt-get autoremove  -y && rm -rf /var/lib/apt/lists/*
+
 EXPOSE 8888
 CMD ["/bin/bash"]
 

docker/Dockerfile.devito

@@ -5,21 +5,14 @@
 # Base image with compilers
 ARG base=devitocodes/bases:cpu-gcc
 
-FROM $base as builder
+FROM $base AS builder
 
 # User/Group Ids
 ARG USER_ID=1000
 ARG GROUP_ID=1000
 
 ##################  Install devito ############################################
 
-# Usefull utilities
-# Nvtop
-RUN git clone https://github.com/Syllo/nvtop.git /app/nvtop && \
-    mkdir -p /app/nvtop/build && cd /app/nvtop/build && \
-    cmake .. -DNVIDIA_SUPPORT=ON -DAMDGPU_SUPPORT=ON -DINTEL_SUPPORT=ON && \
-    make && make install
-
 # Install pip dependencies
 RUN python3 -m venv /venv && \
     /venv/bin/pip install --no-cache-dir --upgrade pip && \
@@ -41,17 +34,29 @@ RUN /venv/bin/pip install --no-cache-dir -e /app/devito[extras,tests] && rm -rf
 
 FROM $base AS utilities
 
+# tmpi
+RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi
+
 # Update if outdated and install extras
-RUN apt-get update && apt-get install -y git cmake \
+RUN apt-get update && apt-get install -y \
     libncurses5-dev libncursesw5-dev libdrm-dev libsystemd-dev \
-    python3-gdb gdb
+    python3-dbg gdb numactl tmux vim
+
+# Install OpenGL library, necessary for the installation of GemPy
+RUN apt-get install -y libgl1-mesa-glx
+
+# Nvtop
+RUN export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH && \
+    git clone https://github.com/Syllo/nvtop.git /app/nvtop && \
+    mkdir -p /app/nvtop/build && cd /app/nvtop/build && \
+    cmake .. -DNVIDIA_SUPPORT=ON -DAMDGPU_SUPPORT=ON -DINTEL_SUPPORT=ON && \
+    make && make install
 
-# Last installs (such as gdb needed in user mode) and cleanup
-RUN apt-get update && apt install gdb -y && \
-    apt-get clean && apt-get autoclean && apt-get autoremove -y && \
+# cleanup
+RUN apt-get clean && apt-get autoclean && apt-get autoremove -y && \
     rm -rf /var/lib/apt/lists/*
 
-FROM utilities as user
+FROM utilities AS user
 # COPY is much faster than RUN chown by order of magnitude so we have a final step that
 # just copies the built image into the user.
 
@@ -73,6 +78,7 @@ RUN groupadd -g ${GROUP_ID} app && \
     chown -R app:app $APP_HOME
 
 COPY --from=builder --chown=app:app /app /app
+COPY --from=utilities --chown=app:app /app/nvtop /app/nvtop
 
 ADD --chown=app:app docker/run-jupyter.sh /jupyter
 ADD --chown=app:app docker/run-tests.sh /tests

docker/Dockerfile.intel

@@ -13,23 +13,16 @@ RUN apt-get update && \
     apt-get install -y dh-autoreconf python3-venv python3-dev python3-pip
 
 # Install for basic base not containing it
-RUN apt-get install -y vim wget git flex libnuma-dev tmux \
-        numactl hwloc curl \
-        autoconf libtool build-essential procps
-
-# Install tmpi
-RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi
-
-# Install OpenGL library, necessary for the installation of GemPy
-RUN apt-get install -y libgl1-mesa-glx
+RUN apt-get install -y wget flex libnuma-dev hwloc curl cmake \
+        autoconf libtool build-essential procps git
 
 EXPOSE 8888
 CMD ["/bin/bash"]
 
 ##############################################################
 # Intel OneAPI standard image
 ##############################################################
-FROM base as oneapi
+FROM base AS oneapi
 
 # Download the key to system keyring
 # https://www.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top/installation/install-using-package-managers/apt.html#apt
@@ -67,7 +60,7 @@ RUN apt-get clean && apt-get autoclean && apt-get autoremove -y && rm -rf /var/l
 # ICC image
 # This is a legacy setup that is not built anymore but kept for reference
 ##############################################################
-FROM oneapi as icc
+FROM oneapi AS icc
 
 RUN apt-get update -y && apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-mpi-devel && \
     apt-get clean && apt-get autoclean && apt-get autoremove -y && \
@@ -85,7 +78,7 @@ ENV MPICC=mpiicc
 ##############################################################
 # ICX OpenMP image
 ##############################################################
-FROM oneapi as icx
+FROM oneapi AS icx
 
 RUN apt-get update -y && apt-get install -y intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi-devel && \
     apt-get clean && apt-get autoclean && apt-get autoremove -y && \
@@ -102,7 +95,7 @@ ENV MPICC=mpiicc
 ##############################################################
 # ICX SYCL CPU image
 ##############################################################
-FROM icx as cpu-sycl
+FROM icx AS cpu-sycl
 
 # Devito config
 ENV DEVITO_ARCH="sycl"

docker/Dockerfile.nvidia

@@ -7,25 +7,22 @@ ARG arch="nvc"
 ########################################################################
 # Build base image with apt setup and common env
 ########################################################################
-FROM ubuntu:22.04 as sdk-base
+FROM ubuntu:22.04 AS sdk-base
 
 SHELL ["/bin/bash", "-c"]
 
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install python
 RUN apt-get update && \
-    apt-get install -y -q gpg apt-utils curl wget vim libnuma-dev tmux numactl \
+    apt-get install -y -q gpg apt-utils curl wget libnuma-dev cmake git \
                           dh-autoreconf python3-venv python3-dev python3-pip
 
 # nodesource: nvdashboard requires nodejs>=10
 RUN curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | gpg --yes --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg
 RUN echo 'deb [trusted=yes, signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | tee /etc/apt/sources.list.d/nvhpc.list
 RUN apt-key update *&& apt-get update -y
 
-# Install tmpi
-RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi
-
 # Install nvhpc. `nvhpc` is the alias for the latest avaialble version
 ARG ver=nvhpc
 # We use the standard apt-get for the default latest nvhpc. For earlier version, apt has a bug that it will always
@@ -133,7 +130,7 @@ CMD ["/bin/bash"]
 ########################################################################
 # NVC for GPUs via OpenACC config
 ########################################################################
-FROM sdk-base as nvc
+FROM sdk-base AS nvc
 
 # Make devito env vars file and extras
 ADD docker/nvdashboard.json /app/nvdashboard.json
@@ -148,7 +145,7 @@ ENV DEVITO_LANGUAGE="openacc"
 ########################################################################
 # NVC for GPUs via CUDA config
 ########################################################################
-FROM nvc as nvcc
+FROM nvc AS nvcc
 
 ENV DEVITO_ARCH="cuda"
 ENV DEVITO_PLATFORM="nvidiaX"
@@ -157,7 +154,7 @@ ENV DEVITO_LANGUAGE="cuda"
 ########################################################################
 # NVC for CPUs config
 ########################################################################
-FROM nvc as nvc-host
+FROM nvc AS nvc-host
 
 ENV DEVITO_ARCH="nvc"
 ENV DEVITO_PLATFORM="cpu64"

requirements-testing.txt

@@ -5,4 +5,4 @@ codecov
 flake8>=2.1.0
 nbval
 scipy
-pooch; python_version >= "3.8"
+pooch