Setup build images for Pytorch & Tensorflow with base ROCm image

opendatahub-io · Jun 14, 2024 · 502eba2 · 502eba2
1 parent c7040f0
commit 502eba2
Show file tree

Hide file tree

Showing 4 changed files with 318 additions and 0 deletions.
diff --git a/jupyter/amd/pytorch/ubi9-python-3.9/Dockerfile b/jupyter/amd/pytorch/ubi9-python-3.9/Dockerfile
@@ -0,0 +1,102 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+LABEL name="odh-notebook-rocm-python-3.9" \
+      summary="ROCm Python 3.9 base image for ODH notebooks" \
+      description="ROCm Python 3.9 builder image based on CentOS Stream 9 for ODH notebooks" \
+      io.k8s.display-name="ROCm Python 3.9 base image for ODH notebooks" \
+      io.k8s.description="ROCm Python 3.9 builder image based on C9S for ODH notebooks" \
+      authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
+      io.openshift.build.commit.ref="main" \
+      io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/amd/python-3.9" \
+      io.openshift.build.image="quay.io/opendatahub/workbench-images:rocm-python-3.9"
+
+USER 0
+WORKDIR /opt/app-root/bin
+
+ARG ROCM_VERSION=6.1
+ARG AMDGPU_VERSION=6.1
+# Base
+RUN yum -y install git java-1.8.0-openjdk python; yum clean all
+# Install ROCm AMD from:
+# https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-centos-7-complete
+# Enable epel-release repositories
+RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
+    yum config-manager --set-enabled crb && \
+    yum clean all && rm -rf /var/cache/yum
+
+
+# Install required base build and packaging commands for ROCm
+RUN yum -y install \
+    bc \
+    bridge-utils \
+    glibc.i686 \
+    numactl-libs \
+    libssh \
+    libunwind-devel \
+    libunwind \
+    pciutils \
+    pciutils-libs \
+    re2c \
+    doxygen \
+    elfutils-libelf-devel \
+    expect \
+    numactl-devel \
+    pciutils-devel \
+    qemu-kvm \
+    subversion \
+    dkms \
+    dpkg \
+    dpkg-dev \
+    dpkg-perl && \
+    yum clean all 
+# These binaries are not available.
+# devscripts \
+# pth \
+# libcxx-devel \
+
+
+# Enable the epel repository for fakeroot
+# RUN yum --enablerepo=extras install -y fakeroot
+# RUN yum clean all
+RUN yum install -y fakeroot && \
+    yum clean all
+
+
+# On CentOS, install package centos-release-scl available in CentOS repository:
+# RUN yum install -y centos-release-scl
+
+# Install the devtoolset-7 collection:
+# RUN yum install -y devtoolset-7
+# RUN yum install -y devtoolset-7-libatomic-devel devtoolset-7-elfutils-libelf-devel
+
+# Install the ROCm rpms
+RUN echo "[ROCm]" > /etc/yum.repos.d/rocm.repo && \
+    echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo && \
+    echo "baseurl=https://repo.radeon.com/rocm/el9/$ROCM_VERSION/main" >> /etc/yum.repos.d/rocm.repo && \
+    echo "enabled=1" >> /etc/yum.repos.d/rocm.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/rocm.repo
+
+RUN echo "[amdgpu]" > /etc/yum.repos.d/amdgpu.repo && \
+    echo "name=amdgpu" >> /etc/yum.repos.d/amdgpu.repo && \
+    echo "baseurl=https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/el/9.2/main/x86_64" >> /etc/yum.repos.d/amdgpu.repo && \
+    echo "enabled=1" >> /etc/yum.repos.d/amdgpu.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/amdgpu.repo
+
+# Install rocm and amdgpu binaries 
+RUN yum install -y amdgpu-dkms rocm && \
+    yum clean all
+
+# Set ENV to enable devtoolset7 by default 
+# Set ENV to enable devtoolset7 by default 
+# ENV PATH=/opt/rh/devtoolset-7/root/usr/bin:/opt/rocm/hcc/bin:/opt/rocm/hip/bin:/opt/rocm/bin:/opt/rocm/hcc/bin:${PATH:+:${PATH}}
+# ENV MANPATH=/opt/rh/devtoolset-7/root/usr/share/man:${MANPATH}
+# ENV INFOPATH=/opt/rh/devtoolset-7/root/usr/share/info${INFOPATH:+:${INFOPATH}}
+# ENV PCP_DIR=/opt/rh/devtoolset-7/root
+# ENV PERL5LIB=/opt/rh/devtoolset-7/root//usr/lib64/perl5/vendor_perl:/opt/rh/devtoolset-7/root/usr/lib/perl5:/opt/rh/devtoolset-7/root//usr/share/perl5/
+# ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib:/opt/rh/devtoolset-7/root$rpmlibdir$rpmlibdir32${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+# ENV PYTHONPATH=/opt/rh/devtoolset-7/root/usr/lib64/python$pythonvers/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python$pythonvers/
+# ENV LDFLAGS="-Wl,-rpath=/opt/rh/devtoolset-7/root/usr/lib64 -Wl,-rpath=/opt/rh/devtoolset-7/root/usr/lib"
+
+# Restore notebook user workspace
+USER 1001
+WORKDIR /opt/app-root/src
diff --git a/jupyter/amd/pytorch/ubi9-python-3.9/Pipfile b/jupyter/amd/pytorch/ubi9-python-3.9/Pipfile
@@ -0,0 +1,53 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://download.pytorch.org/whl/nightly/rocm6.1/"
+verify_ssl = true
+name = "pytorch"
+
+[dev-packages]
+
+[packages]
+# PyTorch packages
+tensorboard = "~=2.16.2"
+torch = {version = "~=2.4.0", index = "pytorch"}
+torchvision = {version = "~=0.19.0", index = "pytorch"}
+# Datascience and useful extensions
+boto3 = "~=1.34.50"
+kafka-python = "~=2.0.2"
+matplotlib = "~=3.8.3"
+numpy = "~=1.26.4"
+pandas = "~=2.2.0"
+plotly = "~=5.20.0"
+scikit-learn = "~=1.4.0"
+scipy = "~=1.12.0"
+skl2onnx = "~=1.16.0"
+onnxconverter-common = "~=1.13.0" # Required for skl2onnx, as upgraded version is not compatible with protobuf
+codeflare-sdk = "~=0.15.1"
+# DB connectors
+pymongo = "~=4.6.2"
+psycopg = "~=3.1.18"
+pyodbc = "~=5.1.0"
+mysql-connector-python = "~=8.3.0"
+# JupyterLab packages
+odh-elyra = "~=3.16.5"
+jupyterlab = "~=3.6.7" # Wait on upgrade till plugins are ready
+jupyter-bokeh = "~=3.0.7" # Upgrade would bring in jupyterlab 4
+jupyter-server = "~=2.13.0"
+jupyter-server-proxy = "~=4.0.0" # Upgrade would bring in jupyterlab 4
+jupyter-server-terminals = "~=0.5.3"
+jupyterlab-git = "~=0.44.0"
+jupyterlab-lsp = "~=4.2.0"
+jupyterlab-widgets = "~=3.0.10"
+jupyter-resource-usage = "~=0.7.2"
+nbdime = "~=3.2.1"
+nbgitpuller = "~=1.2.0"
+# Base packages
+wheel = "~=0.43.0"
+setuptools = "~=69.2.0"
+
+[requires]
+python_version = "3.9"
diff --git a/jupyter/amd/tensorflow/ubi9-python-3.9/Dockerfile b/jupyter/amd/tensorflow/ubi9-python-3.9/Dockerfile
@@ -0,0 +1,102 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+LABEL name="odh-notebook-rocm-python-3.9" \
+      summary="ROCm Python 3.9 base image for ODH notebooks" \
+      description="ROCm Python 3.9 builder image based on CentOS Stream 9 for ODH notebooks" \
+      io.k8s.display-name="ROCm Python 3.9 base image for ODH notebooks" \
+      io.k8s.description="ROCm Python 3.9 builder image based on C9S for ODH notebooks" \
+      authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
+      io.openshift.build.commit.ref="main" \
+      io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/amd/python-3.9" \
+      io.openshift.build.image="quay.io/opendatahub/workbench-images:rocm-python-3.9"
+
+USER 0
+WORKDIR /opt/app-root/bin
+
+ARG ROCM_VERSION=6.1
+ARG AMDGPU_VERSION=6.1
+# Base
+RUN yum -y install git java-1.8.0-openjdk python; yum clean all
+# Install ROCm AMD from:
+# https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-centos-7-complete
+# Enable epel-release repositories
+RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
+    yum config-manager --set-enabled crb && \
+    yum clean all && rm -rf /var/cache/yum
+
+
+# Install required base build and packaging commands for ROCm
+RUN yum -y install \
+    bc \
+    bridge-utils \
+    glibc.i686 \
+    numactl-libs \
+    libssh \
+    libunwind-devel \
+    libunwind \
+    pciutils \
+    pciutils-libs \
+    re2c \
+    doxygen \
+    elfutils-libelf-devel \
+    expect \
+    numactl-devel \
+    pciutils-devel \
+    qemu-kvm \
+    subversion \
+    dkms \
+    dpkg \
+    dpkg-dev \
+    dpkg-perl && \
+    yum clean all 
+# These binaries are not available.
+# devscripts \
+# pth \
+# libcxx-devel \
+
+
+# Enable the epel repository for fakeroot
+# RUN yum --enablerepo=extras install -y fakeroot
+# RUN yum clean all
+RUN yum install -y fakeroot && \
+    yum clean all
+
+
+# On CentOS, install package centos-release-scl available in CentOS repository:
+# RUN yum install -y centos-release-scl
+
+# Install the devtoolset-7 collection:
+# RUN yum install -y devtoolset-7
+# RUN yum install -y devtoolset-7-libatomic-devel devtoolset-7-elfutils-libelf-devel
+
+# Install the ROCm rpms
+RUN echo "[ROCm]" > /etc/yum.repos.d/rocm.repo && \
+    echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo && \
+    echo "baseurl=https://repo.radeon.com/rocm/el9/$ROCM_VERSION/main" >> /etc/yum.repos.d/rocm.repo && \
+    echo "enabled=1" >> /etc/yum.repos.d/rocm.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/rocm.repo
+
+RUN echo "[amdgpu]" > /etc/yum.repos.d/amdgpu.repo && \
+    echo "name=amdgpu" >> /etc/yum.repos.d/amdgpu.repo && \
+    echo "baseurl=https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/el/9.2/main/x86_64" >> /etc/yum.repos.d/amdgpu.repo && \
+    echo "enabled=1" >> /etc/yum.repos.d/amdgpu.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/amdgpu.repo
+
+# Install rocm and amdgpu binaries 
+RUN yum install -y amdgpu-dkms rocm && \
+    yum clean all
+
+# Set ENV to enable devtoolset7 by default 
+# Set ENV to enable devtoolset7 by default 
+# ENV PATH=/opt/rh/devtoolset-7/root/usr/bin:/opt/rocm/hcc/bin:/opt/rocm/hip/bin:/opt/rocm/bin:/opt/rocm/hcc/bin:${PATH:+:${PATH}}
+# ENV MANPATH=/opt/rh/devtoolset-7/root/usr/share/man:${MANPATH}
+# ENV INFOPATH=/opt/rh/devtoolset-7/root/usr/share/info${INFOPATH:+:${INFOPATH}}
+# ENV PCP_DIR=/opt/rh/devtoolset-7/root
+# ENV PERL5LIB=/opt/rh/devtoolset-7/root//usr/lib64/perl5/vendor_perl:/opt/rh/devtoolset-7/root/usr/lib/perl5:/opt/rh/devtoolset-7/root//usr/share/perl5/
+# ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib:/opt/rh/devtoolset-7/root$rpmlibdir$rpmlibdir32${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+# ENV PYTHONPATH=/opt/rh/devtoolset-7/root/usr/lib64/python$pythonvers/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python$pythonvers/
+# ENV LDFLAGS="-Wl,-rpath=/opt/rh/devtoolset-7/root/usr/lib64 -Wl,-rpath=/opt/rh/devtoolset-7/root/usr/lib"
+
+# Restore notebook user workspace
+USER 1001
+WORKDIR /opt/app-root/src
diff --git a/jupyter/amd/tensorflow/ubi9-python-3.9/Pipfile b/jupyter/amd/tensorflow/ubi9-python-3.9/Pipfile
@@ -0,0 +1,61 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://repo.radeon.com/rocm/manylinux/rocm-rel-6.1/"
+verify_ssl = true
+name = "tensorflow"
+
+[dev-packages]
+
+[packages]
+# TensorFlow packages
+tensorflow = {version = "~=2.15.0", index = "tensorflow"}
+tensorboard = {version = "~=2.14.1", index = "tensorflow"}
+tf2onnx = "~= 1.16.1"
+
+
+# Datascience and useful extensions
+boto3 = "~=1.34.50"
+kafka-python = "~=2.0.2"
+matplotlib = "~=3.8.3"
+numpy = "~=1.26.4"
+pandas = "~=2.2.0"
+plotly = "~=5.20.0"
+scikit-learn = "~=1.4.0"
+scipy = "~=1.12.0"
+skl2onnx = "~=1.16.0"
+onnxconverter-common = "~=1.13.0" # Required for skl2onnx, as upgraded version is not compatible with protobuf
+codeflare-sdk = "~=0.15.0"
+
+# DB connectors
+pymongo = "~=4.6.2"
+psycopg = "~=3.1.18"
+pyodbc = "~=5.1.0"
+mysql-connector-python = "~=8.3.0"
+
+# JupyterLab packages
+odh-elyra = "~=3.16.5"
+kfp = "~=2.5.0"
+kfp-kubernetes = "~=1.0.0"
+
+jupyterlab = "~=3.6.7" # Wait on upgrade till plugins are ready
+jupyter-bokeh = "~=3.0.7" # Upgrade would bring in jupyterlab 4
+jupyter-server = "~=2.13.0"
+jupyter-server-proxy = "~=4.0.0" # Upgrade would bring in jupyterlab 4
+jupyter-server-terminals = "~=0.5.3"
+jupyterlab-git = "~=0.44.0"
+jupyterlab-lsp = "~=4.2.0"
+jupyterlab-widgets = "~=3.0.10"
+jupyter-resource-usage = "~=0.7.2"
+nbdime = "~=3.2.1"
+nbgitpuller = "~=1.2.0"
+
+# Base packages
+wheel = "~=0.43.0"
+setuptools = "~=69.2.0"
+
+[requires]
+python_version = "3.9"