88* https://learn.microsoft.com/en-us/azure/machine-learning/reference-migrate-sdk-v1-mlflow-tracking?view=azureml-api-2&tabs=aml%2Ccli%2Cmlflow
99"""
1010import pathlib
11- import tempfile
11+ import re
1212
1313from azure .ai .ml import MLClient , command
1414from azure .ai .ml .entities import AmlCompute , BuildContext , Environment , Workspace
1515from azure .ai .ml .exceptions import JobException
1616from azure .core .exceptions import ResourceExistsError
1717from azure .identity import DefaultAzureCredential
1818
19+
1920def get_client (subscription_id , resource_group , workspace_name ):
2021 """
2122 Get the client with specified AzureML workspace, or create one if not existing.
@@ -61,9 +62,8 @@ def get_or_create_environment(
6162 environment_name ,
6263 use_gpu ,
6364 use_spark ,
64- conda_pkg_jdk ,
65+ conda_openjdk_version ,
6566 python_version ,
66- commit_sha ,
6767):
6868 """
6969 AzureML requires the run environment to be setup prior to submission.
@@ -77,81 +77,39 @@ def get_or_create_environment(
7777 added to the conda environment, else False
7878 use_spark (bool): True if PySpark packages should be
7979 added to the conda environment, else False
80- conda_pkg_jdk (str): "openjdk=8" by default
81- python_version (str): python version, such as "3.9"
82- commit_sha (str): the commit that triggers the workflow
80+ conda_openjdk_version (str): "21" by default
81+ python_version (str): python version, such as "3.11"
8382 """
84- conda_env_name = "reco"
85- conda_env_yml = "environment.yml"
86- condafile = fr"""
87- name: { conda_env_name }
88- channels:
89- - conda-forge
90- dependencies:
91- - python={ python_version }
92- - { conda_pkg_jdk }
93- - pip
94- - pip:
95- - recommenders[dev{ ",gpu" if use_gpu else "" } { ",spark" if use_spark else "" } { commit_sha }
96- """
97- # See https://github.com/Azure/AzureML-Containers/blob/master/base/cpu/openmpi4.1.0-ubuntu22.04
98- image = "mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04"
99- # See https://github.com/Azure/AzureML-Containers/blob/master/base/gpu/openmpi4.1.0-cuda11.8-cudnn8-ubuntu22.04
100- dockerfile = fr"""# syntax=docker/dockerfile:1
101- FROM nvcr.io/nvidia/cuda:12.5.1-devel-ubuntu22.04
102- SHELL ["/bin/bash", "-c"]
103- USER root:root
104- ENV NVIDIA_VISIBLE_DEVICES all
105- ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
106- ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
107- ENV DEBIAN_FRONTEND noninteractive
108- RUN apt-get update && \
109- apt-get install -y wget git-all && \
110- apt-get clean -y && \
111- rm -rf /var/lib/apt/lists/*
112-
113- # Install Conda
114- ENV CONDA_PREFIX /opt/miniconda
115- RUN wget -qO /tmp/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py311_24.5.0-0-Linux-x86_64.sh && \
116- bash /tmp/miniconda.sh -bf -p ${{CONDA_PREFIX}} && \
117- ${{CONDA_PREFIX}}/bin/conda update --all -c conda-forge -y && \
118- ${{CONDA_PREFIX}}/bin/conda clean -ay && \
119- rm -rf ${{CONDA_PREFIX}}/pkgs && \
120- rm /tmp/miniconda.sh && \
121- find / -type d -name __pycache__ | xargs rm -rf
122-
123- # Create Conda environment
124- COPY { conda_env_yml } { conda_env_yml }
125- RUN ${{CONDA_PREFIX}}/bin/conda env create -f /tmp/{ conda_env_yml }
126-
127- # Activate Conda environment
128- ENV CONDA_DEFAULT_ENV { conda_env_name }
129- ENV CONDA_PREFIX ${{CONDA_PREFIX}}/envs/${{CONDA_DEFAULT_ENV}}
130- ENV PATH="${{CONDA_PREFIX}}/bin:${{PATH}}" LD_LIBRARY_PATH="${{CONDA_PREFIX}}/lib:$LD_LIBRARY_PATH"
131- """
132-
133- with tempfile .TemporaryDirectory () as tmpdir :
134- tmpdir = pathlib .Path (tmpdir )
135- dockerfile_path = tmpdir / "Dockerfile"
136- condafile_path = tmpdir / conda_env_yml
137- build = BuildContext (path = tmpdir , dockerfile_path = dockerfile_path .name )
138-
139- with open (dockerfile_path , "w" ) as file :
140- file .write (dockerfile )
141- with open (condafile_path , "w" ) as file :
142- file .write (condafile )
143-
144- try :
145- client .environments .create_or_update (
146- Environment (
147- name = environment_name ,
148- image = None if use_gpu else image ,
149- build = build if use_gpu else None ,
150- conda_file = None if use_gpu else condafile_path ,
151- )
83+ compute = "gpu" if use_gpu else "cpu"
84+ extras = (
85+ "[dev" + (",gpu" if use_gpu else "" ) + (",spark" if use_spark else "" ) + "]"
86+ )
87+ dockerfile = pathlib .Path ("tools/docker/Dockerfile" )
88+
89+ # Docker's --build-args is not supported by AzureML Python SDK v2 as shown
90+ # in [the issue #33902](https://github.com/Azure/azure-sdk-for-python/issues/33902)
91+ # so the build args are configured by regex substituion
92+ text = dockerfile .read_text ()
93+ text = re .sub (r"(ARG\sCOMPUTE=).*" , rf'\1"{ compute } , text )
94+ text = re .sub (r"(ARG\sEXTRAS=).*" , rf'\1"{ extras } , text )
95+ text = re .sub (r"(ARG\sGIT_REF=).*" , r'\1""' , text )
96+ text = re .sub (r"(ARG\sJDK_VERSION=).*" , rf'\1"{ conda_openjdk_version } , text )
97+ text = re .sub (r"(ARG\sPYTHON_VERSION=).*" , rf'\1"{ python_version } , text )
98+ dockerfile .write_text (text )
99+
100+ try :
101+ client .environments .create_or_update (
102+ Environment (
103+ name = environment_name ,
104+ build = BuildContext (
105+ # Set path for Docker to access to Recommenders root
106+ path = "." ,
107+ dockerfile_path = dockerfile ,
108+ ),
152109 )
153- except ResourceExistsError :
154- pass
110+ )
111+ except ResourceExistsError :
112+ pass
155113
156114
157115def run_tests (
0 commit comments