Add support for Python 3.10 and 3.11

.github/workflows/azureml-cpu-nightly.yml

@@ -67,7 +67,7 @@ jobs:
     strategy:
       max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
       matrix:
-        python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"']
+        python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"']
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
     steps:
       - name: Check out repository code

.github/workflows/azureml-gpu-nightly.yml

@@ -67,7 +67,7 @@ jobs:
     strategy:
       max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
       matrix:
-        python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"']
+        python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"']
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
     steps:
       - name: Check out repository code

.github/workflows/azureml-spark-nightly.yml

@@ -66,7 +66,7 @@ jobs:
     strategy:
       max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
       matrix:
-        python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"']
+        python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"']
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
     steps:
       - name: Check out repository code

.github/workflows/azureml-unit-tests.yml

@@ -54,7 +54,7 @@ jobs:
     strategy:
       max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
       matrix:
-        python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"']
+        python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"']
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
     steps:
       - name: Check out repository code

setup.py

@@ -8,7 +8,7 @@
 import sys
 import time
 
-# workround for enabling editable user pip installs
+# workaround for enabling editable user pip installs
 site.ENABLE_USER_SITE = "--user" in sys.argv[1:]
 
 # version
@@ -27,57 +27,48 @@
     version += ".post" + str(int(time.time()))
 
 install_requires = [
-    "numpy>=1.19",  # 1.19 required by tensorflow 2.6
-    "pandas>1.0.3,<2",
-    "scipy>=1.0.0,<2",
-    "tqdm>=4.31.1,<5",
-    "matplotlib>=2.2.2,<4",
-    "scikit-learn>=0.22.1,<1.0.3",
-    "numba>=0.38.1,<1",
-    "lightfm>=1.15,<2",
-    "lightgbm>=2.2.1",
-    "memory_profiler>=0.54.0,<1",
-    "nltk>=3.4,<4",
-    "seaborn>=0.8.1,<1",
-    "transformers>=2.5.0,<5",
-    "category_encoders>=1.3.0,<2",
-    "jinja2>=2,<3.1",
-    "requests>=2.0.0,<3",
-    "cornac>=1.1.2,<1.15.2;python_version<='3.7'",
-    "cornac>=1.15.2,<2;python_version>='3.8'", # After 1.15.2, Cornac requires python 3.8
-    "retrying>=1.3.3",
-    "pandera[strategies]>=0.6.5",  # For generating fake datasets
-    "scikit-surprise>=1.0.6",
-    "scrapbook>=0.5.0,<1.0.0",
+    "pandas>1.5.2,<2.1",  # requires numpy
+    "scikit-learn>=1.1.3,<2",  # requires scipy
+    "numba>=0.57.0,<1",
+    "lightfm>=1.17,<2",
+    "lightgbm>=3.3.2,<4",
+    "memory-profiler>=0.61.0,<1",
+    "nltk>=3.8.1,<4",  # requires tqdm
+    "seaborn>=0.12.0,<1",  # requires matplotlib
+    "transformers>=4.26.0,<5",  # requires pyyaml, tqdm
+    "category-encoders>=2.6.0,<3",
+    "jinja2>=3.1.0,<3.2",
+    "cornac>=1.15.2,<2",  # requires tqdm
+    "retrying>=1.3.4",
+    "pandera[strategies]>=0.15.0",  # For generating fake datasets
+    "scikit-surprise>=1.1.3",
+    "scrapbook>=0.5.0,<1.0.0",  # requires tqdm, papermill
 ]
 
 # shared dependencies
 extras_require = {
     "examples": [
-        "hyperopt>=0.1.2,<1",
-        "ipykernel>=4.6.1,<7",
-        "jupyter>=1,<2",
-        "locust>=1,<2",
-        "papermill>=2.1.2,<3",
+        "hyperopt>=0.2.7,<1",
+        "notebook>=6.5.4,<8",  # requires jupyter, ipykernel
+        "locust>=2.15.1,<3",
     ],
     "gpu": [
-        "nvidia-ml-py3>=7.352.0",
-        # TensorFlow compiled with CUDA 11.2, cudnn 8.1
-        "tensorflow~=2.6.1;python_version=='3.6'",
-        "tensorflow~=2.7.0;python_version>='3.7'",
+        "nvidia-ml-py>=11.510.69",
+        # TensorFlow compiled with CUDA 11.8, cudnn 8.6.0.163
+        "tensorflow~=2.12.0",
         "tf-slim>=1.1.0",
-        "torch>=1.8",  # for CUDA 11 support
-        "fastai>=1.0.46,<2",
+        "torch>=2.0.1",
+        "fastai>=2.7.11,<3",
     ],
     "spark": [
-        "pyarrow>=0.12.1,<7.0.0",
-        "pyspark>=2.4.5,<3.3.0",
+        "pyarrow>=10.0.1",
+        "pyspark>=3.0.1,<=3.4.0",
     ],
     "dev": [
-        "black>=18.6b4,<21",
-        "pytest>=3.6.4",
-        "pytest-cov>=2.12.1",
-        "pytest-mock>=3.6.1",  # for access to mock fixtures in pytest
+        "black>=23.3.0,<24",
+        "pytest>=7.2.1",
+        "pytest-cov>=4.1.0",
+        "pytest-mock>=3.10.0",  # for access to mock fixtures in pytest
     ],
 }
 # for the brave of heart
@@ -123,6 +114,7 @@
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Operating System :: Microsoft :: Windows",
         "Operating System :: POSIX :: Linux",
         "Operating System :: MacOS",
@@ -132,7 +124,7 @@
     "machine learning python spark gpu",
     install_requires=install_requires,
     package_dir={"recommenders": "recommenders"},
-    python_requires=">=3.6, <3.10",
+    python_requires=">=3.8, <3.11",
     packages=find_packages(
         where=".",
         exclude=["contrib", "docs", "examples", "scenarios", "tests", "tools"],

tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py

@@ -37,7 +37,6 @@
 """
 import argparse
 import logging
-import glob
 
 from azureml.core.authentication import AzureCliAuthentication
 from azureml.core import Workspace
@@ -146,8 +145,7 @@ def setup_persistent_compute_target(workspace, cluster_name, vm_size, max_nodes)
 
 def create_run_config(
     cpu_cluster,
-    docker_proc_type,
-    workspace,
+    docker_image,
     add_gpu_dependencies,
     add_spark_dependencies,
     conda_pkg_jdk,
@@ -166,8 +164,7 @@ def create_run_config(
                                                 the following:
                                                 - Reco_cpu_test
                                                 - Reco_gpu_test
-            docker_proc_type (str)          : processor type, cpu or gpu
-            workspace                       : workspace reference
+            docker_image (str)              : docker image for cpu or gpu
             add_gpu_dependencies (bool)     : True if gpu packages should be
                                         added to the conda environment, else False
             add_spark_dependencies (bool)   : True if PySpark packages should be
@@ -181,7 +178,20 @@ def create_run_config(
     run_azuremlcompute = RunConfiguration()
     run_azuremlcompute.target = cpu_cluster
     run_azuremlcompute.environment.docker.enabled = True
-    run_azuremlcompute.environment.docker.base_image = docker_proc_type
+    # See https://learn.microsoft.com/en-us/azure/machine-learning/how-to-train-with-custom-image?view=azureml-api-1#use-a-custom-dockerfile-optional
+    run_azuremlcompute.environment.docker.base_image = None
+    run_azuremlcompute.environment.docker.base_dockerfile = f"""
+    FROM {docker_image}
+    # Install system-level deps for scipy.  See
+    # https://docs.scipy.org/doc/scipy/dev/contributor/building.html
+    RUN apt-get update && \
+        apt-get install -y \
+        gfortran \
+        libopenblas-dev \
+        liblapack-dev \
+        pkg-config
+    RUN apt-get install -y git
+    """
 
     # Use conda_dependencies.yml to create a conda environment in
     # the Docker image for execution
@@ -425,13 +435,11 @@ def create_arg_parser():
     args = create_arg_parser()
 
     if args.dockerproc == "cpu":
-        from azureml.core.runconfig import DEFAULT_CPU_IMAGE
-
-        docker_proc_type = DEFAULT_CPU_IMAGE
+        # https://github.com/Azure/AzureML-Containers/blob/master/base/cpu/openmpi4.1.0-ubuntu22.04
+        docker_image = "mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04"
     else:
-        from azureml.core.runconfig import DEFAULT_GPU_IMAGE
-
-        docker_proc_type = DEFAULT_GPU_IMAGE
+        # https://github.com/Azure/AzureML-Containers/blob/master/base/gpu/openmpi4.1.0-cuda11.8-cudnn8-ubuntu22.04
+        docker_image = "mcr.microsoft.com/azureml/openmpi4.1.0-cuda11.8-cudnn8-ubuntu22.04"
 
     cli_auth = AzureCliAuthentication()
 
@@ -452,8 +460,7 @@ def create_arg_parser():
 
     run_config = create_run_config(
         cpu_cluster=cpu_cluster,
-        docker_proc_type=docker_proc_type,
-        workspace=workspace,
+        docker_image=docker_image,
         add_gpu_dependencies=args.add_gpu_dependencies,
         add_spark_dependencies=args.add_spark_dependencies,
         conda_pkg_jdk=args.conda_pkg_jdk,