Migrate ci.yml to AKS cpubuilder cluster. (#3967)

This commit contains the ccache and python changes neccessary to migrate
the ci.yml workflow to an azure kubernetes cpubuilder cluster.
Timings before and after the infra change are in the same ballpark (~10
mins).
Previously, `test/python/compile.py` was unnecessarily excluding the
python site packages, which needed to be changed for the cluster.
We also make use of `hendrikmuhs/ccache-action` now and use a key of
`${{ github.job }}-${{ matrix.torch-version }}` which works as intended:
https://github.com/llvm/torch-mlir/actions/runs/12828039126/job/35771248056#step:7:4657

Author: saienduri

.github/workflows/ci.yml

@@ -23,20 +23,10 @@ jobs:
       matrix:
         torch-version: [nightly, stable]
     name: Build and Test (Linux, torch-${{ matrix.torch-version }}, assertions)
-    runs-on: torch-mlir-cpubuilder-manylinux-x86-64
+    runs-on: torch-mlir-cpubuilder-linux-x86-64-scale
     env:
       CACHE_DIR: ${{ github.workspace }}/.container-cache
     steps:
-      - name: Configure local git mirrors
-        run: |
-          # Our stock runners have access to certain local git caches. If these
-          # files are available, it will prime the cache and configure git to
-          # use them. Practically, this eliminates network/latency for cloning
-          # llvm.
-          if [[ -x /gitmirror/scripts/trigger_update_mirrors.sh ]]; then
-            /gitmirror/scripts/trigger_update_mirrors.sh
-            /gitmirror/scripts/git_config.sh
-          fi
       - name: "Checking out repository"
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
         with:
@@ -50,11 +40,25 @@ jobs:
           restore-keys: |
             build-test-cpp-asserts-manylinux-${{ matrix.torch-version }}-v2-
 
+      - name: "Setting up Python"
+        run: |
+          sudo apt update
+          sudo apt install software-properties-common -y
+          sudo add-apt-repository ppa:deadsnakes/ppa -y
+          sudo apt install python3.11 python3-pip -y
+          sudo apt-get install python3.11-dev python3.11-venv build-essential -y
+
       - name: Install python deps (torch-${{ matrix.torch-version }})
         run: |
           export cache_dir="${{ env.CACHE_DIR }}"
           bash build_tools/ci/install_python_deps.sh ${{ matrix.torch-version }}
 
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@53911442209d5c18de8a31615e0923161e435875 # v1.2.16
+        with:
+          key: ${{ github.job }}-${{ matrix.torch-version }}
+          save: ${{ needs.setup.outputs.write-caches == 1 }}
+
       - name: Build project
         run: |
           export cache_dir="${{ env.CACHE_DIR }}"

build_tools/ci/build_posix.sh

@@ -20,7 +20,7 @@ echo "Caching to ${cache_dir}"
 mkdir -p "${cache_dir}/ccache"
 mkdir -p "${cache_dir}/pip"
 
-python="$(which python)"
+python="$(which python3)"
 echo "Using python: $python"
 
 export CMAKE_TOOLCHAIN_FILE="$this_dir/linux_default_toolchain.cmake"
@@ -40,7 +40,7 @@ echo "::group::CMake configure"
 cmake -S "$repo_root/externals/llvm-project/llvm" -B "$build_dir" \
   -GNinja \
   -DCMAKE_BUILD_TYPE=Release \
-  -DPython3_EXECUTABLE="$(which python)" \
+  -DPython3_EXECUTABLE="$(which python3)" \
   -DLLVM_ENABLE_ASSERTIONS=ON \
   -DTORCH_MLIR_ENABLE_WERROR_FLAG=ON \
   -DCMAKE_INSTALL_PREFIX="$install_dir" \

build_tools/ci/install_python_deps.sh

@@ -7,7 +7,7 @@ repo_root="$(cd $this_dir/../.. && pwd)"
 torch_version="${1:-unknown}"
 
 echo "::group::installing llvm python deps"
-python -m pip install --no-cache-dir -r $repo_root/externals/llvm-project/mlir/python/requirements.txt
+python3 -m pip install --no-cache-dir -r $repo_root/externals/llvm-project/mlir/python/requirements.txt
 echo "::endgroup::"
 
 case $torch_version in
@@ -30,5 +30,5 @@ case $torch_version in
 esac
 
 echo "::group::installing test requirements"
-python -m pip install --no-cache-dir -r $repo_root/test-requirements.txt
+python3 -m pip install --no-cache-dir -r $repo_root/test-requirements.txt
 echo "::endgroup::"

build_tools/ci/test_posix.sh

@@ -9,7 +9,7 @@ torch_version="${1:-unknown}"
 export PYTHONPATH="$repo_root/build/tools/torch-mlir/python_packages/torch_mlir:$repo_root/projects/pt1"
 
 echo "::group::Run ONNX e2e integration tests"
-python -m e2e_testing.main --config=onnx -v
+python3 -m e2e_testing.main --config=onnx -v
 echo "::endgroup::"
 
 case $torch_version in
@@ -23,12 +23,12 @@ case $torch_version in
 
     # TODO: Need to verify in the stable version
     echo "::group::Run FxImporter e2e integration tests"
-    python -m e2e_testing.main --config=fx_importer -v
+    python3 -m e2e_testing.main --config=fx_importer -v
     echo "::endgroup::"
 
     # TODO: Need to verify in the stable version
     echo "::group::Run FxImporter2Stablehlo e2e integration tests"
-    python -m e2e_testing.main --config=fx_importer_stablehlo -v
+    python3 -m e2e_testing.main --config=fx_importer_stablehlo -v
     echo "::endgroup::"
     ;;
   stable)

build_tools/update_abstract_interp_lib.sh

@@ -44,7 +44,7 @@ fi
 # To enable this python package, manually build torch_mlir with:
 #   -DTORCH_MLIR_ENABLE_JIT_IR_IMPORTER=ON
 # TODO: move this package out of JIT_IR_IMPORTER.
-PYTHONPATH="${pypath}" python \
+PYTHONPATH="${pypath}" python3 \
   -m torch_mlir.jit_ir_importer.build_tools.abstract_interp_lib_gen \
   --pytorch_op_extensions=${ext_module:-""} \
   --torch_transforms_cpp_dir="${torch_transforms_cpp_dir}"

build_tools/update_torch_ods.sh

@@ -45,7 +45,7 @@ set +u
 # To enable this python package, manually build torch_mlir with:
 #   -DTORCH_MLIR_ENABLE_PYTORCH_EXTENSIONS=ON
 # TODO: move this package out of JIT_IR_IMPORTER.
-PYTHONPATH="${PYTHONPATH}:${pypath}" python \
+PYTHONPATH="${PYTHONPATH}:${pypath}" python3 \
   -m torch_mlir.jit_ir_importer.build_tools.torch_ods_gen \
   --torch_ir_include_dir="${torch_ir_include_dir}" \
   --pytorch_op_extensions="${ext_module}" \

test/python/compile.py

@@ -1,4 +1,4 @@
-# RUN: %PYTHON -s %s 2>&1 | FileCheck %s
+# RUN: %PYTHON %s 2>&1 | FileCheck %s
 
 import gc
 import sys