Skip to content

Commit c1391ba

Browse files
csadorfcsadorf
authored
CI: Run extended accel mode tests (#6377)
Authors: - Simon Adorf (https://github.com/csadorf) Approvers: - Bradley Dice (https://github.com/bdice) URL: #6377
1 parent 80b98b1 commit c1391ba

File tree

6 files changed

+191
-0
lines changed

6 files changed

+191
-0
lines changed

‎.github/workflows/pr.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ jobs:
2323
- conda-python-build
2424
- conda-python-tests-singlegpu
2525
- conda-python-tests-dask
26+
- conda-python-scikit-learn-accel-tests
2627
- conda-notebook-tests
2728
- docs-build
2829
- telemetry-setup
@@ -162,6 +163,14 @@ jobs:
162163
with:
163164
build_type: pull-request
164165
script: "ci/test_python_dask.sh"
166+
conda-python-scikit-learn-accel-tests:
167+
needs: [conda-python-build, changed-files]
168+
secrets: inherit
169+
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
170+
if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
171+
with:
172+
build_type: pull-request
173+
script: "ci/test_python_scikit_learn_tests.sh"
165174
conda-notebook-tests:
166175
needs: [conda-python-build, changed-files]
167176
secrets: inherit

‎.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ tmp/
3333
wheels/
3434
wheelhouse/
3535
_skbuild/
36+
junit-*.xml
37+
3638

3739
## files pickled in notebook when ran during python docstring generation
3840
docs/source/*.model
@@ -73,3 +75,6 @@ compile_commands.json
7375
# ref: https://github.com/rapidsai/cuml/pull/6201
7476
pytest.ini
7577
!python/cuml/cuml/benchmark/automated/pytest.ini
78+
79+
# testing
80+
ci/**/scikit-learn/

‎ci/accel/scikit-learn-tests/README.md

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# scikit-learn Acceleration Tests
2+
3+
This suite provides infrastructure to run and analyze tests for scikit-learn with cuML acceleration support.
4+
5+
## Components
6+
7+
- `run-tests.sh`
8+
Executes scikit-learn tests using GPU-accelerated paths. Any arguments passed to the script are forwarded directly to pytest.
9+
10+
Example usage:
11+
```bash
12+
./run-tests.sh # Run all tests
13+
./run-tests.sh -v -k test_kmeans # Run specific test with verbosity
14+
./run-tests.sh -x --pdb # Stop on first failure and debug
15+
```
16+
17+
- `summarize-results.sh`
18+
Analyzes test results from an XML report file and prints a summary.
19+
Options:
20+
- `-v, --verbose` : Display detailed failure information
21+
- `-f, --fail-below VALUE` : Set a minimum pass rate threshold (0-100)
22+
23+
## Usage
24+
25+
### 1. Run tests
26+
Run all tests:
27+
```bash
28+
./run-tests.sh
29+
```
30+
31+
Run specific tests using pytest arguments:
32+
```bash
33+
./run-tests.sh -v -k "test_logistic"
34+
```
35+
36+
### 2. Summarize test results
37+
Generate a summary from the XML report with a pass rate threshold:
38+
```bash
39+
./summarize-results.sh -v -f 80 report.xml
40+
```

‎ci/accel/scikit-learn-tests/run-tests.sh

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
# Copyright (c) 2025, NVIDIA CORPORATION.
3+
4+
# This script runs scikit-learn tests with the cuml.accel plugin.
5+
# Any arguments passed to this script are forwarded directly to pytest.
6+
#
7+
# Example usage:
8+
# ./run-tests.sh # Run all tests
9+
# ./run-tests.sh -v -k test_kmeans # Run specific test with verbosity
10+
# ./run-tests.sh -x --pdb # Stop on first failure and debug
11+
12+
set -eu
13+
14+
pytest -p cuml.accel --pyargs sklearn -v $@

‎ci/accel/scikit-learn-tests/summarize-results.sh

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#!/bin/bash
2+
# Copyright (c) 2025, NVIDIA CORPORATION.
3+
4+
usage() {
5+
echo "Usage: $0 [options] REPORT_FILE"
6+
echo ""
7+
echo "Options:"
8+
echo " -h, --help Show this help message"
9+
echo " -v, --verbose Show detailed failure information"
10+
echo " -f, --fail-below VALUE Minimum pass rate threshold [0-100] (default: 0)"
11+
exit 1
12+
}
13+
14+
# Parse command line arguments
15+
THRESHOLD=0
16+
VERBOSE=0
17+
18+
while [[ $# -gt 0 ]]; do
19+
case $1 in
20+
-h|--help)
21+
usage
22+
;;
23+
-f|--fail-below)
24+
THRESHOLD="$2"
25+
shift 2
26+
;;
27+
-v|--verbose)
28+
VERBOSE=1
29+
shift
30+
;;
31+
*)
32+
if [ -z "${REPORT_FILE:-}" ]; then
33+
REPORT_FILE="$1"
34+
else
35+
echo "Unknown option: $1"
36+
usage
37+
fi
38+
shift
39+
;;
40+
esac
41+
done
42+
43+
if [ -z "${REPORT_FILE:-}" ]; then
44+
echo "Error: No report file specified"
45+
usage
46+
fi
47+
48+
# Validate threshold is a number between 0 and 100
49+
if ! [[ "$THRESHOLD" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
50+
echo "Error: Threshold must be a number"
51+
exit 1
52+
fi
53+
54+
if ! awk -v t="$THRESHOLD" 'BEGIN{exit !(t >= 0 && t <= 100)}'; then
55+
echo "Error: Threshold must be between 0 and 100"
56+
exit 1
57+
fi
58+
59+
# Extract test statistics using xmllint
60+
total_tests=$(xmllint --xpath "string(/testsuites/testsuite/@tests)" "${REPORT_FILE}")
61+
failures=$(xmllint --xpath "string(/testsuites/testsuite/@failures)" "${REPORT_FILE}")
62+
errors=$(xmllint --xpath "string(/testsuites/testsuite/@errors)" "${REPORT_FILE}")
63+
skipped=$(xmllint --xpath "string(/testsuites/testsuite/@skipped)" "${REPORT_FILE}")
64+
time=$(xmllint --xpath "string(/testsuites/testsuite/@time)" "${REPORT_FILE}")
65+
66+
# Calculate passed tests and pass rate using awk
67+
passed=$((total_tests - failures - errors - skipped))
68+
pass_rate=$(awk -v passed="$passed" -v total="$total_tests" 'BEGIN { printf "%.2f", (passed/total) * 100 }')
69+
70+
# Print summary
71+
echo "Test Summary:"
72+
echo " Total Tests: ${total_tests}"
73+
echo " Passed: ${passed}"
74+
echo " Failed: ${failures}"
75+
echo " Errors: ${errors}"
76+
echo " Skipped: ${skipped}"
77+
echo " Pass Rate: ${pass_rate}%"
78+
echo " Total Time: ${time}s"
79+
80+
# List failed tests only in verbose mode
81+
if [ "$((failures + errors))" -gt 0 ] && [ "${VERBOSE}" -eq 1 ]; then
82+
echo ""
83+
echo "Failed Tests:"
84+
xmllint --xpath "//testcase[failure or error]/@name" "${REPORT_FILE}" | tr ' ' '\n' | sed 's/name=//g' | sed 's/"//g' | grep .
85+
fi
86+
87+
# Check if threshold is nonzero before applying the check.
88+
if awk -v rate="$pass_rate" -v threshold="$THRESHOLD" 'BEGIN { exit (rate >= threshold) }'; then
89+
echo ""
90+
echo "Error: Pass rate ${pass_rate}% is below threshold ${THRESHOLD}%"
91+
exit 1
92+
fi
93+
94+
# In all other cases, return with success code.
95+
exit 0

‎ci/test_python_scikit_learn_tests.sh

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
# Copyright (c) 2025, NVIDIA CORPORATION.
3+
4+
# Support invoking test script outside the script directory
5+
cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
6+
7+
# Common setup steps shared by Python test jobs
8+
source ./ci/test_python_common.sh
9+
10+
EXITCODE=0
11+
trap "EXITCODE=1" ERR
12+
set +e
13+
14+
# Run scikit-learn tests with acceleration enabled
15+
rapids-logger "Running scikit-learn tests with cuML acceleration"
16+
17+
# Run the tests
18+
./ci/accel/scikit-learn-tests/run-tests.sh \
19+
--junitxml="${RAPIDS_TESTS_DIR}/junit-cuml-accel-scikit-learn.xml" || true
20+
21+
# Analyze results and check pass rate threshold
22+
rapids-logger "Analyzing test results"
23+
./ci/accel/scikit-learn-tests/summarize-results.sh \
24+
--fail-below 80 \
25+
"${RAPIDS_TESTS_DIR}/junit-cuml-accel-scikit-learn.xml"
26+
27+
rapids-logger "Test script exiting with value: $EXITCODE"
28+
exit ${EXITCODE}

0 commit comments

Comments
 (0)