Skip to content

Commit c6a03b7

Browse files
authored
Require hypothesis tests have explicit examples (#6409)
Addresses #6399 Authors: - Anupam (https://github.com/aamijar) Approvers: - Simon Adorf (https://github.com/csadorf) URL: #6409
1 parent 4384f17 commit c6a03b7

9 files changed

+303
-10
lines changed

python/cuml/cuml/tests/conftest.py

+23-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright (c) 2018-2024, NVIDIA CORPORATION.
2+
# Copyright (c) 2018-2025, NVIDIA CORPORATION.
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -148,6 +148,28 @@ def pytest_addoption(parser):
148148

149149
def pytest_collection_modifyitems(config, items):
150150

151+
# Check for hypothesis tests without examples
152+
tests_without_examples = []
153+
for item in items:
154+
if isinstance(item, pytest.Function):
155+
# Check if function has @given decorator
156+
has_given = hasattr(item.obj, "hypothesis")
157+
# Check if function has @example decorator
158+
has_example = hasattr(item.obj, "hypothesis_explicit_examples")
159+
160+
if has_given and not has_example:
161+
tests_without_examples.append(
162+
f"Test {item.name} uses @given but has no @example cases."
163+
)
164+
165+
if tests_without_examples:
166+
msg = (
167+
"\nCollection failed because the following tests lack examples:\n"
168+
+ "\n".join(f" - {e}" for e in tests_without_examples)
169+
)
170+
raise pytest.UsageError(msg)
171+
172+
# Handle test categories (unit/quality/stress)
151173
should_run_quality = config.getoption("--run_quality")
152174
should_run_stress = config.getoption("--run_stress")
153175

python/cuml/cuml/tests/explainer/test_gpu_treeshap.py

+21-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright (c) 2021-2023, NVIDIA CORPORATION.
2+
# Copyright (c) 2021-2025, NVIDIA CORPORATION.
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -22,7 +22,14 @@
2222
from cuml.internals.import_utils import has_sklearn
2323
from cuml.internals.import_utils import has_lightgbm, has_shap
2424
from cuml.explainer.tree_shap import TreeExplainer
25-
from hypothesis import given, settings, assume, HealthCheck, strategies as st
25+
from hypothesis import (
26+
example,
27+
given,
28+
settings,
29+
assume,
30+
HealthCheck,
31+
strategies as st,
32+
)
2633
from cuml.internals.safe_imports import gpu_only_import
2734
import json
2835
import pytest
@@ -914,6 +921,17 @@ def check_efficiency_interactions(expected_value, pred, shap_values):
914921
max_examples=20,
915922
suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large],
916923
)
924+
@example(
925+
params=(
926+
pd.DataFrame(np.ones((10, 5), dtype=np.float32)),
927+
np.ones(10, dtype=np.float32),
928+
curfr(max_features=1.0, random_state=0, n_streams=1, n_bins=10).fit(
929+
np.ones((10, 5), dtype=np.float32), np.ones(10, dtype=np.float32)
930+
),
931+
np.ones(10, dtype=np.float32),
932+
),
933+
interactions_method="shapley-interactions",
934+
)
917935
@given(
918936
shap_strategy(),
919937
st.sampled_from(["shapley-interactions", "shapley-taylor"]),
@@ -989,6 +1007,7 @@ def test_different_algorithms_different_output():
9891007

9901008

9911009
@settings(deadline=None)
1010+
@example(input_type="numpy")
9921011
@given(st.sampled_from(["numpy", "cupy", "cudf", "pandas"]))
9931012
def test_input_types(input_type):
9941013
# simple test to not crash on different input data-frames

python/cuml/cuml/tests/test_array.py

+54-1
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@
5858
squeezed_shape,
5959
to_nparray,
6060
)
61-
from hypothesis import assume, given, settings
61+
from hypothesis import assume, example, given, settings
6262
from hypothesis import strategies as st
6363

6464
cp = gpu_only_import("cupy")
@@ -125,6 +125,14 @@ def _assert_equal(array_like, cuml_array):
125125
)
126126

127127

128+
@example(
129+
input_type="numpy",
130+
dtype=np.float32,
131+
shape=(10, 10),
132+
order="C",
133+
mem_type="device",
134+
force_gc=False,
135+
)
128136
@given(
129137
input_type=cuml_array_input_types(),
130138
dtype=cuml_array_dtypes(),
@@ -164,6 +172,13 @@ def test_array_init(input_type, dtype, shape, order, mem_type, force_gc):
164172
_assert_equal(input_array_copy, cuml_array)
165173

166174

175+
@example(
176+
data_type=bytes,
177+
dtype=np.float32,
178+
shape=(10, 10),
179+
order="C",
180+
mem_type="device",
181+
)
167182
@given(
168183
data_type=st.sampled_from([bytes, bytearray, memoryview]),
169184
dtype=cuml_array_dtypes(),
@@ -193,6 +208,13 @@ def test_array_init_from_bytes(data_type, dtype, shape, order, mem_type):
193208
assert cp.all(cp.asarray(array_copy) == array_copy)
194209

195210

211+
@example(
212+
input_type="numpy",
213+
dtype=np.float32,
214+
shape=(10, 10),
215+
order="C",
216+
mem_type="device",
217+
)
196218
@given(
197219
input_type=cuml_array_input_types(),
198220
dtype=cuml_array_dtypes(),
@@ -226,6 +248,7 @@ def test_array_mem_type(input_type, dtype, shape, order, mem_type):
226248
)
227249

228250

251+
@example(inp=np.array([1, 2, 3]), indices=slice(1, 3), mem_type="device")
229252
@given(
230253
inp=cuml_array_inputs(),
231254
indices=st.slices(10), # TODO: should be basic_indices() as shown below
@@ -263,6 +286,7 @@ def test_get_set_item(inp, indices, mem_type):
263286
_assert_equal(inp, ary)
264287

265288

289+
@example(shape=(10, 10), dtype=np.float32, order="C", mem_type="device")
266290
@given(
267291
shape=cuml_array_shapes(),
268292
dtype=cuml_array_dtypes(),
@@ -278,6 +302,7 @@ def test_create_empty(shape, dtype, order, mem_type):
278302
assert ary.dtype == np.dtype(dtype)
279303

280304

305+
@example(shape=(10, 10), dtype=np.float32, order="C", mem_type="device")
281306
@given(
282307
shape=cuml_array_shapes(),
283308
dtype=cuml_array_dtypes(),
@@ -293,6 +318,7 @@ def test_create_zeros(shape, dtype, order, mem_type):
293318
assert mem_type.xpy.all(test == mem_type.xpy.asarray(ary))
294319

295320

321+
@example(shape=(10, 10), dtype=np.float32, order="C", mem_type="device")
296322
@given(
297323
shape=cuml_array_shapes(),
298324
dtype=cuml_array_dtypes(),
@@ -308,6 +334,7 @@ def test_create_ones(shape, dtype, order, mem_type):
308334
assert mem_type.xpy.all(test == mem_type.xpy.asarray(ary))
309335

310336

337+
@example(shape=(10, 10), dtype=np.float32, order="C", mem_type="device")
311338
@given(
312339
shape=cuml_array_shapes(),
313340
dtype=cuml_array_dtypes(),
@@ -332,6 +359,7 @@ def cudf_compatible_dtypes(dtype):
332359
return dtype not in UNSUPPORTED_CUDF_DTYPES
333360

334361

362+
@example(inp=np.array([1, 2, 3]), input_mem_type="device", output_type="cupy")
335363
@given(
336364
inp=cuml_array_inputs(),
337365
input_mem_type=cuml_array_mem_types(),
@@ -391,6 +419,7 @@ def assert_data_equal_(res):
391419
assert_data_equal_(res)
392420

393421

422+
@example(inp=np.array([1, 2, 3]), output_type="cupy", mem_type="device")
394423
@given(
395424
inp=cuml_array_inputs(),
396425
output_type=cuml_array_output_types(),
@@ -441,6 +470,14 @@ def test_end_to_end_conversion_via_intermediate(inp, output_type, mem_type):
441470
_assert_equal(inp, array2)
442471

443472

473+
@example(
474+
output_type="cupy",
475+
shape=(10, 10),
476+
dtype=np.float32,
477+
order="C",
478+
out_dtype=np.float32,
479+
mem_type="device",
480+
)
444481
@given(
445482
output_type=cuml_array_output_types(),
446483
shape=cuml_array_shapes(),
@@ -472,6 +509,7 @@ def test_output_dtype(output_type, shape, dtype, order, out_dtype, mem_type):
472509
res.dtype is out_dtype
473510

474511

512+
@example(inp=np.array([1, 2, 3]), mem_type="device")
475513
@given(inp=cuml_array_inputs(), mem_type=cuml_array_mem_types())
476514
@settings(deadline=None)
477515
def test_array_interface(inp, mem_type):
@@ -522,6 +560,11 @@ def test_array_interface(inp, mem_type):
522560
)
523561

524562

563+
@example(
564+
inp=np.array([1, 2, 3]),
565+
to_serialize_mem_type="device",
566+
from_serialize_mem_type="device",
567+
)
525568
@given(
526569
inp=cuml_array_inputs(),
527570
to_serialize_mem_type=cuml_array_mem_types(),
@@ -561,6 +604,11 @@ def test_serialize(inp, to_serialize_mem_type, from_serialize_mem_type):
561604

562605

563606
@pytest.mark.parametrize("protocol", [4, 5])
607+
@example(
608+
inp=np.array([1, 2, 3]),
609+
to_serialize_mem_type="device",
610+
from_serialize_mem_type="device",
611+
)
564612
@given(
565613
inp=cuml_array_inputs(),
566614
to_serialize_mem_type=cuml_array_mem_types(),
@@ -601,6 +649,7 @@ def test_pickle(protocol, inp, to_serialize_mem_type, from_serialize_mem_type):
601649
assert ary.order == b.order
602650

603651

652+
@example(inp=np.array([1, 2, 3]), mem_type="device")
604653
@given(inp=cuml_array_inputs(), mem_type=cuml_array_mem_types())
605654
@settings(deadline=None)
606655
def test_deepcopy(inp, mem_type):
@@ -633,6 +682,7 @@ def test_deepcopy(inp, mem_type):
633682

634683

635684
@pytest.mark.parametrize("operation", [operator.add, operator.sub])
685+
@example(a=np.array([1, 2, 3]), mem_type="device")
636686
@given(
637687
a=cuml_array_inputs(),
638688
mem_type=cuml_array_mem_types(),
@@ -652,6 +702,7 @@ def test_cumlary_binops(operation, a, mem_type):
652702

653703

654704
@pytest.mark.parametrize("order", ["F", "C"])
705+
@example(mem_type="device")
655706
@given(mem_type=cuml_array_mem_types())
656707
@settings(deadline=None)
657708
def test_sliced_array_owner(order, mem_type):
@@ -701,6 +752,7 @@ def test_sliced_array_owner(order, mem_type):
701752
)
702753

703754

755+
@example(input_type="numpy", dtype=np.float32, shape=(10, 10), order="C")
704756
@given(
705757
input_type=cuml_array_input_types(),
706758
dtype=cuml_array_dtypes(),
@@ -713,6 +765,7 @@ def test_array_to_memory_order(input_type, dtype, shape, order):
713765
assert array_to_memory_order(input_array, default=order) == order
714766

715767

768+
@example(input_type="numpy", dtype=np.float32, shape=(10, 10), order="C")
716769
@given(
717770
input_type=st.sampled_from(("cupy", "numpy")),
718771
dtype=cuml_array_dtypes(),

python/cuml/cuml/tests/test_kernel_density.py

+12-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
from sklearn.model_selection import GridSearchCV
1919
import pytest
2020
from hypothesis.extra.numpy import arrays
21-
from hypothesis import given, settings, assume, strategies as st
21+
from hypothesis import example, given, settings, assume, strategies as st
2222
from cuml.neighbors import KernelDensity, VALID_KERNELS, logsumexp_kernel
2323
from cuml.common.exceptions import NotFittedError
2424
from sklearn.metrics import pairwise_distances as skl_pairwise_distances
@@ -82,6 +82,17 @@ def array_strategy(draw):
8282

8383

8484
@settings(deadline=None)
85+
@example(
86+
arrays=as_type(
87+
"numpy",
88+
np.array([[1.0, 2.0], [3.0, 4.0]]),
89+
np.array([[1.5, 2.5]]),
90+
None,
91+
),
92+
kernel="gaussian",
93+
metric="euclidean",
94+
bandwidth=1.0,
95+
)
8596
@given(
8697
array_strategy(),
8798
st.sampled_from(VALID_KERNELS),

python/cuml/cuml/tests/test_kernel_ridge.py

+21-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
#
1515
from cuml.testing.utils import as_type
1616
from hypothesis.extra.numpy import arrays
17-
from hypothesis import given, settings, assume, strategies as st
17+
from hypothesis import example, given, settings, assume, strategies as st
1818
from sklearn.kernel_ridge import KernelRidge as sklKernelRidge
1919
import inspect
2020
import math
@@ -187,6 +187,12 @@ def array_strategy(draw):
187187
return as_type(type, X, Y)
188188

189189

190+
@example(
191+
kernel_arg=("linear", {}),
192+
XY=as_type(
193+
"numpy", np.array([[1.0, 2.0], [3.0, 4.0]]), np.array([[1.5, 2.5]])
194+
),
195+
)
190196
@given(kernel_arg_strategy(), array_strategy())
191197
@settings(deadline=None)
192198
@pytest.mark.skip("https://github.com/rapidsai/cuml/issues/5177")
@@ -250,6 +256,20 @@ def estimator_array_strategy(draw):
250256
return (*as_type(type, X, y, X_test, alpha, sample_weight), dtype)
251257

252258

259+
@example(
260+
kernel_arg=("linear", {}),
261+
arrays=(
262+
np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]), # X
263+
np.array([1.0, 2.0, 3.0]), # y
264+
np.array([[2.0, 3.0], [4.0, 5.0]]), # X_test
265+
np.array([0.1]), # alpha
266+
None, # sample_weight
267+
np.float32, # dtype
268+
),
269+
gamma=1.0,
270+
degree=1,
271+
coef0=0.0,
272+
)
253273
@given(
254274
kernel_arg_strategy(),
255275
estimator_array_strategy(),

0 commit comments

Comments
 (0)