From 6144fdbd5a5ecaa436c1bddea569c971492a41b3 Mon Sep 17 00:00:00 2001 From: Zac-HD Date: Sun, 6 Jan 2019 15:16:13 +1100 Subject: [PATCH] Hook for managing any Random instance This generalises our management of the global random and numpy.random states to work for any registered Random instance. That's useful for simulation and scheduling frameworks that maintain their own random state, such as Trio. --- hypothesis-python/RELEASE.rst | 9 +++ .../src/hypothesis/_strategies.py | 27 +++++--- .../src/hypothesis/internal/entropy.py | 64 ++++++++++++++++--- .../tests/cover/test_random_module.py | 54 +++++++++++++++- 4 files changed, 134 insertions(+), 20 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 00000000000..f85fcc045e5 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,9 @@ +RELEASE_TYPE: minor + +This release adds :func:`random_module.register `, +which registers ``random.Random`` instances to be seeded and reset by +Hypothesis to ensure that test cases are deterministic. + +We still recommend explicitly passing a ``random.Random`` instance from +:func:`~hypothesis.strategies.randoms` if possible, but registering a +framework-global state for Hypothesis to manage is better than flaky tests! diff --git a/hypothesis-python/src/hypothesis/_strategies.py b/hypothesis-python/src/hypothesis/_strategies.py index c49360fb2b7..790b6651911 100644 --- a/hypothesis-python/src/hypothesis/_strategies.py +++ b/hypothesis-python/src/hypothesis/_strategies.py @@ -21,7 +21,6 @@ import enum import math import operator -import random import string import sys from decimal import Context, Decimal, localcontext @@ -56,6 +55,7 @@ choice, integer_range, ) +from hypothesis.internal.entropy import get_seeder_and_restorer, register from hypothesis.internal.floats import ( count_between_floats, float_of, @@ -135,6 +135,7 @@ numpy = None if False: + import random # noqa from types import ModuleType # noqa from typing import Any, Dict, Union, Sequence, Callable, Pattern # noqa from typing import TypeVar, Tuple, List, Set, FrozenSet, overload # noqa @@ -1191,26 +1192,23 @@ def __init__(self, seed): self.seed = seed def __repr__(self): - return "random.seed(%r)" % (self.seed,) + return "RandomSeeder(%r)" % (self.seed,) class RandomModule(SearchStrategy): def do_draw(self, data): data.can_reproduce_example_from_repr = False seed = data.draw(integers(0, 2 ** 32 - 1)) - state = random.getstate() - random.seed(seed) - cleanup(lambda: random.setstate(state)) - if numpy is not None: # pragma: no cover - npstate = numpy.random.get_state() - numpy.random.seed(seed) - cleanup(lambda: numpy.random.set_state(npstate)) + seed_all, restore_all = get_seeder_and_restorer(seed) + seed_all() + cleanup(restore_all) return RandomSeeder(seed) @cacheable @defines_strategy def random_module(): + # type: () -> SearchStrategy[RandomSeeder] """The Hypothesis engine handles PRNG state for the stdlib and Numpy random modules internally, always seeding them to zero and restoring the previous state after the test. @@ -1223,10 +1221,21 @@ def random_module(): If ``numpy.random`` is available, that state is also managed. Examples from these strategy shrink to seeds closer to zero. + + You can pass additional ``random.Random`` instances to + ``random_module.register(r)`` to have their states seeded and restored + in the same way. All global random states, from e.g. simulation or + scheduling frameworks, should be registered to prevent flaky tests. + + Registered Randoms are also zeroed by the engine for any Hypothesis + tests that do not use the ``random_module`` strategy. """ return shared(RandomModule(), "hypothesis.strategies.random_module()") +setattr(random_module, "register", register) + + @cacheable @defines_strategy def builds( diff --git a/hypothesis-python/src/hypothesis/internal/entropy.py b/hypothesis-python/src/hypothesis/internal/entropy.py index 98b12f67d61..f4d84258e1f 100644 --- a/hypothesis-python/src/hypothesis/internal/entropy.py +++ b/hypothesis-python/src/hypothesis/internal/entropy.py @@ -20,10 +20,60 @@ import contextlib import random +from hypothesis.internal.compat import integer_types +from hypothesis.internal.validation import check_type + +RANDOMS_TO_MANAGE = [random] # type: list + try: import numpy.random as npr except ImportError: - npr = None + pass +else: + + class NumpyRandomWrapper(object): + """A shim to remove those darn underscores.""" + + seed = npr.seed + getstate = npr.get_state + setstate = npr.set_state + + RANDOMS_TO_MANAGE.append(NumpyRandomWrapper) + + +def register(r): + # type: (random.Random) -> None + """Register the given Random instance for management by Hypothesis.""" + check_type(random.Random, r, "r") + if r not in RANDOMS_TO_MANAGE: + RANDOMS_TO_MANAGE.append(r) + + +def get_seeder_and_restorer(seed=0): + """Return a pair of functions which respectively seed all and restore + the state of all registered PRNGs. + + This is used by the core engine via `deterministic_PRNG`, and by users + via `random_module.register`. We support registration of additional + random.Random instances for simulation or scheduling frameworks which + avoid using the global random state. See e.g. #1709. + """ + assert isinstance(seed, integer_types) and 0 <= seed < 2 ** 32 + states = [] # type: list + + def seed_all(): + assert not states + for r in RANDOMS_TO_MANAGE: + states.append(r.getstate()) + r.seed(seed) + + def restore_all(): + assert len(states) == len(RANDOMS_TO_MANAGE) + for r, state in zip(RANDOMS_TO_MANAGE, states): + r.setstate(state) + del states[:] + + return seed_all, restore_all @contextlib.contextmanager @@ -35,15 +85,9 @@ def deterministic_PRNG(): bad idea in principle, and breaks all kinds of independence assumptions in practice. """ - _random_state = random.getstate() - random.seed(0) - # These branches are covered by tests/numpy/, not tests/cover/ - if npr is not None: # pragma: no cover - _npr_state = npr.get_state() - npr.seed(0) + seed_all, restore_all = get_seeder_and_restorer() + seed_all() try: yield finally: - random.setstate(_random_state) - if npr is not None: # pragma: no cover - npr.set_state(_npr_state) + restore_all() diff --git a/hypothesis-python/tests/cover/test_random_module.py b/hypothesis-python/tests/cover/test_random_module.py index 4da2e505cf1..664312c6b46 100644 --- a/hypothesis-python/tests/cover/test_random_module.py +++ b/hypothesis-python/tests/cover/test_random_module.py @@ -23,6 +23,8 @@ import hypothesis.strategies as st from hypothesis import given, reporting +from hypothesis.errors import InvalidArgument +from hypothesis.internal import entropy from tests.common.utils import capture_out @@ -36,7 +38,7 @@ def test(r): assert False test() - assert "random.seed(0)" in out.getvalue() + assert "RandomSeeder(0)" in out.getvalue() @given(st.random_module(), st.random_module()) @@ -47,3 +49,53 @@ def test_seed_random_twice(r, r2): @given(st.random_module()) def test_does_not_fail_health_check_if_randomness_is_used(r): random.getrandbits(128) + + +def test_cannot_register_non_Random(): + with pytest.raises(InvalidArgument): + st.random_module.register("not a Random instance") + + +def test_registering_a_Random_is_idempotent(): + r = random.Random() + st.random_module.register(r) + st.random_module.register(r) + assert entropy.RANDOMS_TO_MANAGE.pop() is r + assert r not in entropy.RANDOMS_TO_MANAGE + + +def test_manages_registered_Random_instance(): + r = random.Random() + st.random_module.register(r) + state = r.getstate() + result = [] + + @given(st.integers()) + def inner(x): + v = r.random() + if result: + assert v == result[0] + else: + result.append(v) + + inner() + entropy.RANDOMS_TO_MANAGE.remove(r) + assert state == r.getstate() + + +def test_registered_Random_is_seeded_by_random_module_strategy(): + r = random.Random() + st.random_module.register(r) + state = r.getstate() + results = set() + count = [0] + + @given(st.integers()) + def inner(x): + results.add(r.random()) + count[0] += 1 + + inner() + assert count[0] > len(results) * 0.9, "too few unique random numbers" + entropy.RANDOMS_TO_MANAGE.remove(r) + assert state == r.getstate()