GH-86275: Implementation of hypothesis stubs for property-based tests, with zoneinfo tests (#22863)

These are stubs to be used for adding hypothesis (https://hypothesis.readthedocs.io/en/latest/) tests to the standard library.

When the tests are run in an environment where `hypothesis` and its various dependencies are not installed, the stubs will turn any tests with examples into simple parameterized tests and any tests without examples are skipped.

It also adds hypothesis tests for the `zoneinfo` module, and a Github Actions workflow to run the hypothesis tests as a non-required CI job.

The full hypothesis interface is not stubbed out — missing stubs can be added as necessary.

Co-authored-by: Zac Hatfield-Dodds <[email protected]>

Author: pganssle

.github/workflows/build.yml

@@ -36,6 +36,7 @@ jobs:
     timeout-minutes: 10
     outputs:
       run_tests: ${{ steps.check.outputs.run_tests }}
+      run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
     steps:
       - uses: actions/checkout@v3
       - name: Check for source changes
@@ -61,6 +62,17 @@ jobs:
             git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
           fi
 
+          # Check if we should run hypothesis tests
+          GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
+          echo $GIT_BRANCH
+          if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then
+            echo "Branch too old for hypothesis tests"
+            echo "run_hypothesis=false" >> $GITHUB_OUTPUT
+          else
+            echo "Run hypothesis tests"
+            echo "run_hypothesis=true" >> $GITHUB_OUTPUT
+          fi
+
   check_generated_files:
     name: 'Check if generated files are up to date'
     runs-on: ubuntu-latest
@@ -291,6 +303,90 @@ jobs:
     - name: SSL tests
       run: ./python Lib/test/ssltests.py
 
+  test_hypothesis:
+    name: "Hypothesis Tests on Ubuntu"
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
+    env:
+      OPENSSL_VER: 1.1.1t
+      PYTHONSTRICTEXTENSIONBUILD: 1
+    steps:
+    - uses: actions/checkout@v3
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install Dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+    - name: 'Restore OpenSSL build'
+      id: cache-openssl
+      uses: actions/cache@v3
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL
+      if: steps.cache-openssl.outputs.cache-hit != 'true'
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/[email protected]
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: Create directories for read-only out-of-tree builds
+      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+    - name: Bind mount sources read-only
+      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Configure CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+    - name: Build CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make -j4
+    - name: Display build info
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make pythoninfo
+    - name: Remount sources writable for tests
+      # some tests write to srcdir, lack of pyc files slows down testing
+      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: "Create hypothesis venv"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        VENV_LOC=$(realpath -m .)/hypovenv
+        VENV_PYTHON=$VENV_LOC/bin/python
+        echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
+        echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
+        ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis
+    - name: "Run tests"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        # Most of the excluded tests are slow test suites with no property tests
+        #
+        # (GH-104097) test_sysconfig is skipped because it has tests that are
+        # failing when executed from inside a virtual environment.
+        ${{ env.VENV_PYTHON }} -m test \
+          -W \
+          -x test_asyncio \
+          -x test_multiprocessing_fork \
+          -x test_multiprocessing_forkserver \
+          -x test_multiprocessing_spawn \
+          -x test_concurrent_futures \
+          -x test_socket \
+          -x test_subprocess \
+          -x test_signal \
+          -x test_sysconfig
+
 
   build_asan:
     name: 'Address sanitizer'

Lib/test/libregrtest/save_env.py

@@ -257,8 +257,10 @@ def restore_sysconfig__INSTALL_SCHEMES(self, saved):
         sysconfig._INSTALL_SCHEMES.update(saved[2])
 
     def get_files(self):
+        # XXX: Maybe add an allow-list here?
         return sorted(fn + ('/' if os.path.isdir(fn) else '')
-                      for fn in os.listdir())
+                      for fn in os.listdir()
+                      if not fn.startswith(".hypothesis"))
     def restore_files(self, saved_value):
         fn = os_helper.TESTFN
         if fn not in saved_value and (fn + '/') not in saved_value:

Lib/test/support/_hypothesis_stubs/__init__.py

@@ -0,0 +1,111 @@
+from enum import Enum
+import functools
+import unittest
+
+__all__ = [
+    "given",
+    "example",
+    "assume",
+    "reject",
+    "register_random",
+    "strategies",
+    "HealthCheck",
+    "settings",
+    "Verbosity",
+]
+
+from . import strategies
+
+
+def given(*_args, **_kwargs):
+    def decorator(f):
+        if examples := getattr(f, "_examples", []):
+
+            @functools.wraps(f)
+            def test_function(self):
+                for example_args, example_kwargs in examples:
+                    with self.subTest(*example_args, **example_kwargs):
+                        f(self, *example_args, **example_kwargs)
+
+        else:
+            # If we have found no examples, we must skip the test. If @example
+            # is applied after @given, it will re-wrap the test to remove the
+            # skip decorator.
+            test_function = unittest.skip(
+                "Hypothesis required for property test with no " +
+                "specified examples"
+            )(f)
+
+        test_function._given = True
+        return test_function
+
+    return decorator
+
+
+def example(*args, **kwargs):
+    if bool(args) == bool(kwargs):
+        raise ValueError("Must specify exactly one of *args or **kwargs")
+
+    def decorator(f):
+        base_func = getattr(f, "__wrapped__", f)
+        if not hasattr(base_func, "_examples"):
+            base_func._examples = []
+
+        base_func._examples.append((args, kwargs))
+
+        if getattr(f, "_given", False):
+            # If the given decorator is below all the example decorators,
+            # it would be erroneously skipped, so we need to re-wrap the new
+            # base function.
+            f = given()(base_func)
+
+        return f
+
+    return decorator
+
+
+def assume(condition):
+    if not condition:
+        raise unittest.SkipTest("Unsatisfied assumption")
+    return True
+
+
+def reject():
+    assume(False)
+
+
+def register_random(*args, **kwargs):
+    pass  # pragma: no cover
+
+
+def settings(*args, **kwargs):
+    return lambda f: f  # pragma: nocover
+
+
+class HealthCheck(Enum):
+    data_too_large = 1
+    filter_too_much = 2
+    too_slow = 3
+    return_value = 5
+    large_base_example = 7
+    not_a_test_method = 8
+
+    @classmethod
+    def all(cls):
+        return list(cls)
+
+
+class Verbosity(Enum):
+    quiet = 0
+    normal = 1
+    verbose = 2
+    debug = 3
+
+
+class Phase(Enum):
+    explicit = 0
+    reuse = 1
+    generate = 2
+    target = 3
+    shrink = 4
+    explain = 5

Lib/test/support/_hypothesis_stubs/_helpers.py

@@ -0,0 +1,43 @@
+# Stub out only the subset of the interface that we actually use in our tests.
+class StubClass:
+    def __init__(self, *args, **kwargs):
+        self.__stub_args = args
+        self.__stub_kwargs = kwargs
+        self.__repr = None
+
+    def _with_repr(self, new_repr):
+        new_obj = self.__class__(*self.__stub_args, **self.__stub_kwargs)
+        new_obj.__repr = new_repr
+        return new_obj
+
+    def __repr__(self):
+        if self.__repr is not None:
+            return self.__repr
+
+        argstr = ", ".join(self.__stub_args)
+        kwargstr = ", ".join(f"{kw}={val}" for kw, val in self.__stub_kwargs.items())
+
+        in_parens = argstr
+        if kwargstr:
+            in_parens += ", " + kwargstr
+
+        return f"{self.__class__.__qualname__}({in_parens})"
+
+
+def stub_factory(klass, name, *, with_repr=None, _seen={}):
+    if (klass, name) not in _seen:
+
+        class Stub(klass):
+            def __init__(self, *args, **kwargs):
+                super().__init__()
+                self.__stub_args = args
+                self.__stub_kwargs = kwargs
+
+        Stub.__name__ = name
+        Stub.__qualname__ = name
+        if with_repr is not None:
+            Stub._repr = None
+
+        _seen.setdefault((klass, name, with_repr), Stub)
+
+    return _seen[(klass, name, with_repr)]

Lib/test/support/_hypothesis_stubs/strategies.py

@@ -0,0 +1,91 @@
+import functools
+
+from ._helpers import StubClass, stub_factory
+
+
+class StubStrategy(StubClass):
+    def __make_trailing_repr(self, transformation_name, func):
+        func_name = func.__name__ or repr(func)
+        return f"{self!r}.{transformation_name}({func_name})"
+
+    def map(self, pack):
+        return self._with_repr(self.__make_trailing_repr("map", pack))
+
+    def flatmap(self, expand):
+        return self._with_repr(self.__make_trailing_repr("flatmap", expand))
+
+    def filter(self, condition):
+        return self._with_repr(self.__make_trailing_repr("filter", condition))
+
+    def __or__(self, other):
+        new_repr = f"one_of({self!r}, {other!r})"
+        return self._with_repr(new_repr)
+
+
+_STRATEGIES = {
+    "binary",
+    "booleans",
+    "builds",
+    "characters",
+    "complex_numbers",
+    "composite",
+    "data",
+    "dates",
+    "datetimes",
+    "decimals",
+    "deferred",
+    "dictionaries",
+    "emails",
+    "fixed_dictionaries",
+    "floats",
+    "fractions",
+    "from_regex",
+    "from_type",
+    "frozensets",
+    "functions",
+    "integers",
+    "iterables",
+    "just",
+    "lists",
+    "none",
+    "nothing",
+    "one_of",
+    "permutations",
+    "random_module",
+    "randoms",
+    "recursive",
+    "register_type_strategy",
+    "runner",
+    "sampled_from",
+    "sets",
+    "shared",
+    "slices",
+    "timedeltas",
+    "times",
+    "text",
+    "tuples",
+    "uuids",
+}
+
+__all__ = sorted(_STRATEGIES)
+
+
+def composite(f):
+    strategy = stub_factory(StubStrategy, f.__name__)
+
+    @functools.wraps(f)
+    def inner(*args, **kwargs):
+        return strategy(*args, **kwargs)
+
+    return inner
+
+
+def __getattr__(name):
+    if name not in _STRATEGIES:
+        raise AttributeError(f"Unknown attribute {name}")
+
+    return stub_factory(StubStrategy, f"hypothesis.strategies.{name}")
+
+
+def __dir__():
+    return __all__

Lib/test/support/hypothesis_helper.py

@@ -0,0 +1,4 @@
+try:
+    import hypothesis
+except ImportError:
+    from . import _hypothesis_stubs as hypothesis

Lib/test/test_zoneinfo/__init__.py

@@ -1 +1,2 @@
 from .test_zoneinfo import *
+from .test_zoneinfo_property import *