From 4e533f7d91da969896feac8aaf83ca3cee79efe5 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Mon, 9 Jun 2025 19:22:40 -0300 Subject: [PATCH] Fix compatibility with Twisted 25 As discussed in https://github.com/pytest-dev/pytest/pull/13502, the fix for compatibility with Twisted 25+ is simpler. Therefore, it makes sense to implement both fixes (for Twisted 24 and Twisted 25) in parallel. This way, we can eventually drop support for Twisted <25 and keep only the simpler workaround. In addition, the `unittestextras` tox environment has been replaced with dedicated test environments for `asynctest`, `Twisted 24`, and `Twisted 25`. Fixes #13497 --- .github/workflows/test.yml | 43 +++++++++- changelog/13497.bugfix.rst | 1 + src/_pytest/unittest.py | 155 ++++++++++++++++++++++++++++--------- tox.ini | 19 +++-- 4 files changed, 174 insertions(+), 44 deletions(-) create mode 100644 changelog/13497.bugfix.rst diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ecc133878f..7873b66a49a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,7 +54,9 @@ jobs: fail-fast: false matrix: name: [ - "windows-py39-unittestextras", + "windows-py39-unittest-asynctest", + "windows-py39-unittest-twisted24", + "windows-py39-unittest-twisted25", "windows-py39-pluggy", "windows-py39-xdist", "windows-py310", @@ -63,6 +65,9 @@ jobs: "windows-py313", "windows-py314", + "ubuntu-py39-unittest-asynctest", + "ubuntu-py39-unittest-twisted24", + "ubuntu-py39-unittest-twisted25", "ubuntu-py39-lsof-numpy-pexpect", "ubuntu-py39-pluggy", "ubuntu-py39-freeze", @@ -85,10 +90,23 @@ jobs: ] include: - - name: "windows-py39-unittestextras" + # Use separate jobs for different unittest flavors (twisted, asynctest) to ensure proper coverage. + - name: "windows-py39-unittest-asynctest" python: "3.9" os: windows-latest - tox_env: "py39-unittestextras" + tox_env: "py39-asynctest" + use_coverage: true + + - name: "windows-py39-unittest-twisted24" + python: "3.9" + os: windows-latest + tox_env: "py39-twisted24" + use_coverage: true + + - name: "windows-py39-unittest-twisted25" + python: "3.9" + os: windows-latest + tox_env: "py39-twisted25" use_coverage: true - name: "windows-py39-pluggy" @@ -126,6 +144,25 @@ jobs: os: windows-latest tox_env: "py314" + # Use separate jobs for different unittest flavors (twisted, asynctest) to ensure proper coverage. + - name: "ubuntu-py39-unittest-asynctest" + python: "3.9" + os: ubuntu-latest + tox_env: "py39-asynctest" + use_coverage: true + + - name: "ubuntu-py39-unittest-twisted24" + python: "3.9" + os: ubuntu-latest + tox_env: "py39-twisted24" + use_coverage: true + + - name: "ubuntu-py39-unittest-twisted25" + python: "3.9" + os: ubuntu-latest + tox_env: "py39-twisted25" + use_coverage: true + - name: "ubuntu-py39-lsof-numpy-pexpect" python: "3.9" os: ubuntu-latest diff --git a/changelog/13497.bugfix.rst b/changelog/13497.bugfix.rst new file mode 100644 index 00000000000..75b4996e5af --- /dev/null +++ b/changelog/13497.bugfix.rst @@ -0,0 +1 @@ +Fixed compatibility with ``Twisted 25+``. diff --git a/src/_pytest/unittest.py b/src/_pytest/unittest.py index 27807274168..b18ac56b811 100644 --- a/src/_pytest/unittest.py +++ b/src/_pytest/unittest.py @@ -6,11 +6,13 @@ from collections.abc import Callable from collections.abc import Generator from collections.abc import Iterable +from collections.abc import Iterator +from enum import auto +from enum import Enum import inspect import sys import traceback import types -from typing import Any from typing import TYPE_CHECKING from typing import Union @@ -18,6 +20,7 @@ from _pytest.compat import is_async_function from _pytest.config import hookimpl from _pytest.fixtures import FixtureRequest +from _pytest.monkeypatch import MonkeyPatch from _pytest.nodes import Collector from _pytest.nodes import Item from _pytest.outcomes import exit @@ -228,8 +231,7 @@ def startTest(self, testcase: unittest.TestCase) -> None: pass def _addexcinfo(self, rawexcinfo: _SysExcInfoType) -> None: - # Unwrap potential exception info (see twisted trial support below). - rawexcinfo = getattr(rawexcinfo, "_rawexcinfo", rawexcinfo) + rawexcinfo = _handle_twisted_exc_info(rawexcinfo) try: excinfo = _pytest._code.ExceptionInfo[BaseException].from_exc_info( rawexcinfo # type: ignore[arg-type] @@ -373,49 +375,130 @@ def pytest_runtest_makereport(item: Item, call: CallInfo[None]) -> None: pass -# Twisted trial support. -classImplements_has_run = False +def _is_skipped(obj) -> bool: + """Return True if the given object has been marked with @unittest.skip.""" + return bool(getattr(obj, "__unittest_skip__", False)) + + +def pytest_configure() -> None: + """Register the TestCaseFunction class as an IReporter if twisted.trial is available.""" + if _get_twisted_version() is not TwistedVersion.NotInstalled: + from twisted.trial.itrial import IReporter + from zope.interface import classImplements + + classImplements(TestCaseFunction, IReporter) + + +class TwistedVersion(Enum): + """ + The Twisted version installed in the environment. + + We have different workarounds in place for different versions of Twisted. + """ + + # Twisted version 24 or prior. + Version24 = auto() + # Twisted version 25 or later. + Version25 = auto() + # Twisted version is not available. + NotInstalled = auto() + + +def _get_twisted_version() -> TwistedVersion: + # We need to check if "twisted.trial.unittest" is specifically present in sys.modules. + # This is because we intend to integrate with Trial only when it's actively running + # the test suite, but not needed when only other Twisted components are in use. + if "twisted.trial.unittest" not in sys.modules: + return TwistedVersion.NotInstalled + + import importlib.metadata + + import packaging.version + + version_str = importlib.metadata.version("twisted") + version = packaging.version.parse(version_str) + if version.major <= 24: + return TwistedVersion.Version24 + else: + return TwistedVersion.Version25 + + +# Name of the attribute in `twisted.python.Failure` instances that stores +# the `sys.exc_info()` tuple. +# See twisted.trial support in `pytest_runtest_protocol`. +TWISTED_RAW_EXCINFO_ATTR = "_twisted_raw_excinfo" @hookimpl(wrapper=True) -def pytest_runtest_protocol(item: Item) -> Generator[None, object, object]: - if isinstance(item, TestCaseFunction) and "twisted.trial.unittest" in sys.modules: - ut: Any = sys.modules["twisted.python.failure"] - global classImplements_has_run - Failure__init__ = ut.Failure.__init__ - if not classImplements_has_run: - from twisted.trial.itrial import IReporter - from zope.interface import classImplements - - classImplements(TestCaseFunction, IReporter) - classImplements_has_run = True - - def excstore( +def pytest_runtest_protocol(item: Item) -> Iterator[None]: + if _get_twisted_version() is TwistedVersion.Version24: + import twisted.python.failure as ut + + # Monkeypatch `Failure.__init__` to store the raw exception info. + original__init__ = ut.Failure.__init__ + + def store_raw_exception_info( self, exc_value=None, exc_type=None, exc_tb=None, captureVars=None - ): + ): # pragma: no cover if exc_value is None: - self._rawexcinfo = sys.exc_info() + raw_exc_info = sys.exc_info() else: if exc_type is None: exc_type = type(exc_value) - self._rawexcinfo = (exc_type, exc_value, exc_tb) + if exc_tb is None: + exc_tb = sys.exc_info()[2] + raw_exc_info = (exc_type, exc_value, exc_tb) + setattr(self, TWISTED_RAW_EXCINFO_ATTR, tuple(raw_exc_info)) try: - Failure__init__( + original__init__( self, exc_value, exc_type, exc_tb, captureVars=captureVars ) - except TypeError: - Failure__init__(self, exc_value, exc_type, exc_tb) + except TypeError: # pragma: no cover + original__init__(self, exc_value, exc_type, exc_tb) - ut.Failure.__init__ = excstore - try: - res = yield - finally: - ut.Failure.__init__ = Failure__init__ + with MonkeyPatch.context() as patcher: + patcher.setattr(ut.Failure, "__init__", store_raw_exception_info) + return (yield) else: - res = yield - return res - - -def _is_skipped(obj) -> bool: - """Return True if the given object has been marked with @unittest.skip.""" - return bool(getattr(obj, "__unittest_skip__", False)) + return (yield) + + +def _handle_twisted_exc_info( + rawexcinfo: _SysExcInfoType | BaseException, +) -> _SysExcInfoType: + """ + Twisted passes a custom Failure instance to `addError()` instead of using `sys.exc_info()`. + Therefore, if `rawexcinfo` is a `Failure` instance, convert it into the equivalent `sys.exc_info()` tuple + as expected by pytest. + """ + twisted_version = _get_twisted_version() + if twisted_version is TwistedVersion.NotInstalled: + # Unfortunately, because we cannot import `twisted.python.failure` at the top of the file + # and use it in the signature, we need to use `type:ignore` here because we cannot narrow + # the type properly in the `if` statement above. + return rawexcinfo # type:ignore[return-value] + elif twisted_version is TwistedVersion.Version24: + # Twisted calls addError() passing its own classes (like `twisted.python.Failure`), which violates + # the `addError()` signature, so we extract the original `sys.exc_info()` tuple which is stored + # in the object. + if hasattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR): + saved_exc_info = getattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR) + # Delete the attribute from the original object to avoid leaks. + delattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR) + return saved_exc_info # type:ignore[no-any-return] + return rawexcinfo # type:ignore[return-value] + elif twisted_version is TwistedVersion.Version25: + if isinstance(rawexcinfo, BaseException): + import twisted.python.failure + + if isinstance(rawexcinfo, twisted.python.failure.Failure): + tb = rawexcinfo.__traceback__ + if tb is None: + tb = sys.exc_info()[2] + return type(rawexcinfo.value), rawexcinfo.value, tb + + return rawexcinfo # type:ignore[return-value] + else: + # Ideally we would use assert_never() here, but it is not available in all Python versions + # we support, plus we do not require `type_extensions` currently. + assert False, f"Unexpected Twisted version: {twisted_version}" diff --git a/tox.ini b/tox.ini index 8f7d8495285..f1283aa8260 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ envlist = py313 py314 pypy3 - py39-{pexpect,xdist,unittestextras,numpy,pluggymain,pylib} + py39-{pexpect,xdist,twisted24,twisted25,asynctest,numpy,pluggymain,pylib} doctesting doctesting-coverage plugins @@ -36,7 +36,9 @@ description = pexpect: against `pexpect` pluggymain: against the bleeding edge `pluggy` from Git pylib: against `py` lib - unittestextras: against the unit test extras + twisted24: against the unit test extras with twisted prior to 24.0 + twisted25: against the unit test extras with twisted 25.0 or later + asynctest: against the unit test extras with asynctest xdist: with pytest in parallel mode under `{basepython}` doctesting: including doctests @@ -51,7 +53,7 @@ passenv = TERM SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST setenv = - _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} + _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} {env:_PYTEST_FILES:} # See https://docs.python.org/3/library/io.html#io-encoding-warning # If we don't enable this, neither can any of our downstream users! @@ -66,6 +68,12 @@ setenv = doctesting: _PYTEST_TOX_POSARGS_DOCTESTING=doc/en + # The configurations below are related only to standard unittest support. + # Run only tests from test_unittest.py. + asynctest: _PYTEST_FILES=testing/test_unittest.py + twisted24: _PYTEST_FILES=testing/test_unittest.py + twisted25: _PYTEST_FILES=testing/test_unittest.py + nobyte: PYTHONDONTWRITEBYTECODE=1 lsof: _PYTEST_TOX_POSARGS_LSOF=--lsof @@ -79,8 +87,9 @@ deps = pexpect: pexpect>=4.8.0 pluggymain: pluggy @ git+https://github.com/pytest-dev/pluggy.git pylib: py>=1.8.2 - unittestextras: twisted - unittestextras: asynctest + twisted24: twisted<25 + twisted25: twisted>=25 + asynctest: asynctest xdist: pytest-xdist>=2.1.0 xdist: -e . {env:_PYTEST_TOX_EXTRA_DEP:}